From 5ca023056243a33ad65cb53b335432e458f06bb6 Mon Sep 17 00:00:00 2001 From: CrazyMax <1951866+crazy-max@users.noreply.github.com> Date: Mon, 25 May 2020 14:08:12 +0200 Subject: [PATCH] Move static to file provider (#71) --- README.md | 4 +- cmd/main.go | 4 +- doc/configuration.md | 66 +-------- doc/faq.md | 42 ++---- doc/notifications.md | 5 +- doc/providers/docker.md | 20 ++- doc/providers/file.md | 171 ++++++++++++++++++++++ doc/providers/static.md | 54 ------- doc/providers/swarm.md | 19 ++- internal/app/diun.go | 6 +- internal/config/config.go | 30 ++-- internal/config/config.test.yml | 34 +---- internal/config/config_test.go | 69 +-------- internal/config/dummy.yml | 0 internal/model/providers.go | 9 +- internal/provider/docker/docker.go | 15 +- internal/provider/file/file.go | 103 +++++++++++++ internal/provider/file/file_test.go | 127 ++++++++++++++++ internal/provider/file/test/bintray.yml | 5 + internal/provider/file/test/dockerhub.yml | 24 +++ internal/provider/file/test/quay.yml | 2 + internal/provider/static/static.go | 40 ----- internal/provider/swarm/swarm.go | 15 +- 23 files changed, 543 insertions(+), 321 deletions(-) create mode 100644 doc/providers/file.md delete mode 100644 doc/providers/static.md create mode 100644 internal/config/dummy.yml create mode 100644 internal/provider/file/file.go create mode 100644 internal/provider/file/file_test.go create mode 100644 internal/provider/file/test/bintray.yml create mode 100644 internal/provider/file/test/dockerhub.yml create mode 100644 internal/provider/file/test/quay.yml delete mode 100644 internal/provider/static/static.go diff --git a/README.md b/README.md index f8dc89f2..405567cc 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ * Internal cron implementation through go routines * Worker pool to parallelize analyses * Allow overriding image os and architecture -* Multi providers available like [Docker](doc/providers/docker.md), [Swarm](doc/providers/swarm.md), [Static](doc/providers/static.md)... +* Multi providers available like [Docker](doc/providers/docker.md), [Swarm](doc/providers/swarm.md), [File](doc/providers/file.md)... * Get notified through Slack, Mail, Telegram and [more](doc/notifications.md) * Enhanced logging * Timezone can be changed @@ -42,7 +42,7 @@ * Providers * [Docker](doc/providers/docker.md) * [Swarm](doc/providers/swarm.md) - * [Static](doc/providers/static.md) + * [File](doc/providers/file.md) * [Notifications](doc/notifications.md) * [FAQ](doc/faq.md) diff --git a/cmd/main.go b/cmd/main.go index 973dc9a6..ae785a4d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -58,12 +58,12 @@ func main() { os.Exit(0) }() - // Load and check configuration + // Load configuration cfg, err := config.Load(cli, version) if err != nil { log.Fatal().Err(err).Msg("Cannot load configuration") } - cfg.Display() + log.Debug().Msg(cfg.Display()) // Init if diun, err = app.New(cfg, location); err != nil { diff --git a/doc/configuration.md b/doc/configuration.md index 7a173db5..ea716180 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -86,36 +86,11 @@ providers: # Watch all services on local Swarm cluster myswarm: watch_by_default: true - static: - # Watch latest tag of crazymax/nextcloud image on docker.io (DockerHub) with registry ID 'someregistryoptions'. - - name: docker.io/crazymax/nextcloud:latest - regopts_id: someregistryoptions - # Watch 4.0.0 tag of jfrog/artifactory-oss image on frog-docker-reg2.bintray.io (Bintray) with registry ID 'onemore'. - - name: jfrog-docker-reg2.bintray.io/jfrog/artifactory-oss:4.0.0 - regopts_id: onemore - # Watch coreos/hyperkube image on quay.io (Quay) and assume latest tag. - - name: quay.io/coreos/hyperkube - # Watch crazymax/swarm-cronjob image and assume docker.io registry and latest tag. - # Only include tags matching regexp ^1\.2\..* - - name: crazymax/swarm-cronjob - watch_repo: true - include_tags: - - ^1\.2\..* - # Watch portainer/portainer image on docker.io (DockerHub) and assume latest tag - # Only watch latest 10 tags and include tags matching regexp ^(0|[1-9]\d*)\..* - - name: docker.io/portainer/portainer - watch_repo: true - max_tags: 10 - include_tags: - - ^(0|[1-9]\d*)\..* - # Watch alpine image (library) and assume docker.io registry and latest tag. - # Force linux/arm64/v8 platform for this image - - name: alpine - watch_repo: true - platform: - os: linux - arch: arm64 - variant: v8 + file: + # Watch images from filename ./myimages.yml + filename: ./myimages.yml + # Watch images from directory ./imagesdir + directory: ./imagesdir ``` ## Reference @@ -187,31 +162,6 @@ providers: ### providers -* `docker`: Map of Docker standalone engines to watch - * ``: An unique identifier for this provider. - * `endpoint`: Server address to connect to. Local if empty. - * `api_version`: Overrides the client version with the specified one. - * `tls_certs_path`: Path to load the TLS certificates from. - * `tls_verify`: Controls whether client verifies the server's certificate chain and hostname (default: `true`). - * `watch_by_default`: Enable watch by default. If false, containers that don't have `diun.enable=true` label will be ignored (default: `false`). - * `watch_stopped`: Include created and exited containers too (default: `false`). - -* `swarm`: Map of Docker Swarm to watch - * ``: An unique identifier for this provider. - * `endpoint`: Server address to connect to. Local if empty. - * `api_version`: Overrides the client version with the specified one. - * `tls_certs_path`: Path to load the TLS certificates from. - * `tls_verify`: Controls whether client verifies the server's certificate chain and hostname (default: `true`). - * `watch_by_default`: Enable watch by default. If false, services that don't have `diun.enable=true` label will be ignored (default: `false`). - -* `static`: Slice of static image to watch - * `name`: Docker image name to watch using `registry/path:tag` format. If registry is omitted, `docker.io` will be used and if tag is omitted, `latest` will be used. **required** - * `regopts_id`: Registry options ID from `regopts` to use. - * `watch_repo`: Watch all tags of this `image` repository (default: `false`). - * `max_tags`: Maximum number of tags to watch if `watch_repo` enabled. 0 means all of them (default: `0`). - * `include_tags`: List of regular expressions to include tags. Can be useful if you enable `watch_repo`. - * `exclude_tags`: List of regular expressions to exclude tags. Can be useful if you enable `watch_repo`. - * `platform`: Check a custom platform. (default is retrieved dynamically based on your operating system). - * `os`: Operating system to use. - * `arch`: CPU architecture to use. - * `variant`: Variant of the CPU to use. +* [docker](providers/docker.md) +* [swarm](providers/swarm.md) +* [file](providers/file.md) diff --git a/doc/faq.md b/doc/faq.md index c1735bb7..789e50ec 100644 --- a/doc/faq.md +++ b/doc/faq.md @@ -4,52 +4,32 @@ ## No image found in manifest list for architecture [], variant [], OS [] -If you encounter this kind of error, you are probably using the [static provider](providers/static.md) containing an image with an erroneous or empty platform. If the platform is not filled in, it will be deduced automatically from the information of your operating system on which Diun is running. +If you encounter this kind of error, you are probably using the [file provider](providers/file.md) containing an image with an erroneous or empty platform. If the platform is not filled in, it will be deduced automatically from the information of your operating system on which Diun is running. In the example below, Diun is running (`diun_x.x.x_windows_i386.zip`) on Windows 10 and tries to analyze the `crazymax/cloudflared` image with the detected platform (`windows/386)`: ```yml -db: - path: diun.db - -watch: - workers: 20 - schedule: "* * * * *" - first_check_notif: true - -providers: - static: - - name: crazymax/cloudflared:2020.2.1 - watch_repo: true +- name: crazymax/cloudflared:2020.2.1 + watch_repo: true ``` But this platform is not supported by this image as you can see [on DockerHub](https://hub.docker.com/layers/crazymax/cloudflared/2020.2.1/images/sha256-137eea4e84ec4c6cb5ceb2017b9788dcd7b04f135d756e1f37e3e6673c0dd9d2?context=explore): ``` -Fri, 27 Mar 2020 01:20:03 UTC ERR Cannot run job error="Error choosing image instance: no image found in manifest list for architecture 386, variant , OS windows" provider=static-0 -Fri, 27 Mar 2020 01:20:03 UTC ERR Cannot list tags from registry error="Error choosing image instance: no image found in manifest list for architecture 386, variant , OS windows" image=crazymax/cloudflared:2020.2.1 provider=static-0 +Fri, 27 Mar 2020 01:20:03 UTC ERR Cannot run job error="Error choosing image instance: no image found in manifest list for architecture 386, variant , OS windows" provider=file +Fri, 27 Mar 2020 01:20:03 UTC ERR Cannot list tags from registry error="Error choosing image instance: no image found in manifest list for architecture 386, variant , OS windows" image=crazymax/cloudflared:2020.2.1 provider=file ``` You have to force the platform for this image if you are not on a supported platform. For example: ```yml -db: - path: diun.db - -watch: - workers: 20 - schedule: "* * * * *" - first_check_notif: true - -providers: - static: - - name: crazymax/cloudflared:2020.2.1 - watch_repo: true - platform: - os: linux - arch: amd64 +- name: crazymax/cloudflared:2020.2.1 + watch_repo: true + platform: + os: linux + arch: amd64 ``` ``` -Fri, 27 Mar 2020 01:24:33 UTC INF New image found image=docker.io/crazymax/cloudflared:2020.2.1 provider=static-0 +Fri, 27 Mar 2020 01:24:33 UTC INF New image found image=docker.io/crazymax/cloudflared:2020.2.1 provider=file ``` diff --git a/doc/notifications.md b/doc/notifications.md index ed8b7c42..71567dc0 100644 --- a/doc/notifications.md +++ b/doc/notifications.md @@ -50,12 +50,11 @@ If you choose `webhook` notification, a HTTP request is sent with a JSON format { "diun_version": "0.3.0", "status": "new", - "provider": "static-0", + "provider": "file", "image": "docker.io/crazymax/swarm-cronjob:0.2.1", "mime_type": "application/vnd.docker.distribution.manifest.v2+json", "digest": "sha256:5913d4b5e8dc15430c2f47f40e43ab2ca7f2b8df5eee5db4d5c42311e08dfb79", "created": "2019-01-24T10:26:49.152006005Z", - "architecture": "amd64", - "os": "linux" + "platform": "linux/amd64" } ``` diff --git a/doc/providers/docker.md b/doc/providers/docker.md index 9a96579d..9334f2e4 100644 --- a/doc/providers/docker.md +++ b/doc/providers/docker.md @@ -2,11 +2,12 @@ * [About](#about) * [Quick start](#quick-start) -* [Configuration](#configuration) +* [Provider configuration](#provider-configuration) +* [Docker labels](#docker-labels) ## About -The Docker provider allows you to analyze the containers of your standalone Docker instance defined in the [Diun configuration](../configuration.md#providers) to extract the images found and check for updates on the registry. +The Docker provider allows you to analyze the containers of your standalone Docker instance to extract the images found and check for updates on the registry. ## Quick start @@ -88,9 +89,20 @@ diun_1 | Sat, 14 Dec 2019 15:30:13 CET INF Cron initialized with schedul diun_1 | Sat, 14 Dec 2019 15:30:13 CET INF Next run in 29 minutes (2019-12-14 16:00:00 +0100 CET) ``` -## Configuration +## Provider configuration -In the same spirit as the [static provider](static.md), you can configure more finely the way to analyze the image of your container. But unlike the static provider, this is done via Docker labels: +The Docker provider configuration is map of Docker standalone engines to watch with the following options available: + +* `endpoint`: Server address to connect to. Local if empty. +* `api_version`: Overrides the client version with the specified one. +* `tls_certs_path`: Path to load the TLS certificates from. +* `tls_verify`: Controls whether client verifies the server's certificate chain and hostname (default: `true`). +* `watch_by_default`: Enable watch by default. If false, containers that don't have `diun.enable=true` label will be ignored (default: `false`). +* `watch_stopped`: Include created and exited containers too (default: `false`). + +## Docker labels + +You can configure more finely the way to analyze the image of your container through Docker labels: * `diun.enable`: Set to true to enable image analysis of this container. Required if `watch_by_default` is disabled for this provider. * `diun.regopts_id`: Registry options ID from [`regopts`](../configuration.md#regopts) to use. diff --git a/doc/providers/file.md b/doc/providers/file.md new file mode 100644 index 00000000..f453da4f --- /dev/null +++ b/doc/providers/file.md @@ -0,0 +1,171 @@ +# File provider + +* [About](#about) +* [Example](#example) +* [Quick start](#quick-start) +* [Provider configuration](#provider-configuration) + * [filename](#filename) + * [directory](#directory) +* [YAML configuration file](#yaml-configuration-file) + +## About + +The file provider lets you define Docker images to analyze through a YAML file or a directory. + +## Example + +Register the file provider: + +```yml +db: + path: diun.db + +watch: + workers: 20 + schedule: "* * * * *" + +regopts: + someregistryoptions: + username: foo + password: bar + timeout: 20 + onemore: + username: foo2 + password: bar2 + insecure_tls: true + +providers: + file: + filename: /path/to/config.yml +``` + +```yml +### /path/to/config.yml + +# Watch latest tag of crazymax/nextcloud image on docker.io (DockerHub) with registry ID 'someregistryoptions'. +- name: docker.io/crazymax/nextcloud:latest + regopts_id: someregistryoptions + +# Watch 4.0.0 tag of jfrog/artifactory-oss image on frog-docker-reg2.bintray.io (Bintray) with registry ID 'onemore'. +- name: jfrog-docker-reg2.bintray.io/jfrog/artifactory-oss:4.0.0 + regopts_id: onemore + +# Watch coreos/hyperkube image on quay.io (Quay) and assume latest tag. +- name: quay.io/coreos/hyperkube + +# Watch crazymax/swarm-cronjob image and assume docker.io registry and latest tag. +# Only include tags matching regexp ^1\.2\..* +- name: crazymax/swarm-cronjob + watch_repo: true + include_tags: + - ^1\.2\..* + +# Watch portainer/portainer image on docker.io (DockerHub) and assume latest tag +# Only watch latest 10 tags and include tags matching regexp ^(0|[1-9]\d*)\..* +- name: docker.io/portainer/portainer + watch_repo: true + max_tags: 10 + include_tags: + - ^(0|[1-9]\d*)\..* + +# Watch alpine image (library) and assume docker.io registry and latest tag. +# Force linux/arm64/v8 platform for this image +- name: alpine + watch_repo: true + platform: + os: linux + arch: arm64 + variant: v8 +``` + +## Quick start + +Let's take a look with a simple example: + +```yml +db: + path: diun.db + +watch: + workers: 20 + schedule: "* * * * *" + +regopts: + jfrog: + username: foo + password: bar + +providers: + file: + filename: /path/to/config.yml +``` + +```yml +# /path/to/config.yml +- name: crazymax/cloudflared + watch_repo: true +- name: docker.bintray.io/jfrog/xray-mongo:3.2.6 + regopts_id: jfrog +``` + +Here we want to analyze all tags of `crazymax/cloudflared` and `docker.bintray.io/jfrog/xray-mongo:3.2.6` tag. Now let's start Diun: + +``` +$ diun --config diun.yml +Sat, 14 Dec 2019 15:32:23 UTC INF Starting Diun 2.0.0 +Sat, 14 Dec 2019 15:32:23 UTC INF Found 2 image(s) to analyze... provider=file +Sat, 14 Dec 2019 15:32:25 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:latest provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.11.3 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.11.0 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.10.1 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.9.0 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.9.2 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.10.2 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.11.2 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.9.1 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.10.4 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=docker.bintray.io/jfrog/xray-mongo:3.2.6 image=docker.bintray.io/jfrog/xray-mongo:3.2.6 provider=file +Sat, 14 Dec 2019 15:32:28 UTC INF Cron initialized with schedule * * * * * +Sat, 14 Dec 2019 15:32:28 UTC INF Next run in 31 seconds (2019-12-14 15:33:00 +0000 UTC) +``` + +## Provider configuration + +### filename + +Defines the path to the [configuration file](#yaml-configuration-file). + +> :warning: `filename` and `directory` are mutually exclusive. + +```yml +providers: + file: + filename: /path/to/config/conf.yml +``` + +### directory + +Defines the path to the directory that contains the [configuration files](#yaml-configuration-file) (`*.yml` or `*.yaml`). + +> :warning: `filename` and `directory` are mutually exclusive. + +```yml +providers: + file: + directory: /path/to/config +``` + +## YAML configuration file + +The configuration file(s) defines a slice of images to analyze with the following fields: + +* `name`: Docker image name to watch using `registry/path:tag` format. If registry omitted, `docker.io` will be used and if tag omitted, `latest` will be used. **required** +* `regopts_id`: Registry options ID from [`regopts`](../configuration.md#regopts) to use. +* `watch_repo`: Watch all tags of this `image` repository (default: `false`). +* `max_tags`: Maximum number of tags to watch if `watch_repo` enabled. 0 means all of them (default: `0`). +* `include_tags`: List of regular expressions to include tags. Can be useful if you enable `watch_repo`. +* `exclude_tags`: List of regular expressions to exclude tags. Can be useful if you enable `watch_repo`. +* `platform`: Check a custom platform. (default will retrieve platform dynamically based on your operating system). + * `os`: Operating system to use. + * `arch`: CPU architecture to use. + * `variant`: Variant of the CPU to use. diff --git a/doc/providers/static.md b/doc/providers/static.md deleted file mode 100644 index 35139ef3..00000000 --- a/doc/providers/static.md +++ /dev/null @@ -1,54 +0,0 @@ -# Static provider - -* [About](#about) -* [Quick start](#quick-start) - -## About - -The static provider is the most basic way to analyse Docker images. Nothing special to see here as everything is configured through the [providers field](../configuration.md#providers). - -## Quick start - -But let's take a look with a simple example: - -```yml -db: - path: diun.db - -watch: - workers: 20 - schedule: "* * * * *" - -regopts: - jfrog: - username: foo - password: bar - -providers: - static: - - name: crazymax/cloudflared - watch_repo: true - - name: docker.bintray.io/jfrog/xray-mongo:3.2.6 - regopts_id: jfrog -``` - -Here we want to analyze all tags of `crazymax/cloudflared` and `docker.bintray.io/jfrog/xray-mongo:3.2.6` tag. Now let's start Diun: - -``` -$ diun --config diun.yml -Sat, 14 Dec 2019 15:32:23 UTC INF Starting Diun 2.0.0 -Sat, 14 Dec 2019 15:32:23 UTC INF Found 2 static provider(s) to analyze... -Sat, 14 Dec 2019 15:32:25 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:latest provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.11.3 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.11.0 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.10.1 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.9.0 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.9.2 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.10.2 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.11.2 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.9.1 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=crazymax/cloudflared image=docker.io/crazymax/cloudflared:2019.10.4 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF New image found id=docker.bintray.io/jfrog/xray-mongo:3.2.6 image=docker.bintray.io/jfrog/xray-mongo:3.2.6 provider=static -Sat, 14 Dec 2019 15:32:28 UTC INF Cron initialized with schedule * * * * * -Sat, 14 Dec 2019 15:32:28 UTC INF Next run in 31 seconds (2019-12-14 15:33:00 +0000 UTC) -``` diff --git a/doc/providers/swarm.md b/doc/providers/swarm.md index fe574253..4bce1c24 100644 --- a/doc/providers/swarm.md +++ b/doc/providers/swarm.md @@ -2,11 +2,12 @@ * [About](#about) * [Quick start](#quick-start) -* [Configuration](#configuration) +* [Provider configuration](#provider-configuration) +* [Docker labels](#docker-labels) ## About -The Swarm provider is closely linked to the [Docker provider](docker.md) except that it allows you to analyze the services of your Swarm cluster defined in the [Diun configuration](../configuration.md#providers) to extract the images found and check for updates on the registry. +The Swarm provider is closely linked to the [Docker provider](docker.md) except that it allows you to analyze the services of your Swarm cluster to extract the images found and check for updates on the registry. ## Quick start @@ -103,9 +104,19 @@ diun_diun.1.i1l4yuiafq6y@docker-desktop | Sat, 14 Dec 2019 16:20:02 CET INF N ... ``` -## Configuration +## Provider configuration -In the same spirit as the [static provider](static.md), you can configure more finely the way to analyze the image of your service. But unlike the static provider, this is done via Docker labels: +The Swarm provider configuration is a map of Docker Swarm clusters to watch with the following options available: + +* `endpoint`: Server address to connect to. Local if empty. +* `api_version`: Overrides the client version with the specified one. +* `tls_certs_path`: Path to load the TLS certificates from. +* `tls_verify`: Controls whether client verifies the server's certificate chain and hostname (default: `true`). +* `watch_by_default`: Enable watch by default. If false, services that don't have `diun.enable=true` label will be ignored (default: `false`). + +## Docker labels + +You can configure more finely the way to analyze the image of your service through Docker labels: * `diun.enable`: Set to true to enable image analysis of this container. Required if `watch_by_default` is disabled for this provider. * `diun.regopts_id`: Registry options ID from [`regopts`](../configuration.md#regopts) to use. diff --git a/internal/app/diun.go b/internal/app/diun.go index 867e05da..482e6aa5 100644 --- a/internal/app/diun.go +++ b/internal/app/diun.go @@ -13,7 +13,7 @@ import ( "github.com/crazy-max/diun/internal/model" "github.com/crazy-max/diun/internal/notif" dockerPrd "github.com/crazy-max/diun/internal/provider/docker" - staticPrd "github.com/crazy-max/diun/internal/provider/static" + filePrd "github.com/crazy-max/diun/internal/provider/file" swarmPrd "github.com/crazy-max/diun/internal/provider/swarm" "github.com/hako/durafmt" "github.com/panjf2000/ants/v2" @@ -127,8 +127,8 @@ func (di *Diun) Run() { di.createJob(job) } - // Static provider - for _, job := range staticPrd.New(di.cfg.Providers.Static).ListJob() { + // File provider + for _, job := range filePrd.New(di.cfg.Providers.File).ListJob() { di.createJob(job) } diff --git a/internal/config/config.go b/internal/config/config.go index 83eae6db..7b2cf035 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -2,7 +2,6 @@ package config import ( "encoding/json" - "errors" "fmt" "io/ioutil" "net/mail" @@ -12,7 +11,7 @@ import ( "github.com/crazy-max/diun/internal/model" "github.com/crazy-max/diun/pkg/utl" "github.com/imdario/mergo" - "github.com/rs/zerolog/log" + "github.com/pkg/errors" "gopkg.in/yaml.v2" ) @@ -123,10 +122,8 @@ func (cfg *Config) validate() error { } } - for key, prdStatic := range cfg.Providers.Static { - if err := cfg.validateStaticProvider(key, prdStatic); err != nil { - return err - } + if err := cfg.validateFileProvider(); err != nil { + return err } if cfg.Notif.Mail.Enable { @@ -183,17 +180,22 @@ func (cfg *Config) validateSwarmProvider(id string, prdSwarm model.PrdSwarm) err return nil } -func (cfg *Config) validateStaticProvider(key int, prdStatic model.PrdStatic) error { - if prdStatic.Name == "" { - return fmt.Errorf("name is required for static provider %d", key) +func (cfg *Config) validateFileProvider() error { + switch { + case len(cfg.Providers.File.Directory) > 0: + if _, err := os.Stat(cfg.Providers.File.Directory); os.IsNotExist(err) { + return errors.Wrap(err, "directory not found for file provider") + } + case len(cfg.Providers.File.Filename) > 0: + if _, err := os.Stat(cfg.Providers.File.Filename); os.IsNotExist(err) { + return errors.Wrap(err, "filename not found for file provider") + } } - - cfg.Providers.Static[key] = prdStatic return nil } -// Display logs configuration in a pretty JSON format -func (cfg *Config) Display() { +// Display configuration in a pretty JSON format +func (cfg *Config) Display() string { b, _ := json.MarshalIndent(cfg, "", " ") - log.Debug().Msg(string(b)) + return string(b) } diff --git a/internal/config/config.test.yml b/internal/config/config.test.yml index cd7c337e..ca6c5b63 100644 --- a/internal/config/config.test.yml +++ b/internal/config/config.test.yml @@ -68,35 +68,5 @@ providers: swarm: local_swarm: watch_by_default: true - static: - - name: docker.io/crazymax/nextcloud:latest - regopts_id: someregopts - - name: crazymax/swarm-cronjob - watch_repo: true - include_tags: - - ^1\.2\..* - - name: jfrog-docker-reg2.bintray.io/jfrog/artifactory-oss:4.0.0 - regopts_id: bintrayoptions - - name: docker.bintray.io/jfrog/xray-server:2.8.6 - watch_repo: true - max_tags: 50 - - name: quay.io/coreos/hyperkube - - name: docker.io/portainer/portainer - watch_repo: true - max_tags: 10 - include_tags: - - ^(0|[1-9]\d*)\..* - - name: traefik - watch_repo: true - - name: alpine - platform: - os: linux - arch: arm64 - variant: v8 - - name: docker.io/graylog/graylog:3.2.0 - - name: jacobalberty/unifi:5.9 - - name: quay.io/coreos/hyperkube:v1.1.7-coreos.1 - - name: crazymax/ddns-route53 - watch_repo: true - include_tags: - - ^1\..* + file: + filename: ./dummy.yml diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 220b0cc6..bab04a2d 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -121,66 +121,8 @@ func TestLoad(t *testing.T) { WatchByDefault: true, }, }, - Static: []model.PrdStatic{ - { - Name: "docker.io/crazymax/nextcloud:latest", - RegOptsID: "someregopts", - }, - { - Name: "crazymax/swarm-cronjob", - WatchRepo: true, - IncludeTags: []string{ - `^1\.2\..*`, - }, - }, - { - Name: "jfrog-docker-reg2.bintray.io/jfrog/artifactory-oss:4.0.0", - RegOptsID: "bintrayoptions", - }, - { - Name: "docker.bintray.io/jfrog/xray-server:2.8.6", - WatchRepo: true, - MaxTags: 50, - }, - { - Name: "quay.io/coreos/hyperkube", - }, - { - Name: "docker.io/portainer/portainer", - WatchRepo: true, - MaxTags: 10, - IncludeTags: []string{ - `^(0|[1-9]\d*)\..*`, - }, - }, - { - Name: "traefik", - WatchRepo: true, - }, - { - Name: "alpine", - Platform: model.ImagePlatform{ - Os: "linux", - Arch: "arm64", - Variant: "v8", - }, - }, - { - Name: "docker.io/graylog/graylog:3.2.0", - }, - { - Name: "jacobalberty/unifi:5.9", - }, - { - Name: "quay.io/coreos/hyperkube:v1.1.7-coreos.1", - }, - { - Name: "crazymax/ddns-route53", - WatchRepo: true, - IncludeTags: []string{ - `^1\..*`, - }, - }, + File: model.PrdFile{ + Filename: "./dummy.yml", }, }, }, @@ -189,8 +131,13 @@ func TestLoad(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { cfg, err := config.Load(tt.cli, "test") + if !tt.wantErr && err != nil { + t.Error(err) + } assert.Equal(t, tt.wantData, cfg) - assert.Equal(t, tt.wantErr, err != nil) + if !tt.wantErr && cfg != nil { + assert.NotEmpty(t, cfg.Display()) + } }) } } diff --git a/internal/config/dummy.yml b/internal/config/dummy.yml new file mode 100644 index 00000000..e69de29b diff --git a/internal/model/providers.go b/internal/model/providers.go index 3a376114..b2fbe5bd 100644 --- a/internal/model/providers.go +++ b/internal/model/providers.go @@ -4,7 +4,7 @@ package model type Providers struct { Docker map[string]PrdDocker `yaml:"docker,omitempty" json:",omitempty"` Swarm map[string]PrdSwarm `yaml:"swarm,omitempty" json:",omitempty"` - Static []PrdStatic `yaml:"static,omitempty" json:",omitempty"` + File PrdFile `yaml:"file,omitempty" json:",omitempty"` } // PrdDocker holds docker provider configuration @@ -26,5 +26,8 @@ type PrdSwarm struct { WatchByDefault bool `yaml:"watch_by_default,omitempty" json:",omitempty"` } -// PrdStatic holds static provider configuration -type PrdStatic Image +// PrdFile holds file provider configuration +type PrdFile struct { + Filename string `yaml:"filename,omitempty" json:",omitempty"` + Directory string `yaml:"directory,omitempty" json:",omitempty"` +} diff --git a/internal/provider/docker/docker.go b/internal/provider/docker/docker.go index 3d8235f0..481c5db0 100644 --- a/internal/provider/docker/docker.go +++ b/internal/provider/docker/docker.go @@ -5,20 +5,25 @@ import ( "github.com/crazy-max/diun/internal/model" "github.com/crazy-max/diun/internal/provider" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) // Client represents an active docker provider object type Client struct { *provider.Client - elts map[string]model.PrdDocker + elts map[string]model.PrdDocker + logger zerolog.Logger } // New creates new docker provider instance func New(elts map[string]model.PrdDocker) *provider.Client { - return &provider.Client{Handler: &Client{ - elts: elts, - }} + return &provider.Client{ + Handler: &Client{ + elts: elts, + logger: log.With().Str("provider", "docker").Logger(), + }, + } } // ListJob returns job list to process @@ -27,7 +32,7 @@ func (c *Client) ListJob() []model.Job { return []model.Job{} } - log.Info().Msgf("Found %d docker provider(s) to analyze...", len(c.elts)) + c.logger.Info().Msgf("Found %d image(s) to analyze", len(c.elts)) var list []model.Job for id, elt := range c.elts { for _, img := range c.listContainerImage(id, elt) { diff --git a/internal/provider/file/file.go b/internal/provider/file/file.go new file mode 100644 index 00000000..1afdb48a --- /dev/null +++ b/internal/provider/file/file.go @@ -0,0 +1,103 @@ +package file + +import ( + "io/ioutil" + "path/filepath" + "strings" + + "github.com/crazy-max/diun/internal/model" + "github.com/crazy-max/diun/internal/provider" + "github.com/rs/zerolog" + "github.com/rs/zerolog/log" + "gopkg.in/yaml.v2" +) + +// Client represents an active file provider object +type Client struct { + *provider.Client + item model.PrdFile + logger zerolog.Logger +} + +// New creates new file provider instance +func New(item model.PrdFile) *provider.Client { + return &provider.Client{ + Handler: &Client{ + item: item, + logger: log.With().Str("provider", "file").Logger(), + }, + } +} + +// ListJob returns job list to process +func (c *Client) ListJob() []model.Job { + images := c.loadImages() + if len(images) == 0 { + return []model.Job{} + } + + c.logger.Info().Msgf("Found %d image(s) to analyze", len(images)) + var list []model.Job + for _, elt := range images { + list = append(list, model.Job{ + Provider: "file", + Image: elt, + }) + } + + return list +} + +func (c *Client) loadImages() []model.Image { + var images []model.Image + + files := c.getFiles() + if len(files) == 0 { + return []model.Image{} + } + + for _, file := range files { + var items []model.Image + bytes, err := ioutil.ReadFile(file) + if err != nil { + c.logger.Error().Err(err).Msgf("Unable to read config file %s", file) + continue + } + if err := yaml.UnmarshalStrict(bytes, &items); err != nil { + c.logger.Error().Err(err).Msgf("Unable to decode into struct %s", file) + continue + } + images = append(images, items...) + } + + return images +} + +func (c *Client) getFiles() []string { + var files []string + + switch { + case len(c.item.Directory) > 0: + fileList, err := ioutil.ReadDir(c.item.Directory) + if err != nil { + c.logger.Error().Err(err).Msgf("Unable to read directory %s", c.item.Directory) + return files + } + for _, file := range fileList { + if file.IsDir() { + continue + } + switch strings.ToLower(filepath.Ext(file.Name())) { + case ".yaml", ".yml": + // noop + default: + continue + } + files = append(files, filepath.Join(c.item.Directory, file.Name())) + } + case len(c.item.Filename) > 0: + files = append(files, c.item.Filename) + } + + return files +} diff --git a/internal/provider/file/file_test.go b/internal/provider/file/file_test.go new file mode 100644 index 00000000..9b6a46e9 --- /dev/null +++ b/internal/provider/file/file_test.go @@ -0,0 +1,127 @@ +package file_test + +import ( + "testing" + + "github.com/crazy-max/diun/internal/model" + "github.com/crazy-max/diun/internal/provider/file" + "github.com/stretchr/testify/assert" +) + +var ( + bintrayFile = []model.Job{ + { + Provider: "file", + Image: model.Image{ + Name: "jfrog-docker-reg2.bintray.io/jfrog/artifactory-oss:4.0.0", + RegOptsID: "bintrayoptions", + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "docker.bintray.io/jfrog/xray-server:2.8.6", + WatchRepo: true, + MaxTags: 50, + }, + }, + } + dockerhubFile = []model.Job{ + { + Provider: "file", + Image: model.Image{ + Name: "docker.io/crazymax/nextcloud:latest", + RegOptsID: "someregopts", + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "crazymax/swarm-cronjob", + WatchRepo: true, + IncludeTags: []string{ + `^1\.2\..*`, + }, + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "docker.io/portainer/portainer", + WatchRepo: true, + MaxTags: 10, + IncludeTags: []string{ + `^(0|[1-9]\d*)\..*`, + }, + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "traefik", + WatchRepo: true, + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "alpine", + Platform: model.ImagePlatform{ + Os: "linux", + Arch: "arm64", + Variant: "v8", + }, + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "docker.io/graylog/graylog:3.2.0", + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "jacobalberty/unifi:5.9", + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "crazymax/ddns-route53", + WatchRepo: true, + IncludeTags: []string{ + `^1\..*`, + }, + }, + }, + } + quayFile = []model.Job{ + { + Provider: "file", + Image: model.Image{ + Name: "quay.io/coreos/hyperkube", + }, + }, + { + Provider: "file", + Image: model.Image{ + Name: "quay.io/coreos/hyperkube:v1.1.7-coreos.1", + }, + }, + } +) + +func TestListJobFilename(t *testing.T) { + fc := file.New(model.PrdFile{ + Filename: "./test/dockerhub.yml", + }) + assert.Equal(t, dockerhubFile, fc.ListJob()) +} + +func TestListJobDirectory(t *testing.T) { + fc := file.New(model.PrdFile{ + Directory: "./test", + }) + assert.Equal(t, append(append(bintrayFile, dockerhubFile...), quayFile...), fc.ListJob()) +} diff --git a/internal/provider/file/test/bintray.yml b/internal/provider/file/test/bintray.yml new file mode 100644 index 00000000..4bd71e0c --- /dev/null +++ b/internal/provider/file/test/bintray.yml @@ -0,0 +1,5 @@ +- name: jfrog-docker-reg2.bintray.io/jfrog/artifactory-oss:4.0.0 + regopts_id: bintrayoptions +- name: docker.bintray.io/jfrog/xray-server:2.8.6 + watch_repo: true + max_tags: 50 diff --git a/internal/provider/file/test/dockerhub.yml b/internal/provider/file/test/dockerhub.yml new file mode 100644 index 00000000..68cd3c92 --- /dev/null +++ b/internal/provider/file/test/dockerhub.yml @@ -0,0 +1,24 @@ +- name: docker.io/crazymax/nextcloud:latest + regopts_id: someregopts +- name: crazymax/swarm-cronjob + watch_repo: true + include_tags: + - ^1\.2\..* +- name: docker.io/portainer/portainer + watch_repo: true + max_tags: 10 + include_tags: + - ^(0|[1-9]\d*)\..* +- name: traefik + watch_repo: true +- name: alpine + platform: + os: linux + arch: arm64 + variant: v8 +- name: docker.io/graylog/graylog:3.2.0 +- name: jacobalberty/unifi:5.9 +- name: crazymax/ddns-route53 + watch_repo: true + include_tags: + - ^1\..* diff --git a/internal/provider/file/test/quay.yml b/internal/provider/file/test/quay.yml new file mode 100644 index 00000000..40c7ae3e --- /dev/null +++ b/internal/provider/file/test/quay.yml @@ -0,0 +1,2 @@ +- name: quay.io/coreos/hyperkube +- name: quay.io/coreos/hyperkube:v1.1.7-coreos.1 diff --git a/internal/provider/static/static.go b/internal/provider/static/static.go deleted file mode 100644 index 50605ca8..00000000 --- a/internal/provider/static/static.go +++ /dev/null @@ -1,40 +0,0 @@ -package static - -import ( - "fmt" - - "github.com/crazy-max/diun/internal/model" - "github.com/crazy-max/diun/internal/provider" - "github.com/rs/zerolog/log" -) - -// Client represents an active static provider object -type Client struct { - *provider.Client - elts []model.PrdStatic -} - -// New creates new static provider instance -func New(elts []model.PrdStatic) *provider.Client { - return &provider.Client{Handler: &Client{ - elts: elts, - }} -} - -// ListJob returns job list to process -func (c *Client) ListJob() []model.Job { - if len(c.elts) == 0 { - return []model.Job{} - } - - log.Info().Msgf("Found %d static provider(s) to analyze...", len(c.elts)) - var list []model.Job - for key, elt := range c.elts { - list = append(list, model.Job{ - Provider: fmt.Sprintf("static-%d", key), - Image: model.Image(elt), - }) - } - - return list -} diff --git a/internal/provider/swarm/swarm.go b/internal/provider/swarm/swarm.go index 601212b7..e18d5761 100644 --- a/internal/provider/swarm/swarm.go +++ b/internal/provider/swarm/swarm.go @@ -5,20 +5,25 @@ import ( "github.com/crazy-max/diun/internal/model" "github.com/crazy-max/diun/internal/provider" + "github.com/rs/zerolog" "github.com/rs/zerolog/log" ) // Client represents an active swarm provider object type Client struct { *provider.Client - elts map[string]model.PrdSwarm + elts map[string]model.PrdSwarm + logger zerolog.Logger } // New creates new swarm provider instance func New(elts map[string]model.PrdSwarm) *provider.Client { - return &provider.Client{Handler: &Client{ - elts: elts, - }} + return &provider.Client{ + Handler: &Client{ + elts: elts, + logger: log.With().Str("provider", "swarm").Logger(), + }, + } } // ListJob returns job list to process @@ -27,7 +32,7 @@ func (c *Client) ListJob() []model.Job { return []model.Job{} } - log.Info().Msgf("Found %d swarm provider(s) to analyze...", len(c.elts)) + c.logger.Info().Msgf("Found %d image(s) to analyze", len(c.elts)) var list []model.Job for id, elt := range c.elts { for _, img := range c.listServiceImage(id, elt) {