diff --git a/docs/assets/watch/healthchecks.png b/docs/assets/watch/healthchecks.png new file mode 100644 index 00000000..f1b29dc6 Binary files /dev/null and b/docs/assets/watch/healthchecks.png differ diff --git a/docs/config/watch.md b/docs/config/watch.md index b148596a..03b2aa39 100644 --- a/docs/config/watch.md +++ b/docs/config/watch.md @@ -1,6 +1,20 @@ # Watch configuration -## `workers` +## Overview + +```yaml +watch: + workers: 10 + schedule: "0 * * * *" + firstCheckNotif: false + healthchecks: + baseURL: https://hc-ping.com/ + uuid: 5bf66975-d4c7-4bf5-bcc8-b8d8a82ea278 +``` + +## Configuration + +### `workers` Maximum number of workers that will execute tasks concurrently. (default `10`) @@ -13,7 +27,7 @@ Maximum number of workers that will execute tasks concurrently. (default `10`) !!! abstract "Environment variables" * `DIUN_WATCH_WORKERS` -## `schedule` +### `schedule` [CRON expression](https://godoc.org/github.com/robfig/cron#hdr-CRON_Expression_Format) to schedule Diun watcher. (default `0 * * * *`) @@ -26,7 +40,7 @@ Maximum number of workers that will execute tasks concurrently. (default `10`) !!! abstract "Environment variables" * `DIUN_WATCH_SCHEDULE` -## `firstCheckNotif` +### `firstCheckNotif` Send notification at the very first analysis of an image. (default `false`) @@ -38,3 +52,29 @@ Send notification at the very first analysis of an image. (default `false`) !!! abstract "Environment variables" * `DIUN_WATCH_FIRSTCHECKNOTIF` + +### `healthchecks` + +Healthchecks allows to monitor Diun watcher by sending start and success notification +events to [healthchecks.io](https://healthchecks.io/). + +!!! tip + A [Docker image for Healthchecks](https://github.com/crazy-max/docker-healthchecks) is available if you want + to self-host your instance. + +![](../assets/watch/healthchecks.png) + +!!! example "Config file" + ```yaml + watch: + healthchecks: + baseURL: https://hc-ping.com/ + uuid: 5bf66975-d4c7-4bf5-bcc8-b8d8a82ea278 + ``` + +!!! abstract "Environment variables" + * `DIUN_WATCH_HEALTHCHECKS_BASEURL` + * `DIUN_WATCH_HEALTHCHECKS_UUID` + +* `baseURL`: Base URL for the Healthchecks Ping API (default `https://hc-ping.com/`). +* `uuid`: UUID of an existing healthcheck (required). diff --git a/go.mod b/go.mod index 31ebed2f..28514cd5 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( github.com/alecthomas/kong v0.2.11 github.com/containers/image/v5 v5.6.0 + github.com/crazy-max/gohealthchecks v0.2.0 github.com/crazy-max/gonfig v0.3.0 github.com/docker/docker v1.4.2-0.20200204220554-5f6d6f3f2203 github.com/docker/go-connections v0.4.0 diff --git a/go.sum b/go.sum index fa0b777c..b6bf50ff 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3E github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/crazy-max/gohealthchecks v0.2.0 h1:r+L9PuTwq+EpmzffDrLvxk7Ps+SVU+f2T5JHvfAHbv0= +github.com/crazy-max/gohealthchecks v0.2.0/go.mod h1:3DB7UfVoI5njYSAyqKkrBuBjf5OlmzjJZ1BlKC5+nWE= github.com/crazy-max/gonfig v0.3.0 h1:/HFdLQjXSNhImgeQgD2eXhc5svX4PhUkGSbl4fJRp4s= github.com/crazy-max/gonfig v0.3.0/go.mod h1:7vmzltkoa1RHpGB5fTom0ebnqelHdd7fzhtXTi8sVoQ= github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= diff --git a/internal/app/diun.go b/internal/app/diun.go index 974eca10..c9aa33de 100644 --- a/internal/app/diun.go +++ b/internal/app/diun.go @@ -1,6 +1,7 @@ package app import ( + "net/url" "sync" "sync/atomic" "time" @@ -15,8 +16,10 @@ import ( kubernetesPrd "github.com/crazy-max/diun/v4/internal/provider/kubernetes" swarmPrd "github.com/crazy-max/diun/v4/internal/provider/swarm" "github.com/crazy-max/diun/v4/pkg/registry" + "github.com/crazy-max/gohealthchecks" "github.com/hako/durafmt" "github.com/panjf2000/ants/v2" + "github.com/pkg/errors" "github.com/robfig/cron/v3" "github.com/rs/zerolog/log" ) @@ -27,6 +30,7 @@ type Diun struct { cfg *config.Config cron *cron.Cron db *db.Client + hc *gohealthchecks.Client notif *notif.Client jobID cron.EntryID locker uint32 @@ -58,6 +62,19 @@ func New(meta model.Meta, cli model.Cli, cfg *config.Config, location *time.Loca } } + if cfg.Watch.Healthchecks != nil { + var hcBaseURL *url.URL + if len(cfg.Watch.Healthchecks.BaseURL) > 0 { + hcBaseURL, err = url.Parse(cfg.Watch.Healthchecks.BaseURL) + if err != nil { + return nil, errors.Wrap(err, "Cannot parse Healthchecks base URL") + } + } + diun.hc = gohealthchecks.NewClient(&gohealthchecks.ClientOptions{ + BaseURL: hcBaseURL, + }) + } + return diun, nil } @@ -104,10 +121,14 @@ func (di *Diun) Run() { } log.Info().Msg("Cron triggered") + entries := new(model.NotifEntries) + di.HealthchecksStart() + defer di.HealthchecksSuccess(entries) + di.wg = new(sync.WaitGroup) di.pool, _ = ants.NewPoolWithFunc(di.cfg.Watch.Workers, func(i interface{}) { job := i.(model.Job) - di.runJob(job) + entries.Add(di.runJob(job)) di.wg.Done() }, ants.WithLogger(new(logging.AntsLogger))) defer di.pool.Release() @@ -133,10 +154,17 @@ func (di *Diun) Run() { } di.wg.Wait() + log.Info(). + Int("added", entries.CountNew). + Int("updated", entries.CountUpdate). + Int("unchanged", entries.CountUnchange). + Int("failed", entries.CountError). + Msg("Jobs completed") } // Close closes diun func (di *Diun) Close() { + di.HealthchecksFail("Application closed") if di.cron != nil { di.cron.Stop() } diff --git a/internal/app/hc.go b/internal/app/hc.go new file mode 100644 index 00000000..aba13a92 --- /dev/null +++ b/internal/app/hc.go @@ -0,0 +1,59 @@ +package app + +import ( + "bytes" + "context" + "text/template" + + "github.com/crazy-max/diun/v4/internal/model" + "github.com/crazy-max/gohealthchecks" + "github.com/rs/zerolog/log" +) + +func (di *Diun) HealthchecksStart() { + if di.hc == nil { + return + } + + if err := di.hc.Start(context.Background(), gohealthchecks.PingingOptions{ + UUID: di.cfg.Watch.Healthchecks.UUID, + }); err != nil { + log.Error().Err(err).Msgf("Cannot send Healthchecks start event") + } +} + +func (di *Diun) HealthchecksSuccess(entries *model.NotifEntries) { + if di.hc == nil { + return + } + + var logsBuf bytes.Buffer + logsTpl := template.Must(template.New("").Parse(`{{ .CountTotal }} tag(s) have been scanned: +* {{ .CountNew }} new tag(s) found +* {{ .CountUpdate }} tag(s) updated +* {{ .CountUnchange }} tag(s) unchanged +* {{ .CountError }} tag(s) with error`)) + if err := logsTpl.Execute(&logsBuf, entries); err != nil { + log.Error().Err(err).Msgf("Cannot create logs for Healthchecks success event") + } + + if err := di.hc.Success(context.Background(), gohealthchecks.PingingOptions{ + UUID: di.cfg.Watch.Healthchecks.UUID, + Logs: logsBuf.String(), + }); err != nil { + log.Error().Err(err).Msgf("Cannot send Healthchecks success event") + } +} + +func (di *Diun) HealthchecksFail(logs string) { + if di.hc == nil { + return + } + + if err := di.hc.Fail(context.Background(), gohealthchecks.PingingOptions{ + UUID: di.cfg.Watch.Healthchecks.UUID, + Logs: logs, + }); err != nil { + log.Error().Err(err).Msgf("Cannot send Healthchecks fail event") + } +} diff --git a/internal/app/job.go b/internal/app/job.go index f8d6eca9..eb26d48d 100644 --- a/internal/app/job.go +++ b/internal/app/job.go @@ -141,7 +141,14 @@ func (di *Diun) createJob(job model.Job) { } } -func (di *Diun) runJob(job model.Job) { +func (di *Diun) runJob(job model.Job) (entry model.NotifEntry) { + var err error + entry = model.NotifEntry{ + Status: model.ImageStatusError, + Provider: job.Provider, + Image: job.RegImage, + } + sublog := log.With(). Str("provider", job.Provider). Str("image", job.RegImage.String()). @@ -155,7 +162,7 @@ func (di *Diun) runJob(job model.Job) { return } - liveManifest, err := job.Registry.Manifest(job.RegImage) + entry.Manifest, err = job.Registry.Manifest(job.RegImage) if err != nil { sublog.Warn().Err(err).Msg("Cannot get remote manifest") return @@ -167,19 +174,19 @@ func (di *Diun) runJob(job model.Job) { return } - status := model.ImageStatusUnchange if len(dbManifest.Name) == 0 { - status = model.ImageStatusNew + entry.Status = model.ImageStatusNew sublog.Info().Msg("New image found") - } else if !liveManifest.Created.Equal(*dbManifest.Created) { - status = model.ImageStatusUpdate + } else if !entry.Manifest.Created.Equal(*dbManifest.Created) { + entry.Status = model.ImageStatusUpdate sublog.Info().Msg("Image update found") } else { + entry.Status = model.ImageStatusUnchange sublog.Debug().Msg("No changes") return } - if err := di.db.PutManifest(job.RegImage, liveManifest); err != nil { + if err := di.db.PutManifest(job.RegImage, entry.Manifest); err != nil { sublog.Error().Err(err).Msg("Cannot write manifest to db") return } @@ -190,10 +197,6 @@ func (di *Diun) runJob(job model.Job) { return } - di.notif.Send(model.NotifEntry{ - Status: status, - Provider: job.Provider, - Image: job.RegImage, - Manifest: liveManifest, - }) + di.notif.Send(entry) + return } diff --git a/internal/config/config.go b/internal/config/config.go index 6bb08373..8a1e775e 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -68,6 +68,10 @@ func (cfg *Config) validate(cli model.Cli) error { } } + if cfg.Watch.Healthchecks != nil && len(cfg.Watch.Healthchecks.UUID) == 0 { + return errors.New("Healthchecks UUID is required") + } + if cfg.Notif == nil && cli.TestNotif { return errors.New("At least one notifier is required") } diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 85c56443..35feb470 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -51,6 +51,10 @@ func TestLoadFile(t *testing.T) { Workers: 100, Schedule: "*/30 * * * *", FirstCheckNotif: utl.NewTrue(), + Healthchecks: &model.Healthchecks{ + BaseURL: "https://hc-ping.com/", + UUID: "5bf66975-d4c7-4bf5-bcc8-b8d8a82ea278", + }, }, Notif: &model.Notif{ Amqp: &model.NotifAmqp{ diff --git a/internal/config/fixtures/config.test.yml b/internal/config/fixtures/config.test.yml index 84f55b03..c77c445a 100644 --- a/internal/config/fixtures/config.test.yml +++ b/internal/config/fixtures/config.test.yml @@ -5,6 +5,9 @@ watch: workers: 100 schedule: "*/30 * * * *" firstCheckNotif: true + healthchecks: + baseURL: https://hc-ping.com/ + uuid: 5bf66975-d4c7-4bf5-bcc8-b8d8a82ea278 notif: amqp: diff --git a/internal/config/fixtures/config.validate.yml b/internal/config/fixtures/config.validate.yml index 6ff79676..7eac3310 100644 --- a/internal/config/fixtures/config.validate.yml +++ b/internal/config/fixtures/config.validate.yml @@ -5,6 +5,9 @@ watch: workers: 100 schedule: "*/30 * * * *" firstCheckNotif: false + healthchecks: + baseURL: https://hc-ping.com/ + uuid: 5bf66975-d4c7-4bf5-bcc8-b8d8a82ea278 notif: amqp: diff --git a/internal/model/image.go b/internal/model/image.go index 062ede1d..15986d02 100644 --- a/internal/model/image.go +++ b/internal/model/image.go @@ -24,6 +24,7 @@ const ( ImageStatusNew = ImageStatus("new") ImageStatusUpdate = ImageStatus("update") ImageStatusUnchange = ImageStatus("unchange") + ImageStatusError = ImageStatus("error") ) // ImageStatus holds Docker image status analysis diff --git a/internal/model/notif.go b/internal/model/notif.go index 23a00a7c..add88dd0 100644 --- a/internal/model/notif.go +++ b/internal/model/notif.go @@ -4,6 +4,16 @@ import ( "github.com/crazy-max/diun/v4/pkg/registry" ) +// NotifEntries represents a list of notification entries +type NotifEntries struct { + Entries []NotifEntry + CountNew int + CountUpdate int + CountUnchange int + CountError int + CountTotal int +} + // NotifEntry represents a notification entry type NotifEntry struct { Status ImageStatus `json:"status,omitempty"` @@ -36,3 +46,22 @@ func (s *Notif) GetDefaults() *Notif { func (s *Notif) SetDefaults() { // noop } + +// Add adds a new notif entry +func (s *NotifEntries) Add(entry NotifEntry) { + s.Entries = append(s.Entries, entry) + switch entry.Status { + case ImageStatusNew: + s.CountNew++ + s.CountTotal++ + case ImageStatusUpdate: + s.CountUpdate++ + s.CountTotal++ + case ImageStatusUnchange: + s.CountUnchange++ + s.CountTotal++ + case ImageStatusError: + s.CountError++ + s.CountTotal++ + } +} diff --git a/internal/model/watch.go b/internal/model/watch.go index 9194e687..072606c9 100644 --- a/internal/model/watch.go +++ b/internal/model/watch.go @@ -6,9 +6,10 @@ import ( // Watch holds data necessary for watch configuration type Watch struct { - Workers int `yaml:"workers,omitempty" json:"workers,omitempty" validate:"required,min=1"` - Schedule string `yaml:"schedule,omitempty" json:"schedule,omitempty" validate:"required"` - FirstCheckNotif *bool `yaml:"firstCheckNotif,omitempty" json:"firstCheckNotif,omitempty" validate:"required"` + Workers int `yaml:"workers,omitempty" json:"workers,omitempty" validate:"required,min=1"` + Schedule string `yaml:"schedule,omitempty" json:"schedule,omitempty" validate:"required"` + FirstCheckNotif *bool `yaml:"firstCheckNotif,omitempty" json:"firstCheckNotif,omitempty" validate:"required"` + Healthchecks *Healthchecks `yaml:"healthchecks,omitempty" json:"healthchecks,omitempty"` } // GetDefaults gets the default values diff --git a/internal/model/watch_healthchecks.go b/internal/model/watch_healthchecks.go new file mode 100644 index 00000000..7d472e6e --- /dev/null +++ b/internal/model/watch_healthchecks.go @@ -0,0 +1,19 @@ +package model + +// Healthchecks holds data necessary for Healthchecks configuration +type Healthchecks struct { + BaseURL string `yaml:"baseURL,omitempty" json:"baseURL,omitempty"` + UUID string `yaml:"uuid,omitempty" json:"uuid,omitempty" validate:"required"` +} + +// GetDefaults gets the default values +func (s *Healthchecks) GetDefaults() *Healthchecks { + n := &Healthchecks{} + n.SetDefaults() + return n +} + +// SetDefaults sets the default values +func (s *Healthchecks) SetDefaults() { + // noop +}