From 889f595523e441f0e3a2712235359c17b0fac609 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Fri, 28 Jun 2019 17:07:20 +0200 Subject: [PATCH] Review config file structure and improve worker pool --- CHANGELOG.md | 5 ++ cmd/main.go | 6 +- go.mod | 5 +- go.sum | 8 +- internal/app/dispatcher.go | 49 ------------ internal/app/diun.go | 155 ++++++++----------------------------- internal/app/image.go | 139 +++++++++++++++++++++++++++++++++ internal/app/worker.go | 49 ------------ internal/config/config.go | 129 +++++++++++++++++------------- internal/model/image.go | 20 +++++ internal/model/item.go | 12 --- internal/model/registry.go | 9 --- 12 files changed, 283 insertions(+), 303 deletions(-) delete mode 100644 internal/app/dispatcher.go create mode 100644 internal/app/image.go delete mode 100644 internal/app/worker.go create mode 100644 internal/model/image.go delete mode 100644 internal/model/item.go delete mode 100644 internal/model/registry.go diff --git a/CHANGELOG.md b/CHANGELOG.md index c169ee80..01e96a7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## 1.0.0 (2019/06/28) + +* Review config file structure +* Improve worker pool + ## 0.5.0 (2019/06/09) * Add worker pool to parallelize analyses diff --git a/cmd/main.go b/cmd/main.go index 09b55dec..30c47641 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -67,10 +67,6 @@ func main() { if err != nil { log.Fatal().Err(err).Msg("Cannot load configuration") } - if err := cfg.Check(); err != nil { - cfg.Display() - log.Fatal().Err(err).Msg("Improper configuration") - } cfg.Display() // Init @@ -85,7 +81,7 @@ func main() { // Start scheduler c = cron.NewWithLocation(location) - log.Info().Msgf("Watcher initialized with schedule %s", cfg.Watch.Schedule) + log.Info().Msgf("Cron initialized with schedule %s", cfg.Watch.Schedule) if err := c.AddJob(cfg.Watch.Schedule, diun); err != nil { log.Fatal().Err(err).Msg("Cannot create cron task") } diff --git a/go.mod b/go.mod index 6394734a..fae4d2cc 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ require ( github.com/BurntSushi/toml v0.3.1 // indirect github.com/Microsoft/go-winio v0.4.12 // indirect github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect - github.com/containers/image v1.5.1 + github.com/containers/image v2.0.0+incompatible github.com/containers/storage v1.12.9 // indirect github.com/crazy-max/cron v1.2.2 github.com/docker/distribution v2.7.1+incompatible // indirect @@ -18,14 +18,17 @@ require ( github.com/go-gomail/gomail v0.0.0-20160411212932-81ebce5c23df github.com/google/go-cmp v0.3.0 // indirect github.com/gorilla/mux v1.7.2 // indirect + github.com/hako/durafmt v0.0.0-20190612201238-650ed9f29a84 github.com/imdario/mergo v0.3.7 github.com/matcornic/hermes/v2 v2.0.2 github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v0.1.1 // indirect + github.com/panjf2000/ants v1.0.0 github.com/prometheus/client_golang v0.9.4 // indirect github.com/rs/zerolog v1.14.3 github.com/sirupsen/logrus v1.4.2 // indirect + github.com/stretchr/testify v1.3.0 go.etcd.io/bbolt v1.3.2 gopkg.in/alecthomas/kingpin.v2 v2.2.6 gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect diff --git a/go.sum b/go.sum index f16fe994..8ccacce0 100644 --- a/go.sum +++ b/go.sum @@ -17,8 +17,8 @@ github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containers/image v1.5.1 h1:ssEuj1c24uJvdMkUa2IrawuEFZBP12p6WzrjNBTQxE0= -github.com/containers/image v1.5.1/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/image v2.0.0+incompatible h1:FTr6Br7jlIKNCKMjSOMbAxKp2keQ0//jzJaYNTVhauk= +github.com/containers/image v2.0.0+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= github.com/containers/storage v1.12.9 h1:kYUE0EBpYv9zwW+MEgXBDsoY5FzmHE5PNKXhWbAkLKg= github.com/containers/storage v1.12.9/go.mod h1:+RirK6VQAqskQlaTBrOG6ulDvn4si2QjFE1NZCn06MM= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -58,6 +58,8 @@ github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/mux v1.7.2 h1:zoNxOV7WjqXptQOVngLmcSQgXmgk4NMz1HibBchjl/I= github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/hako/durafmt v0.0.0-20190612201238-650ed9f29a84 h1:RvcDqcKLua4b/jtXez7ZVe9s6Iq5N6ujVevqY4FBQmM= +github.com/hako/durafmt v0.0.0-20190612201238-650ed9f29a84/go.mod h1:5Scbynm8dF1XAPwIwkGPqzkM/shndPm79Jd1003hTjE= github.com/huandu/xstrings v1.2.0 h1:yPeWdRnmynF7p+lLYz0H2tthW9lqhMJrQV/U7yy4wX0= github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -92,6 +94,8 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/panjf2000/ants v1.0.0 h1:MZBsUt8W6ktQfhIswUZpw17IJlXY6ly2+U5b9jxwad4= +github.com/panjf2000/ants v1.0.0/go.mod h1:AaACblRPzq35m1g3enqYcxspbbiOJJYaxU2wMpm1cXY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/internal/app/dispatcher.go b/internal/app/dispatcher.go deleted file mode 100644 index d9378caa..00000000 --- a/internal/app/dispatcher.go +++ /dev/null @@ -1,49 +0,0 @@ -package app - -type Collector struct { - Job chan Job - end chan bool -} - -var workerChannel = make(chan chan Job) - -func (di *Diun) StartDispatcher(workerCount int) Collector { - var i int - var workers []worker - input := make(chan Job) - end := make(chan bool) - collector := Collector{ - Job: input, - end: end, - } - - for i < workerCount { - i++ - worker := worker{ - id: i, - diun: di, - workerPool: workerChannel, - jobChannel: make(chan Job), - end: make(chan bool), - } - worker.Start() - workers = append(workers, worker) - } - - go func() { - for { - select { - case <-end: - for _, w := range workers { - w.Stop() - } - return - case work := <-input: - worker := <-workerChannel - worker <- work - } - } - }() - - return collector -} diff --git a/internal/app/diun.go b/internal/app/diun.go index a4fb06e9..07d31b3c 100644 --- a/internal/app/diun.go +++ b/internal/app/diun.go @@ -1,28 +1,26 @@ package app import ( - "fmt" "sync" "sync/atomic" "time" "github.com/crazy-max/diun/internal/config" "github.com/crazy-max/diun/internal/db" - "github.com/crazy-max/diun/internal/model" "github.com/crazy-max/diun/internal/notif" - "github.com/crazy-max/diun/internal/utl" - "github.com/crazy-max/diun/pkg/docker" - "github.com/crazy-max/diun/pkg/docker/registry" + "github.com/hako/durafmt" + "github.com/panjf2000/ants" "github.com/rs/zerolog/log" ) // Diun represents an active diun object type Diun struct { - cfg *config.Config - db *db.Client - notif *notif.Client - locker uint32 - collector Collector + cfg *config.Config + db *db.Client + notif *notif.Client + locker uint32 + pool *ants.PoolWithFunc + wg *sync.WaitGroup } // New creates new diun instance @@ -54,125 +52,34 @@ func (di *Diun) Run() { } defer atomic.StoreUint32(&di.locker, 0) - log.Info().Msg("Running process") - var wg sync.WaitGroup - di.collector = di.StartDispatcher(di.cfg.Watch.Workers) + start := time.Now() + defer di.trackTime(start, "Finished, total time spent: ") - // Iterate items - for _, item := range di.cfg.Items { - reg, err := docker.NewRegistryClient(docker.RegistryOptions{ - Os: di.cfg.Watch.Os, - Arch: di.cfg.Watch.Arch, - Username: item.Registry.Username, - Password: item.Registry.Password, - Timeout: time.Duration(item.Registry.Timeout) * time.Second, - InsecureTLS: item.Registry.InsecureTLS, - }) - if err != nil { - log.Error().Err(err).Str("image", item.Image).Msg("Cannot create registry client") - continue - } - - image, err := registry.ParseImage(item.Image) - if err != nil { - log.Error().Err(err).Str("image", item.Image).Msg("Cannot parse image") - continue - } - - wg.Add(1) - di.collector.Job <- Job{ - ImageStr: item.Image, - Item: item, - Reg: reg, - Wg: &wg, - } - - if image.Domain != "" && item.WatchRepo { - tags, err := reg.Tags(docker.TagsOptions{ - Image: image, - Max: item.MaxTags, - Include: item.IncludeTags, - Exclude: item.ExcludeTags, - }) + log.Info().Msg("Starting Diun...") + di.wg = new(sync.WaitGroup) + di.pool, _ = ants.NewPoolWithFunc(di.cfg.Watch.Workers, func(i interface{}) { + var err error + switch t := i.(type) { + case imageJob: + err = di.imageJob(t) if err != nil { - log.Error().Err(err).Str("image", image.String()).Msg("Cannot retrieve tags") - continue + log.Error().Err(err).Msg("Job image error") } - - log.Debug().Str("image", image.String()).Msgf("%d tag(s) found in repository. %d will be analyzed (%d max, %d not included, %d excluded).", - tags.Total, - len(tags.List), - item.MaxTags, - tags.NotIncluded, - tags.Excluded, - ) - - for _, tag := range tags.List { - wg.Add(1) - di.collector.Job <- Job{ - ImageStr: fmt.Sprintf("%s/%s:%s", image.Domain, image.Path, tag), - Item: item, - Reg: reg, - Wg: &wg, - } + err = di.imageRepoJob(t) + if err != nil { + log.Error().Err(err).Msg("Job image repo error") } } - } - - wg.Wait() -} - -func (di *Diun) analyze(job Job, workerID int) error { - defer job.Wg.Done() - image, err := registry.ParseImage(job.ImageStr) - if err != nil { - return err - } - - if !utl.IsIncluded(image.Tag, job.Item.IncludeTags) { - log.Warn().Str("image", image.String()).Int("worker_id", workerID).Msg("Tag not included") - return nil - } else if utl.IsExcluded(image.Tag, job.Item.ExcludeTags) { - log.Warn().Str("image", image.String()).Int("worker_id", workerID).Msg("Tag excluded") - return nil - } - - liveManifest, err := job.Reg.Manifest(image) - if err != nil { - return err - } - /*b, _ := json.MarshalIndent(liveManifest, "", " ") - log.Debug().Msg(string(b))*/ - - dbManifest, err := di.db.GetManifest(image) - if err != nil { - return err - } - - status := model.ImageStatusUnchange - if dbManifest.Name == "" { - status = model.ImageStatusNew - log.Info().Str("image", image.String()).Int("worker_id", workerID).Msg("New image found") - } else if !liveManifest.Created.Equal(*dbManifest.Created) { - status = model.ImageStatusUpdate - log.Info().Str("image", image.String()).Int("worker_id", workerID).Msg("Image update found") - } else { - log.Debug().Str("image", image.String()).Int("worker_id", workerID).Msg("No changes") - return nil - } - - if err := di.db.PutManifest(image, liveManifest); err != nil { - return err - } - log.Debug().Str("image", image.String()).Int("worker_id", workerID).Msg("Manifest saved to database") - - di.notif.Send(model.NotifEntry{ - Status: status, - Image: image, - Manifest: liveManifest, + di.wg.Done() }) + defer func() { + if err := di.pool.Release(); err != nil { + log.Warn().Err(err).Msg("Cannot release pool") + } + }() - return nil + di.procImages() + di.wg.Wait() } // Close closes diun @@ -181,3 +88,7 @@ func (di *Diun) Close() { log.Warn().Err(err).Msg("Cannot close database") } } + +func (di *Diun) trackTime(start time.Time, prefix string) { + log.Info().Msgf("%s%s", prefix, durafmt.ParseShort(time.Since(start)).String()) +} diff --git a/internal/app/image.go b/internal/app/image.go new file mode 100644 index 00000000..1d849386 --- /dev/null +++ b/internal/app/image.go @@ -0,0 +1,139 @@ +package app + +import ( + "fmt" + "time" + + "github.com/crazy-max/diun/internal/model" + "github.com/crazy-max/diun/internal/utl" + "github.com/crazy-max/diun/pkg/docker" + "github.com/crazy-max/diun/pkg/docker/registry" + "github.com/rs/zerolog/log" +) + +type imageJob struct { + origin bool + image model.Image + registry *docker.RegistryClient +} + +func (di *Diun) procImages() { + // Iterate images + for _, img := range di.cfg.Image { + reg, err := docker.NewRegistryClient(docker.RegistryOptions{ + Os: di.cfg.Watch.Os, + Arch: di.cfg.Watch.Arch, + Username: img.RegOpts.Username, + Password: img.RegOpts.Password, + Timeout: time.Duration(img.RegOpts.Timeout) * time.Second, + InsecureTLS: img.RegOpts.InsecureTLS, + }) + if err != nil { + log.Error().Err(err).Str("image", img.Name).Msg("Cannot create registry client") + continue + } + + di.wg.Add(1) + err = di.pool.Invoke(imageJob{ + origin: true, + image: img, + registry: reg, + }) + if err != nil { + log.Error().Err(err).Msgf("Invoking image job") + } + } +} + +func (di *Diun) imageJob(job imageJob) error { + image, err := registry.ParseImage(job.image.Name) + if err != nil { + return err + } + + if !utl.IsIncluded(image.Tag, job.image.IncludeTags) { + log.Warn().Str("image", image.String()).Msg("Tag not included") + return nil + } else if utl.IsExcluded(image.Tag, job.image.ExcludeTags) { + log.Warn().Str("image", image.String()).Msg("Tag excluded") + return nil + } + + liveManifest, err := job.registry.Manifest(image) + if err != nil { + return err + } + /*b, _ := json.MarshalIndent(liveManifest, "", " ") + log.Debug().Msg(string(b))*/ + + dbManifest, err := di.db.GetManifest(image) + if err != nil { + return err + } + + status := model.ImageStatusUnchange + if dbManifest.Name == "" { + status = model.ImageStatusNew + log.Info().Str("image", image.String()).Msg("New image found") + } else if !liveManifest.Created.Equal(*dbManifest.Created) { + status = model.ImageStatusUpdate + log.Info().Str("image", image.String()).Msg("Image update found") + } else { + log.Debug().Str("image", image.String()).Msg("No changes") + return nil + } + + if err := di.db.PutManifest(image, liveManifest); err != nil { + return err + } + log.Debug().Str("image", image.String()).Msg("Manifest saved to database") + + di.notif.Send(model.NotifEntry{ + Status: status, + Image: image, + Manifest: liveManifest, + }) + + return nil +} + +func (di *Diun) imageRepoJob(job imageJob) error { + image, err := registry.ParseImage(job.image.Name) + if err != nil { + return err + } + + if !job.origin || image.Domain == "" || !job.image.WatchRepo { + return nil + } + + tags, err := job.registry.Tags(docker.TagsOptions{ + Image: image, + Max: job.image.MaxTags, + Include: job.image.IncludeTags, + Exclude: job.image.ExcludeTags, + }) + if err != nil { + return err + } + + log.Debug().Str("image", image.String()).Msgf("%d tag(s) found in repository. %d will be analyzed (%d max, %d not included, %d excluded).", + tags.Total, + len(tags.List), + job.image.MaxTags, + tags.NotIncluded, + tags.Excluded, + ) + + job.origin = false + for _, tag := range tags.List { + job.image.Name = fmt.Sprintf("%s/%s:%s", image.Domain, image.Path, tag) + di.wg.Add(1) + err = di.pool.Invoke(job) + if err != nil { + log.Error().Err(err).Msgf("Invoking repo image job") + } + } + + return nil +} diff --git a/internal/app/worker.go b/internal/app/worker.go deleted file mode 100644 index 54f3b8ed..00000000 --- a/internal/app/worker.go +++ /dev/null @@ -1,49 +0,0 @@ -package app - -import ( - "sync" - - "github.com/crazy-max/diun/internal/model" - "github.com/crazy-max/diun/pkg/docker" - "github.com/rs/zerolog/log" -) - -type Job struct { - ImageStr string - Item model.Item - Reg *docker.RegistryClient - Wg *sync.WaitGroup -} - -type worker struct { - id int - diun *Diun - workerPool chan chan Job - jobChannel chan Job - end chan bool -} - -// Start method starts the run loop for the worker -func (w *worker) Start() { - go func() { - for { - w.workerPool <- w.jobChannel - select { - case job := <-w.jobChannel: - if err := w.diun.analyze(job, w.id); err != nil { - log.Error().Err(err). - Str("image", job.ImageStr). - Int("worker_id", w.id). - Msg("Error analyzing image") - } - case <-w.end: - return - } - } - }() -} - -// Stop signals the worker to stop listening for work requests. -func (w *worker) Stop() { - w.end <- true -} diff --git a/internal/config/config.go b/internal/config/config.go index 5b1f8090..20cea2db 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,20 +18,20 @@ import ( // Config holds configuration details type Config struct { - Flags model.Flags - App model.App - Db model.Db `yaml:"db,omitempty"` - Watch model.Watch `yaml:"watch,omitempty"` - Notif model.Notif `yaml:"notif,omitempty"` - Registries map[string]model.Registry `yaml:"registries,omitempty"` - Items []model.Item `yaml:"items,omitempty"` + Flags model.Flags + App model.App + Db model.Db `yaml:"db,omitempty"` + Watch model.Watch `yaml:"watch,omitempty"` + Notif model.Notif `yaml:"notif,omitempty"` + RegOpts map[string]model.RegOpts `yaml:"regopts,omitempty"` + Image []model.Image `yaml:"image,omitempty"` } // Load returns Configuration struct -func Load(fl model.Flags, version string) (*Config, error) { +func Load(flags model.Flags, version string) (*Config, error) { var err error var cfg = Config{ - Flags: fl, + Flags: flags, App: model.App{ ID: "diun", Name: "Diun", @@ -65,24 +65,27 @@ func Load(fl model.Flags, version string) (*Config, error) { }, } - if _, err = os.Lstat(fl.Cfgfile); err != nil { + if _, err = os.Lstat(flags.Cfgfile); err != nil { return nil, fmt.Errorf("unable to open config file, %s", err) } - bytes, err := ioutil.ReadFile(fl.Cfgfile) + bytes, err := ioutil.ReadFile(flags.Cfgfile) if err != nil { return nil, fmt.Errorf("unable to read config file, %s", err) } - if err := yaml.Unmarshal(bytes, &cfg); err != nil { + if err := yaml.UnmarshalStrict(bytes, &cfg); err != nil { return nil, fmt.Errorf("unable to decode into struct, %v", err) } + if err := cfg.validate(); err != nil { + return nil, err + } + return &cfg, nil } -// Check verifies Config values -func (cfg *Config) Check() error { +func (cfg *Config) validate() error { if cfg.Flags.Docker { cfg.Db.Path = "/data/diun.db" } @@ -92,49 +95,14 @@ func (cfg *Config) Check() error { } cfg.Db.Path = path.Clean(cfg.Db.Path) - for id, reg := range cfg.Registries { - if err := mergo.Merge(®, model.Registry{ - InsecureTLS: false, - Timeout: 10, - }); err != nil { - return fmt.Errorf("cannot set default registry values for %s: %v", id, err) + for id, regopts := range cfg.RegOpts { + if err := cfg.validateRegOpts(id, regopts); err != nil { + return err } - cfg.Registries[id] = reg } - for key, item := range cfg.Items { - if item.Image == "" { - return fmt.Errorf("image is required for item %d", key) - } - - if err := mergo.Merge(&item, model.Item{ - WatchRepo: false, - MaxTags: 25, - }); err != nil { - return fmt.Errorf("cannot set default item values for %s: %v", item.Image, err) - } - - if item.RegistryID != "" { - reg, found := cfg.Registries[item.RegistryID] - if !found { - return fmt.Errorf("registry ID '%s' not found", item.RegistryID) - } - cfg.Items[key].Registry = reg - } - - for _, includeTag := range item.IncludeTags { - if _, err := regexp.Compile(includeTag); err != nil { - return fmt.Errorf("include tag regex '%s' for '%s' image cannot compile, %v", item.Image, includeTag, err) - } - } - - for _, excludeTag := range item.ExcludeTags { - if _, err := regexp.Compile(excludeTag); err != nil { - return fmt.Errorf("exclude tag regex '%s' for '%s' image cannot compile, %v", item.Image, excludeTag, err) - } - } - - if err := mergo.Merge(&cfg.Items[key], item); err != nil { + for key, img := range cfg.Image { + if err := cfg.validateImage(key, img); err != nil { return err } } @@ -151,6 +119,59 @@ func (cfg *Config) Check() error { return nil } +func (cfg *Config) validateRegOpts(id string, regopts model.RegOpts) error { + defTimeout := 10 + if regopts.Timeout <= 0 { + defTimeout = 0 + } + + if err := mergo.Merge(®opts, model.RegOpts{ + InsecureTLS: false, + Timeout: defTimeout, + }); err != nil { + return fmt.Errorf("cannot set default registry options values for %s: %v", id, err) + } + + cfg.RegOpts[id] = regopts + return nil +} + +func (cfg *Config) validateImage(key int, img model.Image) error { + if img.Name == "" { + return fmt.Errorf("name is required for image %d", key) + } + + if err := mergo.Merge(&img, model.Image{ + WatchRepo: false, + MaxTags: 0, + }); err != nil { + return fmt.Errorf("cannot set default image values for %s: %v", img.Name, err) + } + + if img.RegOptsID != "" { + regopts, found := cfg.RegOpts[img.RegOptsID] + if !found { + return fmt.Errorf("registry options %s not found for %s", img.RegOptsID, img.Name) + } + cfg.Image[key].RegOpts = regopts + } + + for _, includeTag := range img.IncludeTags { + if _, err := regexp.Compile(includeTag); err != nil { + return fmt.Errorf("include tag regex '%s' for %s cannot compile, %v", includeTag, img.Name, err) + } + } + + for _, excludeTag := range img.ExcludeTags { + if _, err := regexp.Compile(excludeTag); err != nil { + return fmt.Errorf("exclude tag regex '%s' for '%s' image cannot compile, %v", img.Name, excludeTag, err) + } + } + + cfg.Image[key] = img + return nil +} + // Display logs configuration in a pretty JSON format func (cfg *Config) Display() { b, _ := json.MarshalIndent(cfg, "", " ") diff --git a/internal/model/image.go b/internal/model/image.go new file mode 100644 index 00000000..5bde1925 --- /dev/null +++ b/internal/model/image.go @@ -0,0 +1,20 @@ +package model + +// RegOpts holds registry options configuration +type RegOpts struct { + Username string `yaml:"username,omitempty" json:",omitempty"` + Password string `yaml:"password,omitempty" json:",omitempty"` + InsecureTLS bool `yaml:"insecure_tls,omitempty" json:",omitempty"` + Timeout int `yaml:"timeout,omitempty" json:",omitempty"` +} + +// Image holds image configuration +type Image struct { + Name string `yaml:"name,omitempty" json:",omitempty"` + RegOptsID string `yaml:"regopts_id,omitempty" json:",omitempty"` + WatchRepo bool `yaml:"watch_repo,omitempty" json:",omitempty"` + MaxTags int `yaml:"max_tags,omitempty" json:",omitempty"` + IncludeTags []string `yaml:"include_tags,omitempty" json:",omitempty"` + ExcludeTags []string `yaml:"exclude_tags,omitempty" json:",omitempty"` + RegOpts RegOpts `yaml:"-" json:"-"` +} diff --git a/internal/model/item.go b/internal/model/item.go deleted file mode 100644 index c3a0cc3c..00000000 --- a/internal/model/item.go +++ /dev/null @@ -1,12 +0,0 @@ -package model - -// Item holds item configuration for a Docker image -type Item struct { - Image string `yaml:"image,omitempty" json:",omitempty"` - RegistryID string `yaml:"registry_id,omitempty" json:",omitempty"` - WatchRepo bool `yaml:"watch_repo,omitempty" json:",omitempty"` - MaxTags int `yaml:"max_tags,omitempty" json:",omitempty"` - IncludeTags []string `yaml:"include_tags,omitempty" json:",omitempty"` - ExcludeTags []string `yaml:"exclude_tags,omitempty" json:",omitempty"` - Registry Registry `yaml:"-" json:"-"` -} diff --git a/internal/model/registry.go b/internal/model/registry.go deleted file mode 100644 index 26c053eb..00000000 --- a/internal/model/registry.go +++ /dev/null @@ -1,9 +0,0 @@ -package model - -// Registry holds registry configuration -type Registry struct { - Username string `yaml:"username,omitempty" json:",omitempty"` - Password string `yaml:"password,omitempty" json:",omitempty"` - InsecureTLS bool `yaml:"insecure_tls,omitempty" json:",omitempty"` - Timeout int `yaml:"timeout,omitempty" json:",omitempty"` -}