diff --git a/go.mod b/go.mod index ca6f6a0a..8a6d0d9b 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 // indirect github.com/nlopes/slack v0.6.0 github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 github.com/panjf2000/ants/v2 v2.4.5 github.com/pkg/errors v0.9.1 github.com/pkg/profile v1.6.0 diff --git a/internal/app/job.go b/internal/app/job.go index 2a1b0e97..bd48fd0d 100644 --- a/internal/app/job.go +++ b/internal/app/job.go @@ -176,7 +176,8 @@ func (di *Diun) runJob(job model.Job) (entry model.NotifEntry) { return } - entry.Manifest, err = job.Registry.Manifest(job.RegImage, dbManifest) + var updated bool + entry.Manifest, updated, err = job.Registry.Manifest(job.RegImage, dbManifest) if err != nil { sublog.Warn().Err(err).Msg("Cannot get remote manifest") return @@ -185,13 +186,12 @@ func (di *Diun) runJob(job model.Job) (entry model.NotifEntry) { if len(dbManifest.Name) == 0 { entry.Status = model.ImageStatusNew sublog.Info().Msg("New image found") - } else if entry.Manifest.Digest.String() != dbManifest.Digest.String() { + } else if updated { entry.Status = model.ImageStatusUpdate sublog.Info().Msg("Image update found") } else { entry.Status = model.ImageStatusUnchange sublog.Debug().Msg("No changes") - return } if err := di.db.PutManifest(job.RegImage, entry.Manifest); err != nil { @@ -199,6 +199,9 @@ func (di *Diun) runJob(job model.Job) (entry model.NotifEntry) { return } sublog.Debug().Msg("Manifest saved to database") + if entry.Status == model.ImageStatusUnchange { + return + } if job.FirstCheck && !*di.cfg.Watch.FirstCheckNotif { sublog.Debug().Msg("Skipping notification (first check)") diff --git a/pkg/registry/manifest.go b/pkg/registry/manifest.go index 25fc580c..6d32ae8a 100644 --- a/pkg/registry/manifest.go +++ b/pkg/registry/manifest.go @@ -6,8 +6,8 @@ import ( "github.com/containers/image/v5/docker" "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) @@ -26,80 +26,91 @@ type Manifest struct { } // Manifest returns the manifest for a specific image -func (c *Client) Manifest(image Image, dbManifest Manifest) (Manifest, error) { +func (c *Client) Manifest(image Image, dbManifest Manifest) (Manifest, bool, error) { ctx, cancel := c.timeoutContext() defer cancel() - if c.sysCtx.DockerAuthConfig == nil { - c.sysCtx.DockerAuthConfig = &types.DockerAuthConfig{} - // TODO: Seek credentials - //auth, err := config.GetCredentials(c.sysCtx, reference.Domain(ref.DockerReference())) - //if err != nil { - // return nil, errors.Wrap(err, "Cannot get registry credentials") - //} - //*c.sysCtx.DockerAuthConfig = auth - } - - imgRef, err := ParseReference(image.String()) + rmRef, err := ParseReference(image.String()) if err != nil { - return Manifest{}, errors.Wrap(err, "Cannot parse reference") + return Manifest{}, false, errors.Wrap(err, "Cannot parse reference") } - var imgDigest digest.Digest - if c.opts.CompareDigest { - imgDigest, err = docker.GetDigest(ctx, c.sysCtx, imgRef) + // Retrieve remote digest through HEAD request + rmDigest, err := docker.GetDigest(ctx, c.sysCtx, rmRef) + if err != nil { + return Manifest{}, false, errors.Wrap(err, "Cannot get image digest from HEAD request") + } + + // Digest match, returns db manifest + if c.opts.CompareDigest && len(dbManifest.Digest) > 0 && dbManifest.Digest == rmDigest { + return dbManifest, false, nil + } + + rmCloser, err := rmRef.NewImage(ctx, c.sysCtx) + if err != nil { + return Manifest{}, false, errors.Wrap(err, "Cannot create image closer") + } + defer rmCloser.Close() + + rmRawManifest, rmManifestMimeType, err := rmCloser.Manifest(ctx) + if err != nil { + return Manifest{}, false, errors.Wrap(err, "Cannot get raw manifest") + } + + // For manifests list compare also digest matching the platform + updated := dbManifest.Digest != rmDigest + if c.opts.CompareDigest && len(dbManifest.Raw) > 0 && dbManifest.isManifestList() && isManifestList(rmManifestMimeType) { + dbManifestList, err := manifest.ListFromBlob(dbManifest.Raw, dbManifest.MIMEType) if err != nil { - return Manifest{}, errors.Wrap(err, "Cannot get image digest from HEAD request") + return Manifest{}, false, errors.Wrap(err, "Cannot parse manifest list") } - - if dbManifest.Digest != "" && dbManifest.Digest == imgDigest { - return dbManifest, nil - } - } - - imgCloser, err := imgRef.NewImage(ctx, c.sysCtx) - if err != nil { - return Manifest{}, errors.Wrap(err, "Cannot create image closer") - } - defer imgCloser.Close() - - rawManifest, _, err := imgCloser.Manifest(ctx) - if err != nil { - return Manifest{}, errors.Wrap(err, "Cannot get raw manifest") - } - - if !c.opts.CompareDigest { - imgDigest, err = manifest.Digest(rawManifest) + dbManifestPlatformDigest, err := dbManifestList.ChooseInstance(c.sysCtx) if err != nil { - return Manifest{}, errors.Wrap(err, "Cannot get digest") + return Manifest{}, false, errors.Wrapf(err, "Error choosing image instance") } + rmManifestList, err := manifest.ListFromBlob(rmRawManifest, rmManifestMimeType) + if err != nil { + return Manifest{}, false, errors.Wrap(err, "Cannot parse manifest list") + } + rmManifestPlatformDigest, err := rmManifestList.ChooseInstance(c.sysCtx) + if err != nil { + return Manifest{}, false, errors.Wrapf(err, "Error choosing image instance") + } + updated = dbManifestPlatformDigest != rmManifestPlatformDigest } - imgInspect, err := imgCloser.Inspect(ctx) + // Metadata describing the Docker image + rmInspect, err := rmCloser.Inspect(ctx) if err != nil { - return Manifest{}, errors.Wrap(err, "Cannot inspect") + return Manifest{}, false, errors.Wrap(err, "Cannot inspect") } - - imgTag := imgInspect.Tag - if len(imgTag) == 0 { - imgTag = image.Tag + rmTag := rmInspect.Tag + if len(rmTag) == 0 { + rmTag = image.Tag } - - imgPlatform := fmt.Sprintf("%s/%s", imgInspect.Os, imgInspect.Architecture) - if imgInspect.Variant != "" { - imgPlatform = fmt.Sprintf("%s/%s", imgPlatform, imgInspect.Variant) + rmPlatform := fmt.Sprintf("%s/%s", rmInspect.Os, rmInspect.Architecture) + if rmInspect.Variant != "" { + rmPlatform = fmt.Sprintf("%s/%s", rmPlatform, rmInspect.Variant) } return Manifest{ - Name: imgCloser.Reference().DockerReference().Name(), - Tag: imgTag, - MIMEType: manifest.GuessMIMEType(rawManifest), - Digest: imgDigest, - Created: imgInspect.Created, - DockerVersion: imgInspect.DockerVersion, - Labels: imgInspect.Labels, - Layers: imgInspect.Layers, - Platform: imgPlatform, - Raw: rawManifest, - }, nil + Name: rmCloser.Reference().DockerReference().Name(), + Tag: rmTag, + MIMEType: rmManifestMimeType, + Digest: rmDigest, + Created: rmInspect.Created, + DockerVersion: rmInspect.DockerVersion, + Labels: rmInspect.Labels, + Layers: rmInspect.Layers, + Platform: rmPlatform, + Raw: rmRawManifest, + }, updated, nil +} + +func (m Manifest) isManifestList() bool { + return isManifestList(m.MIMEType) +} + +func isManifestList(mimeType string) bool { + return mimeType == manifest.DockerV2ListMediaType || mimeType == imgspecv1.MediaTypeImageIndex } diff --git a/pkg/registry/manifest_test.go b/pkg/registry/manifest_test.go index 89523b48..26d062d8 100644 --- a/pkg/registry/manifest_test.go +++ b/pkg/registry/manifest_test.go @@ -22,24 +22,11 @@ func TestCompareDigest(t *testing.T) { t.Error(err) } - manifest, err := rc.Manifest(img, registry.Manifest{ - Name: "docker.io/crazymax/diun", - Tag: "2.5.0", - MIMEType: "application/vnd.docker.distribution.manifest.list.v2+json", - Digest: "sha256:db618981ef3d07699ff6cd8b9d2a81f51a021747bc08c85c1b0e8d11130c2be5", - DockerVersion: "", - Labels: map[string]string{ - "maintainer": "CrazyMax", - "org.label-schema.build-date": "2020-03-01T18:00:42Z", - "org.label-schema.description": "Docker image update notifier", - "org.label-schema.name": "Diun", - "org.label-schema.schema-version": "1.0", - "org.label-schema.url": "https://github.com/crazy-max/diun", - "org.label-schema.vcs-ref": "488ce441", - "org.label-schema.vcs-url": "https://github.com/crazy-max/diun", - "org.label-schema.vendor": "CrazyMax", - "org.label-schema.version": "2.5.0", - }, + manifest, _, err := rc.Manifest(img, registry.Manifest{ + Name: "docker.io/crazymax/diun", + Tag: "2.5.0", + MIMEType: "application/vnd.docker.distribution.manifest.list.v2+json", + Digest: "sha256:db618981ef3d07699ff6cd8b9d2a81f51a021747bc08c85c1b0e8d11130c2be5", Platform: "linux/amd64", }) assert.NoError(t, err) @@ -50,6 +37,224 @@ func TestCompareDigest(t *testing.T) { assert.Empty(t, manifest.DockerVersion) } +func TestManifest(t *testing.T) { + rc, err := registry.New(registry.Options{ + CompareDigest: true, + ImageOs: "linux", + ImageArch: "amd64", + }) + if err != nil { + t.Error(err) + } + + img, err := registry.ParseImage(registry.ParseImageOptions{ + Name: "portainer/portainer-ce:linux-amd64-2.5.1", + }) + if err != nil { + t.Error(err) + } + + manifest, updated, err := rc.Manifest(img, registry.Manifest{ + Name: "docker.io/portainer/portainer-ce", + Tag: "linux-amd64-2.5.1", + MIMEType: "application/vnd.docker.distribution.manifest.v2+json", + Digest: "sha256:653057af0d2d961f436c75deda1ca7fe3defc89664bed6bd3da8c91c88c1ce05", + Platform: "linux/amd64", + Raw: []byte(`{ + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "schemaVersion": 2, + "config": { + "mediaType": "application/vnd.docker.container.image.v1+json", + "digest": "sha256:45be17a5903a1129362792537fc6b18bc91fe03e2581501b514ac5d45ede128e", + "size": 1704 + }, + "layers": [ + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:94cfa856b2b17d5e36c7df9875ebbbed7e939a8292df5fe22d2dfce0434330f2", + "size": 122403 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:49d59ee0881a4f04166d438b27055e2b29327abbbb0f274951255ee880912056", + "size": 92 + }, + { + "mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip", + "digest": "sha256:3fc1bc38fb56bce4b3913d0fab6072822541142793a6d997a4a69d5d81fa46e0", + "size": 74850629 + } + ] +}`), + }) + + assert.NoError(t, err) + assert.Equal(t, false, updated) + assert.Equal(t, "docker.io/portainer/portainer-ce", manifest.Name) + assert.Equal(t, "linux-amd64-2.5.1", manifest.Tag) + assert.Equal(t, "application/vnd.docker.distribution.manifest.v2+json", manifest.MIMEType) + assert.Equal(t, "sha256:653057af0d2d961f436c75deda1ca7fe3defc89664bed6bd3da8c91c88c1ce05", manifest.Digest.String()) + assert.Equal(t, "linux/amd64", manifest.Platform) +} + +func TestManifestMultiUpdatedPlatform(t *testing.T) { + rc, err := registry.New(registry.Options{ + CompareDigest: true, + ImageOs: "linux", + ImageArch: "amd64", + }) + if err != nil { + t.Error(err) + } + + img, err := registry.ParseImage(registry.ParseImageOptions{ + Name: "mongo:3.6.21", + }) + if err != nil { + t.Error(err) + } + + manifest, updated, err := rc.Manifest(img, registry.Manifest{ + Name: "docker.io/library/mongo", + Tag: "3.6.21", + MIMEType: "application/vnd.docker.distribution.manifest.list.v2+json", + Digest: "sha256:61f5dce8422d36b2a4ad0077bc499b1b68320e13fd30aa0b201c080fef42a39a", + Platform: "linux/amd64", + Raw: []byte(`{ + "manifests": [ + { + "digest": "sha256:98f22b0bf33479e2c34d99c820d9ded79cdf46b2c6f54af5a11191a90ff369ed", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "amd64", + "os": "linux" + }, + "size": 3030 + }, + { + "digest": "sha256:8226c9734c19533d5cc52748e35ae10085f3b4ef0a3bd4537017bc2484589511", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "arm64", + "os": "linux", + "variant": "v8" + }, + "size": 3030 + }, + { + "digest": "sha256:fb9e9376b228ba8d75d62b10aadaa3ed445266f85e27af3da531666d992f9621", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.17763.1697" + }, + "size": 2771 + }, + { + "digest": "sha256:f0534dfb20d90f152a7b4ae8812c61381cff7de983c2b17fc1fe3558a237fdac", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.14393.4169" + }, + "size": 2693 + } + ], + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "schemaVersion": 2 +}`), + }) + + assert.NoError(t, err) + assert.Equal(t, true, updated) + assert.Equal(t, "docker.io/library/mongo", manifest.Name) + assert.Equal(t, "3.6.21", manifest.Tag) + assert.Equal(t, "application/vnd.docker.distribution.manifest.list.v2+json", manifest.MIMEType) + assert.Equal(t, "sha256:3cff2069adb34a330552695659c261bca69148e325863763b78b0285dd1a25c9", manifest.Digest.String()) + assert.Equal(t, "linux/amd64", manifest.Platform) +} + +func TestManifestMultiNotUpdatedPlatform(t *testing.T) { + rc, err := registry.New(registry.Options{ + CompareDigest: true, + ImageOs: "linux", + ImageArch: "amd64", + }) + if err != nil { + t.Error(err) + } + + img, err := registry.ParseImage(registry.ParseImageOptions{ + Name: "mongo:3.6.21", + }) + if err != nil { + t.Error(err) + } + + manifest, updated, err := rc.Manifest(img, registry.Manifest{ + Name: "docker.io/library/mongo", + Tag: "3.6.21", + MIMEType: "application/vnd.docker.distribution.manifest.list.v2+json", + Digest: "sha256:61f5dce8422d36b2a4ad0077bc499b1b68320e13fd30aa0b201c080fef42a39a", + Platform: "linux/amd64", + Raw: []byte(`{ + "manifests": [ + { + "digest": "sha256:6e5d3405a510988d96f0fa3ec7220040be27ce783eb4cd576feb1a69b382ea20", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "amd64", + "os": "linux" + }, + "size": 3030 + }, + { + "digest": "sha256:8226c9734c19533d5cc52748e35ae10085f3b4ef0a3bd4537017bc2484589511", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "arm64", + "os": "linux", + "variant": "v8" + }, + "size": 3030 + }, + { + "digest": "sha256:0fcde35d138739e27b79a8b9863dedc1fdd65fd3a82a319842f86edc87d11594", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.17763.1817" + }, + "size": 2771 + }, + { + "digest": "sha256:6f54fda6a88a56c0953e901f0285a74a16b4cf1bec021b2434e3bfe78cabfada", + "mediaType": "application/vnd.docker.distribution.manifest.v2+json", + "platform": { + "architecture": "amd64", + "os": "windows", + "os.version": "10.0.14393.4283" + }, + "size": 2693 + } + ], + "mediaType": "application/vnd.docker.distribution.manifest.list.v2+json", + "schemaVersion": 2 +}`), + }) + + assert.NoError(t, err) + assert.Equal(t, false, updated) + assert.Equal(t, "docker.io/library/mongo", manifest.Name) + assert.Equal(t, "3.6.21", manifest.Tag) + assert.Equal(t, "application/vnd.docker.distribution.manifest.list.v2+json", manifest.MIMEType) + assert.Equal(t, "sha256:3cff2069adb34a330552695659c261bca69148e325863763b78b0285dd1a25c9", manifest.Digest.String()) + assert.Equal(t, "linux/amd64", manifest.Platform) +} + func TestManifestVariant(t *testing.T) { rc, err := registry.New(registry.Options{ ImageOs: "linux", @@ -67,7 +272,7 @@ func TestManifestVariant(t *testing.T) { t.Error(err) } - manifest, err := rc.Manifest(img, registry.Manifest{}) + manifest, _, err := rc.Manifest(img, registry.Manifest{}) assert.NoError(t, err) assert.Equal(t, "docker.io/crazymax/diun", manifest.Name) assert.Equal(t, "2.5.0", manifest.Tag) diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index 98543d61..f1ade4dd 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -37,6 +37,16 @@ func New(opts Options) (*Client, error) { } } + if auth == nil { + auth = &types.DockerAuthConfig{} + // TODO: Seek credentials + //auth, err := config.GetCredentials(c.sysCtx, reference.Domain(ref.DockerReference())) + //if err != nil { + // return nil, errors.Wrap(err, "Cannot get registry credentials") + //} + //*c.sysCtx.DockerAuthConfig = auth + } + // Sys context sysCtx := &types.SystemContext{ DockerAuthConfig: auth,