diff options
| author | jannfis <jann@mistrust.net> | 2022-01-11 16:35:51 +0100 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2022-01-11 16:35:51 +0100 |
| commit | e1f65012575eac6cd46fc6fdfc2d1ef03ad9b930 (patch) | |
| tree | 7e4bd77d6e96ee77d8cd3f383165448c9ad01501 /pkg | |
| parent | 77a6e8f1f8afa1d5b5b5f440dd9341830e14f7ef (diff) | |
feat: Support manifestlist and multi-arch images (#341)
* feat: Support manifestlist and multi-arch images
Signed-off-by: jannfis <jann@mistrust.net>
* Add unit test
Signed-off-by: jannfis <jann@mistrust.net>
* Fix linter issue
Signed-off-by: jannfis <jann@mistrust.net>
Diffstat (limited to 'pkg')
| -rw-r--r-- | pkg/argocd/update.go | 2 | ||||
| -rw-r--r-- | pkg/common/constants.go | 1 | ||||
| -rw-r--r-- | pkg/image/options.go | 61 | ||||
| -rw-r--r-- | pkg/image/options_test.go | 78 | ||||
| -rw-r--r-- | pkg/image/version.go | 2 | ||||
| -rw-r--r-- | pkg/options/options.go | 87 | ||||
| -rw-r--r-- | pkg/options/options_test.go | 74 | ||||
| -rw-r--r-- | pkg/registry/client.go | 166 | ||||
| -rw-r--r-- | pkg/registry/client_test.go | 10 | ||||
| -rw-r--r-- | pkg/registry/mocks/RegistryClient.go | 94 | ||||
| -rw-r--r-- | pkg/registry/registry.go | 4 | ||||
| -rw-r--r-- | pkg/registry/registry_test.go | 2 | ||||
| -rw-r--r-- | pkg/tag/tag.go | 5 |
13 files changed, 528 insertions, 58 deletions
diff --git a/pkg/argocd/update.go b/pkg/argocd/update.go index 541a0f8..0a92ff2 100644 --- a/pkg/argocd/update.go +++ b/pkg/argocd/update.go @@ -42,6 +42,7 @@ type UpdateConfiguration struct { GitCommitEmail string GitCommitMessage *template.Template DisableKubeEvents bool + IgnorePlatforms bool } type GitCredsSource func(app *v1alpha1.Application) (git.Creds, error) @@ -196,6 +197,7 @@ func UpdateApplication(updateConf *UpdateConfiguration, state *SyncIterationStat vc.SortMode = applicationImage.GetParameterUpdateStrategy(updateConf.UpdateApp.Application.Annotations) vc.MatchFunc, vc.MatchArgs = applicationImage.GetParameterMatch(updateConf.UpdateApp.Application.Annotations) vc.IgnoreList = applicationImage.GetParameterIgnoreTags(updateConf.UpdateApp.Application.Annotations) + vc.Options = applicationImage.GetPlatformOptions(updateConf.UpdateApp.Application.Annotations, updateConf.IgnorePlatforms) // The endpoint can provide default credentials for pulling images err = rep.SetEndpointCredentials(updateConf.KubeClient) diff --git a/pkg/common/constants.go b/pkg/common/constants.go index 6c7fec3..84f39c8 100644 --- a/pkg/common/constants.go +++ b/pkg/common/constants.go @@ -33,6 +33,7 @@ const ( IgnoreTagsOptionAnnotation = ImageUpdaterAnnotationPrefix + "/%s.ignore-tags" ForceUpdateOptionAnnotation = ImageUpdaterAnnotationPrefix + "/%s.force-update" UpdateStrategyAnnotation = ImageUpdaterAnnotationPrefix + "/%s.update-strategy" + PlatformsAnnotation = ImageUpdaterAnnotationPrefix + "/%s.platforms" ) // Image pull secret related annotations diff --git a/pkg/image/options.go b/pkg/image/options.go index b1bd6a6..a8908b8 100644 --- a/pkg/image/options.go +++ b/pkg/image/options.go @@ -3,10 +3,12 @@ package image import ( "fmt" "regexp" + "runtime" "strings" "github.com/argoproj-labs/argocd-image-updater/pkg/common" "github.com/argoproj-labs/argocd-image-updater/pkg/log" + "github.com/argoproj-labs/argocd-image-updater/pkg/options" ) // GetParameterHelmImageName gets the value for image-name option for the image @@ -177,6 +179,65 @@ func (img *ContainerImage) GetParameterIgnoreTags(annotations map[string]string) return ignoreList } +// GetPlatformOptions sets up platform constraints for an image. If no platform +// is specified in the annotations, we restrict the platform for images to the +// platform we're executed on unless unrestricted is set to true, in which case +// we do not setup a platform restriction if no platform annotation is found. +func (img *ContainerImage) GetPlatformOptions(annotations map[string]string, unrestricted bool) *options.ManifestOptions { + var opts *options.ManifestOptions = options.NewManifestOptions() + key := fmt.Sprintf(common.PlatformsAnnotation, img.normalizedSymbolicName()) + logCtx := log.WithContext(). + AddField("image", img.ImageName). + AddField("registry", img.RegistryURL) + + val, ok := annotations[key] + if !ok { + if !unrestricted { + os := runtime.GOOS + arch := runtime.GOARCH + variant := "" + if strings.Contains(runtime.GOARCH, "/") { + a := strings.SplitN(runtime.GOARCH, "/", 2) + arch = a[0] + variant = a[1] + } + logCtx.Tracef("Using runtime platform constraint %s", options.PlatformKey(os, arch, variant)) + opts = opts.WithPlatform(os, arch, variant) + } + } else { + platforms := strings.Split(val, ",") + for _, ps := range platforms { + pt := strings.TrimSpace(ps) + os, arch, variant, err := ParsePlatform(pt) + if err != nil { + // If the platform identifier could not be parsed, we set the + // constraint intentionally to the invalid value so we don't + // end up updating to the wrong architecture possibly. + os = ps + logCtx.Warnf("could not parse platform identifier '%v': invalid format", pt) + } + logCtx.Tracef("Adding platform constraint %s", options.PlatformKey(os, arch, variant)) + opts = opts.WithPlatform(os, arch, variant) + } + } + + return opts +} + +func ParsePlatform(platformID string) (string, string, string, error) { + p := strings.SplitN(platformID, "/", 3) + if len(p) < 2 { + return "", "", "", fmt.Errorf("could not parse platform constraint '%s'", platformID) + } + os := p[0] + arch := p[1] + variant := "" + if len(p) == 3 { + variant = p[2] + } + return os, arch, variant, nil +} + func (img *ContainerImage) normalizedSymbolicName() string { return strings.ReplaceAll(img.ImageAlias, "/", "_") } diff --git a/pkg/image/options_test.go b/pkg/image/options_test.go index 88a0266..4313ed5 100644 --- a/pkg/image/options_test.go +++ b/pkg/image/options_test.go @@ -3,9 +3,11 @@ package image import ( "fmt" "regexp" + "runtime" "testing" "github.com/argoproj-labs/argocd-image-updater/pkg/common" + "github.com/argoproj-labs/argocd-image-updater/pkg/options" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -193,3 +195,79 @@ func Test_GetIgnoreTags(t *testing.T) { assert.Equal(t, "tag4", tags[3]) }) } + +func Test_GetPlatformOptions(t *testing.T) { + t.Run("Empty platform options with restriction", func(t *testing.T) { + annotations := map[string]string{} + img := NewFromIdentifier("dummy=foo/bar:1.12") + opts := img.GetPlatformOptions(annotations, false) + os := runtime.GOOS + arch := runtime.GOARCH + assert.True(t, opts.WantsPlatform(os, arch, "")) + assert.False(t, opts.WantsPlatform(os, arch, "invalid")) + }) + t.Run("Empty platform options without restriction", func(t *testing.T) { + annotations := map[string]string{} + img := NewFromIdentifier("dummy=foo/bar:1.12") + opts := img.GetPlatformOptions(annotations, true) + os := runtime.GOOS + arch := runtime.GOARCH + assert.True(t, opts.WantsPlatform(os, arch, "")) + assert.True(t, opts.WantsPlatform(os, arch, "invalid")) + assert.True(t, opts.WantsPlatform("windows", "amd64", "")) + }) + t.Run("Single platform without variant requested", func(t *testing.T) { + os := "linux" + arch := "arm64" + variant := "" + annotations := map[string]string{ + fmt.Sprintf(common.PlatformsAnnotation, "dummy"): options.PlatformKey(os, arch, variant), + } + img := NewFromIdentifier("dummy=foo/bar:1.12") + opts := img.GetPlatformOptions(annotations, false) + assert.True(t, opts.WantsPlatform(os, arch, variant)) + assert.False(t, opts.WantsPlatform(os, arch, "invalid")) + }) + t.Run("Single platform with variant requested", func(t *testing.T) { + os := "linux" + arch := "arm" + variant := "v6" + annotations := map[string]string{ + fmt.Sprintf(common.PlatformsAnnotation, "dummy"): options.PlatformKey(os, arch, variant), + } + img := NewFromIdentifier("dummy=foo/bar:1.12") + opts := img.GetPlatformOptions(annotations, false) + assert.True(t, opts.WantsPlatform(os, arch, variant)) + assert.False(t, opts.WantsPlatform(os, arch, "")) + assert.False(t, opts.WantsPlatform(runtime.GOOS, runtime.GOARCH, "")) + assert.False(t, opts.WantsPlatform(runtime.GOOS, runtime.GOARCH, variant)) + }) + t.Run("Multiple platforms requested", func(t *testing.T) { + os := "linux" + arch := "arm" + variant := "v6" + annotations := map[string]string{ + fmt.Sprintf(common.PlatformsAnnotation, "dummy"): options.PlatformKey(os, arch, variant) + ", " + options.PlatformKey(runtime.GOOS, runtime.GOARCH, ""), + } + img := NewFromIdentifier("dummy=foo/bar:1.12") + opts := img.GetPlatformOptions(annotations, false) + assert.True(t, opts.WantsPlatform(os, arch, variant)) + assert.True(t, opts.WantsPlatform(runtime.GOOS, runtime.GOARCH, "")) + assert.False(t, opts.WantsPlatform(os, arch, "")) + assert.False(t, opts.WantsPlatform(runtime.GOOS, runtime.GOARCH, variant)) + }) + t.Run("Invalid platform requested", func(t *testing.T) { + os := "linux" + arch := "arm" + variant := "v6" + annotations := map[string]string{ + fmt.Sprintf(common.PlatformsAnnotation, "dummy"): "invalid", + } + img := NewFromIdentifier("dummy=foo/bar:1.12") + opts := img.GetPlatformOptions(annotations, false) + assert.False(t, opts.WantsPlatform(os, arch, variant)) + assert.False(t, opts.WantsPlatform(runtime.GOOS, runtime.GOARCH, "")) + assert.False(t, opts.WantsPlatform(os, arch, "")) + assert.False(t, opts.WantsPlatform(runtime.GOOS, runtime.GOARCH, variant)) + }) +} diff --git a/pkg/image/version.go b/pkg/image/version.go index 8461e17..3e1b39f 100644 --- a/pkg/image/version.go +++ b/pkg/image/version.go @@ -4,6 +4,7 @@ import ( "path/filepath" "github.com/argoproj-labs/argocd-image-updater/pkg/log" + "github.com/argoproj-labs/argocd-image-updater/pkg/options" "github.com/argoproj-labs/argocd-image-updater/pkg/tag" "github.com/Masterminds/semver" @@ -42,6 +43,7 @@ type VersionConstraint struct { MatchArgs interface{} IgnoreList []string SortMode VersionSortMode + Options *options.ManifestOptions } type MatchFuncFn func(tagName string, pattern interface{}) bool diff --git a/pkg/options/options.go b/pkg/options/options.go new file mode 100644 index 0000000..c1a6462 --- /dev/null +++ b/pkg/options/options.go @@ -0,0 +1,87 @@ +package options + +import ( + "sort" + "sync" + + "github.com/argoproj-labs/argocd-image-updater/pkg/log" +) + +// ManifestOptions define some options when retrieving image manifests +type ManifestOptions struct { + platforms map[string]bool + mutex sync.RWMutex + metadata bool +} + +// NewManifestOptions returns an initialized ManifestOptions struct +func NewManifestOptions() *ManifestOptions { + return &ManifestOptions{ + platforms: make(map[string]bool), + metadata: false, + } +} + +// PlatformKey returns a string usable as platform key +func PlatformKey(os string, arch string, variant string) string { + key := os + "/" + arch + if variant != "" { + key += "/" + variant + } + return key +} + +// MatchesPlatform returns true if given OS name matches the OS set in options +func (o *ManifestOptions) WantsPlatform(os string, arch string, variant string) bool { + o.mutex.RLock() + defer o.mutex.RUnlock() + if len(o.platforms) == 0 { + return true + } + _, ok := o.platforms[PlatformKey(os, arch, variant)] + return ok +} + +// WithPlatform sets a platform filter for options o +func (o *ManifestOptions) WithPlatform(os string, arch string, variant string) *ManifestOptions { + o.mutex.Lock() + defer o.mutex.Unlock() + if o.platforms == nil { + o.platforms = map[string]bool{} + } + log.Debugf("Adding platform " + PlatformKey(os, arch, variant)) + o.platforms[PlatformKey(os, arch, variant)] = true + return o +} + +func (o *ManifestOptions) Platforms() []string { + o.mutex.RLock() + defer o.mutex.RUnlock() + if len(o.platforms) == 0 { + return []string{} + } + keys := make([]string, 0, len(o.platforms)) + for k := range o.platforms { + keys = append(keys, k) + } + // We sort the slice before returning it, to guarantee stable order + sort.Strings(keys) + return keys +} + +// WantsMetdata returns true if metadata should be requested +func (o *ManifestOptions) WantsMetdata() bool { + return o.metadata +} + +// WithMetadata sets metadata to be requested +func (o *ManifestOptions) WithMetadata() *ManifestOptions { + o.metadata = true + return o +} + +// WithoutMetadata sets metadata not not be requested +func (o *ManifestOptions) WithoutMetadata() *ManifestOptions { + o.metadata = false + return o +} diff --git a/pkg/options/options_test.go b/pkg/options/options_test.go new file mode 100644 index 0000000..d0a7fdc --- /dev/null +++ b/pkg/options/options_test.go @@ -0,0 +1,74 @@ +package options + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_PlatformKey(t *testing.T) { + t.Run("Without variant", func(t *testing.T) { + assert.Equal(t, PlatformKey("linux", "amd64", ""), "linux/amd64") + }) + t.Run("With variant", func(t *testing.T) { + assert.Equal(t, PlatformKey("linux", "arm", "v6"), "linux/arm/v6") + }) +} + +func Test_WantsPlatform(t *testing.T) { + opts := NewManifestOptions() + t.Run("Empty options", func(t *testing.T) { + assert.True(t, opts.WantsPlatform("linux", "arm", "v7")) + assert.True(t, opts.WantsPlatform("linux", "amd64", "")) + }) + t.Run("Single platform", func(t *testing.T) { + opts = opts.WithPlatform("linux", "arm", "v7") + assert.True(t, opts.WantsPlatform("linux", "arm", "v7")) + }) + t.Run("Platform appended", func(t *testing.T) { + opts = opts.WithPlatform("linux", "arm", "v8") + assert.True(t, opts.WantsPlatform("linux", "arm", "v7")) + assert.True(t, opts.WantsPlatform("linux", "arm", "v8")) + }) + t.Run("Uninitialized options", func(t *testing.T) { + opts := &ManifestOptions{} + assert.True(t, opts.WantsPlatform("linux", "amd64", "")) + opts = (&ManifestOptions{}).WithPlatform("linux", "amd64", "") + assert.True(t, opts.WantsPlatform("linux", "amd64", "")) + }) +} + +func Test_WantsMetadata(t *testing.T) { + opts := NewManifestOptions() + t.Run("Empty options", func(t *testing.T) { + assert.False(t, opts.WantsMetdata()) + }) + t.Run("Wants metadata", func(t *testing.T) { + opts = opts.WithMetadata() + assert.True(t, opts.WantsMetdata()) + }) + t.Run("Does not want metadata", func(t *testing.T) { + opts = opts.WithoutMetadata() + assert.False(t, opts.WantsMetdata()) + }) +} + +func Test_Platforms(t *testing.T) { + opts := NewManifestOptions() + t.Run("Empty platforms returns empty array", func(t *testing.T) { + ps := opts.Platforms() + assert.Len(t, ps, 0) + }) + t.Run("Single platform without variant", func(t *testing.T) { + ps := opts.WithPlatform("linux", "amd64", "").Platforms() + require.Len(t, ps, 1) + assert.Equal(t, ps[0], PlatformKey("linux", "amd64", "")) + }) + t.Run("Single platform with variant", func(t *testing.T) { + ps := opts.WithPlatform("linux", "arm", "v8").Platforms() + require.Len(t, ps, 2) + assert.Equal(t, ps[0], PlatformKey("linux", "amd64", "")) + assert.Equal(t, ps[1], PlatformKey("linux", "arm", "v8")) + }) +} diff --git a/pkg/registry/client.go b/pkg/registry/client.go index 11eca9e..2f8a7c9 100644 --- a/pkg/registry/client.go +++ b/pkg/registry/client.go @@ -9,9 +9,11 @@ import ( "github.com/argoproj-labs/argocd-image-updater/pkg/log" "github.com/argoproj-labs/argocd-image-updater/pkg/metrics" + "github.com/argoproj-labs/argocd-image-updater/pkg/options" "github.com/argoproj-labs/argocd-image-updater/pkg/tag" "github.com/distribution/distribution/v3" + "github.com/distribution/distribution/v3/manifest/manifestlist" "github.com/distribution/distribution/v3/manifest/ocischema" "github.com/distribution/distribution/v3/manifest/schema1" "github.com/distribution/distribution/v3/manifest/schema2" @@ -37,8 +39,9 @@ import ( type RegistryClient interface { NewRepository(nameInRepository string) error Tags() ([]string, error) - Manifest(tagStr string) (distribution.Manifest, error) - TagMetadata(manifest distribution.Manifest) (*tag.TagInfo, error) + ManifestForTag(tagStr string) (distribution.Manifest, error) + ManifestForDigest(dgst digest.Digest) (distribution.Manifest, error) + TagMetadata(manifest distribution.Manifest, opts *options.ManifestOptions) (*tag.TagInfo, error) } type NewRegistryClient func(*RegistryEndpoint, string, string) (RegistryClient, error) @@ -151,12 +154,12 @@ func (clt *registryClient) Tags() ([]string, error) { } // Manifest returns a Manifest for a given tag in repository -func (clt *registryClient) Manifest(tagStr string) (distribution.Manifest, error) { +func (clt *registryClient) ManifestForTag(tagStr string) (distribution.Manifest, error) { manService, err := clt.regClient.Manifests(context.Background()) if err != nil { return nil, err } - mediaType := []string{ocischema.SchemaVersion.MediaType, schema1.MediaTypeSignedManifest, schema2.SchemaVersion.MediaType} + mediaType := []string{ocischema.SchemaVersion.MediaType, schema1.MediaTypeSignedManifest, schema2.SchemaVersion.MediaType, manifestlist.SchemaVersion.MediaType} manifest, err := manService.Get( context.Background(), digest.FromString(tagStr), @@ -167,18 +170,40 @@ func (clt *registryClient) Manifest(tagStr string) (distribution.Manifest, error return manifest, nil } +// ManifestForDigest returns a Manifest for a given digest in repository +func (clt *registryClient) ManifestForDigest(dgst digest.Digest) (distribution.Manifest, error) { + manService, err := clt.regClient.Manifests(context.Background()) + if err != nil { + return nil, err + } + mediaType := []string{ocischema.SchemaVersion.MediaType, schema1.MediaTypeSignedManifest, schema2.SchemaVersion.MediaType, manifestlist.SchemaVersion.MediaType} + manifest, err := manService.Get( + context.Background(), + dgst, + distribution.WithManifestMediaTypes(mediaType)) + if err != nil { + return nil, err + } + return manifest, nil +} + // TagMetadata retrieves metadata for a given manifest of given repository -func (client *registryClient) TagMetadata(manifest distribution.Manifest) (*tag.TagInfo, error) { +func (client *registryClient) TagMetadata(manifest distribution.Manifest, opts *options.ManifestOptions) (*tag.TagInfo, error) { ti := &tag.TagInfo{} var info struct { Arch string `json:"architecture"` Created string `json:"created"` OS string `json:"os"` + Variant string `json:"variant"` } + + // We support the following types of manifests as returned by the registry: + // + // V1 (legacy, might go away), V2 and OCI + // + // Also ManifestLists (e.g. on multi-arch images) are supported. // - // We support both V1,V2 AND OCI manifest schemas. Everything else will trigger - // an error. switch deserialized := manifest.(type) { case *schema1.SignedManifest: @@ -186,25 +211,119 @@ func (client *registryClient) TagMetadata(manifest distribution.Manifest) (*tag. if len(man.History) == 0 { return nil, fmt.Errorf("no history information found in schema V1") } + + _, mBytes, err := manifest.Payload() + if err != nil { + return nil, err + } + ti.Digest = sha256.Sum256(mBytes) + + log.Tracef("v1 SHA digest is %s", ti.EncodedDigest()) if err := json.Unmarshal([]byte(man.History[0].V1Compatibility), &info); err != nil { return nil, err } + if !opts.WantsPlatform(info.OS, info.Arch, "") { + log.Debugf("ignoring v1 manifest %v. Manifest platform: %s, requested: %s", + ti.EncodedDigest(), options.PlatformKey(info.OS, info.Arch, info.Variant), strings.Join(opts.Platforms(), ",")) + return nil, nil + } if createdAt, err := time.Parse(time.RFC3339Nano, info.Created); err != nil { return nil, err } else { ti.CreatedAt = createdAt } - _, mBytes, err := manifest.Payload() + return ti, nil + + case *manifestlist.DeserializedManifestList: + var list manifestlist.DeserializedManifestList = *deserialized + var ml []distribution.Descriptor + platforms := []string{} + + // List must contain at least one image manifest + if len(list.Manifests) == 0 { + return nil, fmt.Errorf("empty manifestlist not supported") + } + + // We use the SHA from the manifest list to let the container engine + // decide which image to pull, in case of multi-arch clusters. + _, mBytes, err := list.Payload() if err != nil { - return nil, err + return nil, fmt.Errorf("could not retrieve manifestlist payload: %v", err) } ti.Digest = sha256.Sum256(mBytes) - log.Tracef("v1 SHA digest is %s", fmt.Sprintf("sha256:%x", ti.Digest)) + + log.Tracef("SHA256 of manifest parent is %v", ti.EncodedDigest()) + + for _, ref := range list.References() { + platforms = append(platforms, ref.Platform.OS+"/"+ref.Platform.Architecture) + log.Tracef("Found %s", options.PlatformKey(ref.Platform.OS, ref.Platform.Architecture, ref.Platform.Variant)) + if !opts.WantsPlatform(ref.Platform.OS, ref.Platform.Architecture, ref.Platform.Variant) { + log.Tracef("Ignoring referenced manifest %v because platform %s does not match any of: %s", + ref.Digest, + options.PlatformKey(ref.Platform.OS, ref.Platform.Architecture, ref.Platform.Variant), + strings.Join(opts.Platforms(), ",")) + continue + } + ml = append(ml, ref) + } + + // We need at least one reference that matches requested plaforms + if len(ml) == 0 { + log.Debugf("Manifest list did not contain any usable reference. Platforms requested: (%s), platforms included: (%s)", + strings.Join(opts.Platforms(), ","), strings.Join(platforms, ",")) + return nil, nil + } + + // For some strategies, we do not need to fetch metadata for further + // processing. + if !opts.WantsMetdata() { + return ti, nil + } + + // Loop through all referenced manifests to get their metadata. We only + // consider manifests for platforms we are interested in. + for _, ref := range ml { + log.Tracef("Inspecting metadata of reference: %v", ref.Digest) + + man, err := client.ManifestForDigest(ref.Digest) + if err != nil { + return nil, fmt.Errorf("could not fetch manifest %v: %v", ref.Digest, err) + } + + cti, err := client.TagMetadata(man, opts) + if err != nil { + return nil, fmt.Errorf("could not fetch metadata for manifest %v: %v", ref.Digest, err) + } + + // We save the timestamp of the most recent pushed manifest for any + // given reference, if the metadata for the tag was correctly + // retrieved. This is important for the latest update strategy to + // be able to handle multi-arch images. The latest strategy will + // consider the most recent reference from a manifest list. + if cti != nil { + if cti.CreatedAt.After(ti.CreatedAt) { + ti.CreatedAt = cti.CreatedAt + } + } else { + log.Warnf("returned metadata for manifest %v is nil, this should not happen.", ref.Digest) + continue + } + } + return ti, nil case *schema2.DeserializedManifest: var man schema2.Manifest = deserialized.Manifest + log.Tracef("Manifest digest is %v", man.Config.Digest.Encoded()) + + _, mBytes, err := manifest.Payload() + if err != nil { + return nil, err + } + ti.Digest = sha256.Sum256(mBytes) + log.Tracef("v2 SHA digest is %s", ti.EncodedDigest()) + // The data we require from a V2 manifest is in a blob that we need to // fetch from the registry. blobReader, err := client.regClient.Blobs(context.Background()).Get(context.Background(), man.Config.Digest) @@ -216,19 +335,26 @@ func (client *registryClient) TagMetadata(manifest distribution.Manifest) (*tag. return nil, err } + if !opts.WantsPlatform(info.OS, info.Arch, info.Variant) { + log.Debugf("ignoring v2 manifest %v. Manifest platform: %s/%s, requested: %s", + ti.EncodedDigest(), info.OS, info.Arch, strings.Join(opts.Platforms(), ",")) + return nil, nil + } + if ti.CreatedAt, err = time.Parse(time.RFC3339Nano, info.Created); err != nil { return nil, err } + return ti, nil + case *ocischema.DeserializedManifest: + var man ocischema.Manifest = deserialized.Manifest + _, mBytes, err := manifest.Payload() if err != nil { return nil, err } ti.Digest = sha256.Sum256(mBytes) - log.Tracef("v2 SHA digest is %s", fmt.Sprintf("sha256:%x", ti.Digest)) - return ti, nil - case *ocischema.DeserializedManifest: - var man ocischema.Manifest = deserialized.Manifest + log.Tracef("OCI SHA digest is %s", ti.EncodedDigest()) // The data we require from a V2 manifest is in a blob that we need to // fetch from the registry. @@ -241,16 +367,16 @@ func (client *registryClient) TagMetadata(manifest distribution.Manifest) (*tag. return nil, err } - if ti.CreatedAt, err = time.Parse(time.RFC3339Nano, info.Created); err != nil { - return nil, err + if !opts.WantsPlatform(info.OS, info.Arch, info.Variant) { + log.Debugf("ignoring OCI manifest %v. Manifest platform: %s/%s, requested: %s", + ti.EncodedDigest(), info.OS, info.Arch, strings.Join(opts.Platforms(), ",")) + return nil, nil } - _, mBytes, err := manifest.Payload() - if err != nil { + if ti.CreatedAt, err = time.Parse(time.RFC3339Nano, info.Created); err != nil { return nil, err } - ti.Digest = sha256.Sum256(mBytes) - log.Tracef("oci SHA digest is %s", fmt.Sprintf("sha256:%x", ti.Digest)) + return ti, nil default: return nil, fmt.Errorf("invalid manifest type") diff --git a/pkg/registry/client_test.go b/pkg/registry/client_test.go index 7d0c3a0..8bf2a8e 100644 --- a/pkg/registry/client_test.go +++ b/pkg/registry/client_test.go @@ -3,6 +3,8 @@ package registry import ( "testing" + "github.com/argoproj-labs/argocd-image-updater/pkg/options" + "github.com/distribution/distribution/v3/manifest/schema1" "github.com/stretchr/testify/require" ) @@ -18,7 +20,7 @@ func Test_TagMetadata(t *testing.T) { require.NoError(t, err) client, err := NewClient(ep, "", "") require.NoError(t, err) - _, err = client.TagMetadata(meta1) + _, err = client.TagMetadata(meta1, &options.ManifestOptions{}) require.Error(t, err) }) @@ -37,7 +39,7 @@ func Test_TagMetadata(t *testing.T) { require.NoError(t, err) client, err := NewClient(ep, "", "") require.NoError(t, err) - _, err = client.TagMetadata(meta1) + _, err = client.TagMetadata(meta1, &options.ManifestOptions{}) require.Error(t, err) }) @@ -56,7 +58,7 @@ func Test_TagMetadata(t *testing.T) { require.NoError(t, err) client, err := NewClient(ep, "", "") require.NoError(t, err) - _, err = client.TagMetadata(meta1) + _, err = client.TagMetadata(meta1, &options.ManifestOptions{}) require.Error(t, err) }) @@ -76,7 +78,7 @@ func Test_TagMetadata(t *testing.T) { require.NoError(t, err) client, err := NewClient(ep, "", "") require.NoError(t, err) - _, err = client.TagMetadata(meta1) + _, err = client.TagMetadata(meta1, &options.ManifestOptions{}) require.Error(t, err) }) } diff --git a/pkg/registry/mocks/RegistryClient.go b/pkg/registry/mocks/RegistryClient.go index c6cdb72..e5309c8 100644 --- a/pkg/registry/mocks/RegistryClient.go +++ b/pkg/registry/mocks/RegistryClient.go @@ -4,7 +4,12 @@ package mocks import ( distribution "github.com/distribution/distribution/v3" + digest "github.com/opencontainers/go-digest" + mock "github.com/stretchr/testify/mock" + + options "github.com/argoproj-labs/argocd-image-updater/pkg/options" + tag "github.com/argoproj-labs/argocd-image-updater/pkg/tag" ) @@ -13,21 +18,22 @@ type RegistryClient struct { mock.Mock } -func (_m *RegistryClient) TagMetadata(manifest distribution.Manifest) (*tag.TagInfo, error) { - ret := _m.Called(manifest) +// ManifestForDigest provides a mock function with given fields: dgst +func (_m *RegistryClient) ManifestForDigest(dgst digest.Digest) (distribution.Manifest, error) { + ret := _m.Called(dgst) - var r0 *tag.TagInfo - if rf, ok := ret.Get(0).(func(distribution.Manifest) *tag.TagInfo); ok { - r0 = rf(manifest) + var r0 distribution.Manifest + if rf, ok := ret.Get(0).(func(digest.Digest) distribution.Manifest); ok { + r0 = rf(dgst) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(*tag.TagInfo) + r0 = ret.Get(0).(distribution.Manifest) } } var r1 error - if rf, ok := ret.Get(1).(func(distribution.Manifest) error); ok { - r1 = rf(manifest) + if rf, ok := ret.Get(1).(func(digest.Digest) error); ok { + r1 = rf(dgst) } else { r1 = ret.Error(1) } @@ -35,21 +41,22 @@ func (_m *RegistryClient) TagMetadata(manifest distribution.Manifest) (*tag.TagI return r0, r1 } -// Tags provides a mock function with given fields: nameInRepository -func (_m *RegistryClient) Tags() ([]string, error) { - ret := _m.Called() - var r0 []string - if rf, ok := ret.Get(0).(func() []string); ok { - r0 = rf() +// ManifestForTag provides a mock function with given fields: tagStr +func (_m *RegistryClient) ManifestForTag(tagStr string) (distribution.Manifest, error) { + ret := _m.Called(tagStr) + + var r0 distribution.Manifest + if rf, ok := ret.Get(0).(func(string) distribution.Manifest); ok { + r0 = rf(tagStr) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) + r0 = ret.Get(0).(distribution.Manifest) } } var r1 error - if rf, ok := ret.Get(1).(func() error); ok { - r1 = rf() + if rf, ok := ret.Get(1).(func(string) error); ok { + r1 = rf(tagStr) } else { r1 = ret.Error(1) } @@ -57,21 +64,36 @@ func (_m *RegistryClient) Tags() ([]string, error) { return r0, r1 } -func (_m *RegistryClient) Manifest(tagStr string) (distribution.Manifest, error) { - ret := _m.Called(tagStr) +// NewRepository provides a mock function with given fields: nameInRepository +func (_m *RegistryClient) NewRepository(nameInRepository string) error { + ret := _m.Called(nameInRepository) - var r0 distribution.Manifest - if rf, ok := ret.Get(0).(func(string) distribution.Manifest); ok { - r0 = rf(tagStr) + var r0 error + if rf, ok := ret.Get(0).(func(string) error); ok { + r0 = rf(nameInRepository) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// TagMetadata provides a mock function with given fields: manifest, opts +func (_m *RegistryClient) TagMetadata(manifest distribution.Manifest, opts *options.ManifestOptions) (*tag.TagInfo, error) { + ret := _m.Called(manifest, opts) + + var r0 *tag.TagInfo + if rf, ok := ret.Get(0).(func(distribution.Manifest, *options.ManifestOptions) *tag.TagInfo); ok { + r0 = rf(manifest, opts) } else { if ret.Get(0) != nil { - r0 = ret.Get(0).(distribution.Manifest) + r0 = ret.Get(0).(*tag.TagInfo) } } var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(tagStr) + if rf, ok := ret.Get(1).(func(distribution.Manifest, *options.ManifestOptions) error); ok { + r1 = rf(manifest, opts) } else { r1 = ret.Error(1) } @@ -79,15 +101,25 @@ func (_m *RegistryClient) Manifest(tagStr string) (distribution.Manifest, error) return r0, r1 } -func (_m *RegistryClient) NewRepository(nameInRepository string) (error){ - ret := _m.Called(nameInRepository) +// Tags provides a mock function with given fields: +func (_m *RegistryClient) Tags() ([]string, error) { + ret := _m.Called() + + var r0 []string + if rf, ok := ret.Get(0).(func() []string); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]string) + } + } var r1 error - if rf, ok := ret.Get(0).(func(string) error); ok { - r1 = rf(nameInRepository) + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() } else { - r1 = ret.Error(0) + r1 = ret.Error(1) } - return r1 + return r0, r1 } diff --git a/pkg/registry/registry.go b/pkg/registry/registry.go index dccba2d..561b85c 100644 --- a/pkg/registry/registry.go +++ b/pkg/registry/registry.go @@ -142,14 +142,14 @@ func (endpoint *RegistryEndpoint) GetTags(img *image.ContainerImage, regClient R // We first try to fetch a V2 manifest, and if that's not available we fall // back to fetching V1 manifest. If that fails also, we just skip this tag. - if ml, err = regClient.Manifest(tagStr); err != nil { + if ml, err = regClient.ManifestForTag(tagStr); err != nil { log.Errorf("Error fetching metadata for %s:%s - neither V1 or V2 or OCI manifest returned by registry: %v", nameInRegistry, tagStr, err) return } // Parse required meta data from the manifest. The metadata contains all // information needed to decide whether to consider this tag or not. - ti, err := regClient.TagMetadata(ml) + ti, err := regClient.TagMetadata(ml, vc.Options) if err != nil { log.Errorf("error fetching metadata for %s:%s: %v", nameInRegistry, tagStr, err) return diff --git a/pkg/registry/registry_test.go b/pkg/registry/registry_test.go index fb61077..b1c2983 100644 --- a/pkg/registry/registry_test.go +++ b/pkg/registry/registry_test.go @@ -90,7 +90,7 @@ func Test_GetTags(t *testing.T) { regClient := mocks.RegistryClient{} regClient.On("NewRepository", mock.Anything).Return(nil) regClient.On("Tags", mock.Anything).Return([]string{"1.2.0", "1.2.1", "1.2.2"}, nil) - regClient.On("Manifest", mock.Anything, mock.Anything).Return(meta1, nil) + regClient.On("ManifestForTag", mock.Anything, mock.Anything).Return(meta1, nil) regClient.On("TagMetadata", mock.Anything, mock.Anything).Return(&tag.TagInfo{}, nil) ep, err := GetRegistryEndpoint("") diff --git a/pkg/tag/tag.go b/pkg/tag/tag.go index d89d263..2a89aec 100644 --- a/pkg/tag/tag.go +++ b/pkg/tag/tag.go @@ -1,6 +1,7 @@ package tag import ( + "encoding/hex" "sort" "sync" "time" @@ -178,3 +179,7 @@ func (il ImageTagList) unlockedContains(tag *ImageTag) bool { } return false } + +func (ti *TagInfo) EncodedDigest() string { + return "sha256:" + hex.EncodeToString(ti.Digest[:]) +} |
