From 64b50e8182def90063bf196885381a3f73cdfe30 Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Thu, 20 Apr 2023 19:16:02 +0300 Subject: [PATCH] feat(multiarch): support cleanup of images built in multiarch mode Signed-off-by: Timofey Kirillov --- pkg/build/build_phase.go | 4 +- pkg/build/image/multiplatform_image.go | 2 +- pkg/cleaning/cleanup.go | 97 +++++++-- pkg/cleaning/stage_manager/manager.go | 1 + pkg/docker_registry/api.go | 197 +++++++++++------- pkg/docker_registry/generic_api.go | 9 +- pkg/docker_registry/interface.go | 4 +- pkg/image/info.go | 13 +- pkg/image/stage.go | 15 +- pkg/image/summary.go | 2 +- pkg/storage/local_stages_storage.go | 6 +- pkg/storage/manager/storage_manager.go | 15 +- pkg/storage/repo_stages_storage.go | 28 ++- pkg/storage/stages_storage.go | 2 +- ...tages_storage_cache_http_handler_legacy.go | 2 +- 15 files changed, 262 insertions(+), 135 deletions(-) diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 10682fa686..4be5f6cd7f 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -372,7 +372,7 @@ func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context, return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, img.GetStageID(), err) } - desc, err := stagesStorage.GetStageDescription(ctx, phase.Conveyor.ProjectName(), img.GetStageID().Digest, img.GetStageID().UniqueID) + desc, err := stagesStorage.GetStageDescription(ctx, phase.Conveyor.ProjectName(), img.GetStageID()) if err != nil { return fmt.Errorf("unable to get image %s %s descriptor: %w", name, img.GetStageID(), err) } @@ -1093,7 +1093,7 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *image.I return fmt.Errorf("unable to store stage %s digest %s image %s into repo %s: %w", stg.LogDetailedName(), stg.GetDigest(), stageImage.Image.Name(), phase.Conveyor.StorageManager.GetStagesStorage().String(), err) } - if desc, err := phase.Conveyor.StorageManager.GetStagesStorage().GetStageDescription(ctx, phase.Conveyor.ProjectName(), stg.GetDigest(), uniqueID); err != nil { + if desc, err := phase.Conveyor.StorageManager.GetStagesStorage().GetStageDescription(ctx, phase.Conveyor.ProjectName(), *imagePkg.NewStageID(stg.GetDigest(), uniqueID)); err != nil { return fmt.Errorf("unable to get stage %s digest %s image %s description from repo %s after stages has been stored into repo: %w", stg.LogDetailedName(), stg.GetDigest(), stageImage.Image.Name(), phase.Conveyor.StorageManager.GetStagesStorage().String(), err) } else { stageImage.Image.SetStageDescription(desc) diff --git a/pkg/build/image/multiplatform_image.go b/pkg/build/image/multiplatform_image.go index 9d3bf05a36..3870fb071e 100644 --- a/pkg/build/image/multiplatform_image.go +++ b/pkg/build/image/multiplatform_image.go @@ -35,7 +35,7 @@ func NewMultiplatformImage(name string, images []*Image, storageManager manager. return stageDesc.StageID.String() }) img.calculatedDigest = util.Sha3_224Hash(metaStageDeps...) - img.stageID = common_image.StageID{Digest: img.GetDigest()} + img.stageID = *common_image.NewStageID(img.GetDigest(), 0) return img } diff --git a/pkg/cleaning/cleanup.go b/pkg/cleaning/cleanup.go index fc5606d551..71f48cd5fd 100644 --- a/pkg/cleaning/cleanup.go +++ b/pkg/cleaning/cleanup.go @@ -669,13 +669,15 @@ func (m *cleanupManager) cleanupUnusedStages(ctx context.Context) error { // skip stages and their relatives covered by Kubernetes- or git history-based cleanup policies stageDescriptionListToDelete := stageDescriptionList + { excludedSDListByReason := make(map[string][]*image.StageDescription) for reason, sdList := range m.stageManager.GetProtectedStageDescriptionListByReason() { for _, sd := range sdList { var excludedSDListBySD []*image.StageDescription - stageDescriptionListToDelete, excludedSDListBySD = m.excludeStageAndRelativesByImageID(stageDescriptionListToDelete, sd.Info.ID) + + stageDescriptionListToDelete, excludedSDListBySD = m.excludeStageAndRelativesByImage(stageDescriptionListToDelete, sd.Info) for _, exclSD := range excludedSDListBySD { if sd.Info.Name == exclSD.Info.Name { @@ -714,7 +716,7 @@ func (m *cleanupManager) cleanupUnusedStages(ctx context.Context) error { for _, sd := range stageDescriptionListToDelete { if (time.Since(sd.Info.GetCreatedAt()).Hours()) <= float64(keepImagesBuiltWithinLastNHours) { var excludedRelativesSDList []*image.StageDescription - stageDescriptionListToDelete, excludedRelativesSDList = m.excludeStageAndRelativesByImageID(stageDescriptionListToDelete, sd.Info.ID) + stageDescriptionListToDelete, excludedRelativesSDList = m.excludeStageAndRelativesByImage(stageDescriptionListToDelete, sd.Info) excludedSDList = append(excludedSDList, excludedRelativesSDList...) } } @@ -854,36 +856,95 @@ func deleteImportsMetadata(ctx context.Context, projectName string, storageManag }) } -func (m *cleanupManager) excludeStageAndRelativesByImageID(stages []*image.StageDescription, imageID string) ([]*image.StageDescription, []*image.StageDescription) { - stage := findStageByImageID(stages, imageID) - if stage == nil { +func (m *cleanupManager) excludeStageAndRelativesByImage(stages []*image.StageDescription, excludeImage *image.Info) ([]*image.StageDescription, []*image.StageDescription) { + excludeStages := findStagesByImageID(stages, excludeImage) + if len(excludeStages) == 0 { return stages, []*image.StageDescription{} } + return m.excludeStageAndRelativesByStages(stages, excludeStages) +} + +// findStagesByImage could return multiple stages when target image.Info is an image index +func findStagesByImage(stages []*image.StageDescription, target *image.Info, imageMatcher, indexMatcher func(stg *image.StageDescription, target *image.Info) bool) (res []*image.StageDescription) { + if target.IsIndex { + if indexMatcher != nil { + for _, stg := range stages { + if indexMatcher(stg, target) { + res = append(res, stg) + } + } + } + for _, subimg := range target.Index { + res = append(res, findStagesByImage(stages, subimg, imageMatcher, indexMatcher)...) + } + } else if imageMatcher != nil { + for _, stg := range stages { + if imageMatcher(stg, target) { + res = append(res, stg) + } + } + } + return +} - return m.excludeStageAndRelativesByStage(stages, stage) +func findStagesByImageID(stages []*image.StageDescription, target *image.Info) []*image.StageDescription { + return findStagesByImage( + stages, target, + func(stg *image.StageDescription, target *image.Info) bool { + return stg.Info.ID == target.ID + }, + func(stg *image.StageDescription, target *image.Info) bool { + return stg.Info.Name == target.Name + }, + ) +} + +func findStagesByImageParentID(stages []*image.StageDescription, target *image.Info) []*image.StageDescription { + return findStagesByImage( + stages, target, + func(stg *image.StageDescription, target *image.Info) bool { + return stg.Info.ID == target.ParentID + }, nil, + ) } func findStageByImageID(stages []*image.StageDescription, imageID string) *image.StageDescription { - for _, stage := range stages { - if stage.Info.ID == imageID { - return stage + for _, stg := range stages { + if stg.Info.ID == imageID { + return stg } } - return nil } -func (m *cleanupManager) excludeStageAndRelativesByStage(stages []*image.StageDescription, stage *image.StageDescription) ([]*image.StageDescription, []*image.StageDescription) { +func appendUniqueStageDescriptions(stages []*image.StageDescription, newStages ...*image.StageDescription) []*image.StageDescription { +appendUnique: + for _, newStg := range newStages { + for _, stg := range stages { + if stg.StageID.IsEqual(*newStg.StageID) { + continue appendUnique + } + } + stages = append(stages, newStg) + } + return stages +} + +func (m *cleanupManager) excludeStageAndRelativesByStages(stages, stagesToExclude []*image.StageDescription) ([]*image.StageDescription, []*image.StageDescription) { var excludedStages []*image.StageDescription - currentStage := stage - for { - stages = excludeStages(stages, currentStage) - excludedStages = append(excludedStages, currentStage) + currentStagesToExclude := stagesToExclude + + for len(currentStagesToExclude) > 0 { + stages = excludeStages(stages, currentStagesToExclude...) + excludedStages = appendUniqueStageDescriptions(excludedStages, currentStagesToExclude...) - currentStage = findStageByImageID(stages, currentStage.Info.ParentID) - if currentStage == nil { - break + var nextStagesToExclude []*image.StageDescription + for _, excludeStg := range currentStagesToExclude { + excludeByParents := findStagesByImageParentID(stages, excludeStg.Info) + nextStagesToExclude = appendUniqueStageDescriptions(nextStagesToExclude, excludeByParents...) } + + currentStagesToExclude = nextStagesToExclude } return stages, excludedStages diff --git a/pkg/cleaning/stage_manager/manager.go b/pkg/cleaning/stage_manager/manager.go index c22631668b..c205efede7 100644 --- a/pkg/cleaning/stage_manager/manager.go +++ b/pkg/cleaning/stage_manager/manager.go @@ -27,6 +27,7 @@ func NewManager() Manager { type stage struct { stageID string + isMultiplatform bool description *image.StageDescription isProtected bool protectionReason string diff --git a/pkg/docker_registry/api.go b/pkg/docker_registry/api.go index 0abe6433e3..bda46b8a23 100644 --- a/pkg/docker_registry/api.go +++ b/pkg/docker_registry/api.go @@ -109,67 +109,114 @@ func (api *api) TryGetRepoImage(ctx context.Context, reference string) (*image.I } } -func (api *api) GetRepoImage(_ context.Context, reference string) (*image.Info, error) { - imageInfo, _, err := api.image(reference) +func (api *api) GetRepoImage(ctx context.Context, reference string) (*image.Info, error) { + desc, _, err := api.getImageDesc(reference) if err != nil { return nil, err } - digest, err := imageInfo.Digest() + referenceParts, err := api.parseReferenceParts(desc.Ref.Name()) if err != nil { - return nil, err + return nil, fmt.Errorf("unable to parse reference %q: %w", desc.Ref.Name(), err) } - manifest, err := imageInfo.Manifest() + return api.getRepoImageByDesc(ctx, referenceParts.tag, desc) +} + +func (api *api) getRepoImageByDesc(ctx context.Context, originalTag string, desc *remote.Descriptor) (*image.Info, error) { + referenceParts, err := api.parseReferenceParts(desc.Ref.Name()) if err != nil { - return nil, err + return nil, fmt.Errorf("unable to parse reference %q: %w", desc.Ref.Name(), err) } - configFile, err := imageInfo.ConfigFile() - if err != nil { - return nil, err + repoImage := &image.Info{ + Name: desc.Ref.Name(), + Repository: strings.Join([]string{referenceParts.registry, referenceParts.repository}, "/"), + Tag: originalTag, } - var totalSize int64 - if layers, err := imageInfo.Layers(); err != nil { - return nil, err - } else { - for _, l := range layers { - if lSize, err := l.Size(); err != nil { - return nil, err - } else { - totalSize += lSize - } + if desc.MediaType.IsIndex() { + repoImage.IsIndex = true + + ii, err := desc.ImageIndex() + if err != nil { + return nil, fmt.Errorf("error getting image index: %w", err) } - } - referenceParts, err := api.parseReferenceParts(reference) - if err != nil { - return nil, fmt.Errorf("unable to parse reference %q: %w", reference, err) - } + digest, err := ii.Digest() + if err != nil { + return nil, fmt.Errorf("error getting image index digest: %w", err) + } + repoImage.RepoDigest = digest.String() - var parentID string - if baseImageID, ok := configFile.Config.Labels["werf.io/base-image-id"]; ok { - parentID = baseImageID + im, err := ii.IndexManifest() + if err != nil { + return nil, fmt.Errorf("error getting image manifest: %w", err) + } + + for _, desc := range im.Manifests { + subref := fmt.Sprintf("%s@%s", repoImage.Repository, desc.Digest) + subDesc, _, err := api.getImageDesc(subref) + if err != nil { + return nil, fmt.Errorf("error getting image %s manifest: %w", subref, err) + } + + subInfo, err := api.getRepoImageByDesc(ctx, originalTag, subDesc) + if err != nil { + return nil, fmt.Errorf("error getting image %s descriptor: %w", subref, err) + } + repoImage.Index = append(repoImage.Index, subInfo) + } } else { - // TODO(1.3): Legacy compatibility mode - parentID = configFile.Config.Image - } + imageInfo, err := desc.Image() + if err != nil { + return nil, fmt.Errorf("error getting image manifest: %w", err) + } - repoImage := &image.Info{ - Name: reference, - Repository: strings.Join([]string{referenceParts.registry, referenceParts.repository}, "/"), - ID: manifest.Config.Digest.String(), - Tag: referenceParts.tag, - RepoDigest: digest.String(), - ParentID: parentID, - Labels: configFile.Config.Labels, - OnBuild: configFile.Config.OnBuild, - Env: configFile.Config.Env, - Size: totalSize, - } + digest, err := imageInfo.Digest() + if err != nil { + return nil, err + } + repoImage.RepoDigest = digest.String() + + manifest, err := imageInfo.Manifest() + if err != nil { + return nil, err + } + repoImage.ID = manifest.Config.Digest.String() - repoImage.SetCreatedAtUnix(configFile.Created.Unix()) + configFile, err := imageInfo.ConfigFile() + if err != nil { + return nil, err + } + repoImage.Labels = configFile.Config.Labels + repoImage.OnBuild = configFile.Config.OnBuild + repoImage.Env = configFile.Config.Env + repoImage.SetCreatedAtUnix(configFile.Created.Unix()) + + var totalSize int64 + if layers, err := imageInfo.Layers(); err != nil { + return nil, err + } else { + for _, l := range layers { + if lSize, err := l.Size(); err != nil { + return nil, err + } else { + totalSize += lSize + } + } + } + repoImage.Size = totalSize + + var parentID string + if baseImageID, ok := configFile.Config.Labels["werf.io/base-image-id"]; ok { + parentID = baseImageID + } else { + // TODO(1.3): Legacy compatibility mode + parentID = configFile.Config.Image + } + repoImage.ParentID = parentID + } return repoImage, nil } @@ -206,10 +253,20 @@ func (api *api) deleteImageByReference(ctx context.Context, reference string) er } func (api *api) MutateAndPushImage(_ context.Context, sourceReference, destinationReference string, mutateConfigFunc func(cfg v1.Config) (v1.Config, error)) error { - img, _, err := api.image(sourceReference) + desc, _, err := api.getImageDesc(sourceReference) if err != nil { return fmt.Errorf("error reading image %q: %w", sourceReference, err) } + + if desc.MediaType.IsIndex() { + panic("unexpected condition, please report a bug") + } + + img, err := desc.Image() + if err != nil { + return fmt.Errorf("error getting image manifest: %w", err) + } + cfgFile, err := img.ConfigFile() if err != nil { return fmt.Errorf("error reading config file %q: %w", sourceReference, err) @@ -235,26 +292,31 @@ func (api *api) MutateAndPushImage(_ context.Context, sourceReference, destinati } func (api *api) CopyImage(ctx context.Context, sourceReference, destinationReference string, opts CopyImageOptions) error { - ref, err := name.ParseReference(destinationReference, api.parseReferenceOptions()...) + dstRef, err := name.ParseReference(destinationReference, api.parseReferenceOptions()...) if err != nil { return fmt.Errorf("parsing reference %q: %w", destinationReference, err) } - if opts.IsImageIndex { - ii, _, err := api.imageIndex(sourceReference) + desc, _, err := api.getImageDesc(sourceReference) + if err != nil { + return fmt.Errorf("unable to get image %s: %w", sourceReference, err) + } + + if desc.MediaType.IsIndex() { + ii, err := desc.ImageIndex() if err != nil { - return fmt.Errorf("error reading image index %q: %w", sourceReference, err) + return fmt.Errorf("getting image index %s: %w", sourceReference, err) } - if err = remote.WriteIndex(ref, ii, remote.WithAuthFromKeychain(authn.DefaultKeychain)); err != nil { - return fmt.Errorf("unable to write %q: %w", ref, err) + if err = remote.WriteIndex(dstRef, ii, remote.WithAuthFromKeychain(authn.DefaultKeychain)); err != nil { + return fmt.Errorf("unable to write %s: %w", dstRef, err) } } else { - img, _, err := api.image(sourceReference) + img, err := desc.Image() if err != nil { - return fmt.Errorf("error reading image %q: %w", sourceReference, err) + return fmt.Errorf("getting image manifest %s: %w", sourceReference, err) } - if err = remote.Write(ref, img, remote.WithAuthFromKeychain(authn.DefaultKeychain)); err != nil { - return fmt.Errorf("unable to write %q: %w", ref, err) + if err = remote.Write(dstRef, img, remote.WithAuthFromKeychain(authn.DefaultKeychain)); err != nil { + return fmt.Errorf("unable to write %s: %w", dstRef, err) } } @@ -290,40 +352,21 @@ func (api *api) pushImage(ctx context.Context, reference string, opts *PushImage return nil } -func (api *api) image(reference string) (v1.Image, name.Reference, error) { +func (api *api) getImageDesc(reference string) (*remote.Descriptor, name.Reference, error) { ref, err := name.ParseReference(reference, api.parseReferenceOptions()...) if err != nil { return nil, nil, fmt.Errorf("parsing reference %q: %w", reference, err) } - img, err := remote.Image( + desc, err := remote.Get( ref, remote.WithAuthFromKeychain(authn.DefaultKeychain), remote.WithTransport(api.getHttpTransport()), ) if err != nil { - return nil, nil, fmt.Errorf("reading image %q: %w", ref, err) - } - - return img, ref, nil -} - -func (api *api) imageIndex(reference string) (v1.ImageIndex, name.Reference, error) { - ref, err := name.ParseReference(reference, api.parseReferenceOptions()...) - if err != nil { - return nil, nil, fmt.Errorf("parsing reference %q: %w", reference, err) + return nil, nil, fmt.Errorf("getting %s: %w", ref, err) } - - ii, err := remote.Index( - ref, - remote.WithAuthFromKeychain(authn.DefaultKeychain), - remote.WithTransport(api.getHttpTransport()), - ) - if err != nil { - return nil, nil, fmt.Errorf("reading image %q: %w", ref, err) - } - - return ii, ref, nil + return desc, ref, nil } func (api *api) newRepositoryOptions() []name.Option { diff --git a/pkg/docker_registry/generic_api.go b/pkg/docker_registry/generic_api.go index 50190e85e8..2039302fdf 100644 --- a/pkg/docker_registry/generic_api.go +++ b/pkg/docker_registry/generic_api.go @@ -52,12 +52,17 @@ func (api *genericApi) GetRepoImageConfigFile(ctx context.Context, reference str } func (api *genericApi) getRepoImageConfigFile(_ context.Context, reference string) (*v1.ConfigFile, error) { - imageInfo, _, err := api.commonApi.image(reference) + desc, _, err := api.commonApi.getImageDesc(reference) if err != nil { return nil, err } - return imageInfo.ConfigFile() + img, err := desc.Image() + if err != nil { + return nil, err + } + + return img.ConfigFile() } func (api *genericApi) GetRepoImage(ctx context.Context, reference string) (*image.Info, error) { diff --git a/pkg/docker_registry/interface.go b/pkg/docker_registry/interface.go index aab77ac168..bc11184fa7 100644 --- a/pkg/docker_registry/interface.go +++ b/pkg/docker_registry/interface.go @@ -44,6 +44,8 @@ type ManifestListOptions struct { Manifests []*image.Info } -type CopyImageOptions struct { +type GetRepoImageOptions struct { IsImageIndex bool } + +type CopyImageOptions struct{} diff --git a/pkg/image/info.go b/pkg/image/info.go index 762f82eb97..06e817d0c1 100644 --- a/pkg/image/info.go +++ b/pkg/image/info.go @@ -24,6 +24,9 @@ type Info struct { Labels map[string]string `json:"labels"` Size int64 `json:"size"` CreatedAtUnixNano int64 `json:"createdAtUnixNano"` + + IsIndex bool + Index []*Info } func (info *Info) SetCreatedAtUnix(seconds int64) { @@ -39,7 +42,7 @@ func (info *Info) GetCreatedAt() time.Time { } func (info *Info) GetCopy() *Info { - return &Info{ + res := &Info{ Name: info.Name, Repository: info.Repository, Tag: info.Tag, @@ -51,7 +54,15 @@ func (info *Info) GetCopy() *Info { Labels: util.CopyMap(info.Labels), Size: info.Size, CreatedAtUnixNano: info.CreatedAtUnixNano, + + IsIndex: info.IsIndex, } + + for _, i := range info.Index { + res.Index = append(res.Index, i.GetCopy()) + } + + return res } func (info *Info) LogName() string { diff --git a/pkg/image/stage.go b/pkg/image/stage.go index a472c58ff3..4fb2c10c72 100644 --- a/pkg/image/stage.go +++ b/pkg/image/stage.go @@ -7,8 +7,17 @@ import ( ) type StageID struct { - Digest string `json:"digest"` - UniqueID int64 `json:"uniqueID"` + Digest string `json:"digest"` + UniqueID int64 `json:"uniqueID"` + IsMultiplatform bool `json:"isMultiplatform"` +} + +func NewStageID(digest string, uniqueID int64) *StageID { + return &StageID{ + Digest: digest, + UniqueID: uniqueID, + IsMultiplatform: (uniqueID == 0), + } } func (id StageID) String() string { @@ -41,7 +50,7 @@ func ParseUniqueIDAsTimestamp(uniqueID string) (int64, error) { func (desc *StageDescription) GetCopy() *StageDescription { return &StageDescription{ - StageID: &StageID{desc.StageID.Digest, desc.StageID.UniqueID}, + StageID: NewStageID(desc.StageID.Digest, desc.StageID.UniqueID), Info: desc.Info.GetCopy(), } } diff --git a/pkg/image/summary.go b/pkg/image/summary.go index 2773f7b67a..7d95568c0f 100644 --- a/pkg/image/summary.go +++ b/pkg/image/summary.go @@ -31,7 +31,7 @@ func (list ImagesList) ConvertToStages() ([]StageID, error) { if digest, uniqueID, err := getDigestAndUniqueIDFromLocalStageImageTag(tag); err != nil { return nil, err } else { - stagesList = append(stagesList, StageID{Digest: digest, UniqueID: uniqueID}) + stagesList = append(stagesList, *NewStageID(digest, uniqueID)) } } } diff --git a/pkg/storage/local_stages_storage.go b/pkg/storage/local_stages_storage.go index d1664e12af..4c380dc0ac 100644 --- a/pkg/storage/local_stages_storage.go +++ b/pkg/storage/local_stages_storage.go @@ -130,8 +130,8 @@ func (storage *LocalStagesStorage) GetStagesIDsByDigest(ctx context.Context, pro return images.ConvertToStages() } -func (storage *LocalStagesStorage) GetStageDescription(ctx context.Context, projectName, digest string, uniqueID int64) (*image.StageDescription, error) { - stageImageName := storage.ConstructStageImageName(projectName, digest, uniqueID) +func (storage *LocalStagesStorage) GetStageDescription(ctx context.Context, projectName string, stageID image.StageID) (*image.StageDescription, error) { + stageImageName := storage.ConstructStageImageName(projectName, stageID.Digest, stageID.UniqueID) info, err := storage.ContainerBackend.GetImageInfo(ctx, stageImageName, container_backend.GetImageInfoOpts{}) if err != nil { return nil, fmt.Errorf("unable to get image %s info: %w", stageImageName, err) @@ -139,7 +139,7 @@ func (storage *LocalStagesStorage) GetStageDescription(ctx context.Context, proj if info != nil { return &image.StageDescription{ - StageID: &image.StageID{Digest: digest, UniqueID: uniqueID}, + StageID: image.NewStageID(stageID.Digest, stageID.UniqueID), Info: info, }, nil } diff --git a/pkg/storage/manager/storage_manager.go b/pkg/storage/manager/storage_manager.go index 68e3335ada..c28fcbbb49 100644 --- a/pkg/storage/manager/storage_manager.go +++ b/pkg/storage/manager/storage_manager.go @@ -402,7 +402,7 @@ func (m *StorageManager) LockStageImage(ctx context.Context, imageName string) e func doFetchStage(ctx context.Context, projectName string, stagesStorage storage.StagesStorage, stageID image.StageID, img container_backend.LegacyImageInterface) error { err := logboek.Context(ctx).Info().LogProcess("Check manifest availability").DoError(func() error { - freshStageDescription, err := stagesStorage.GetStageDescription(ctx, projectName, stageID.Digest, stageID.UniqueID) + freshStageDescription, err := stagesStorage.GetStageDescription(ctx, projectName, stageID) if err != nil { return fmt.Errorf("unable to get stage description: %w", err) } @@ -662,7 +662,7 @@ func (m *StorageManager) CopyStageIntoFinalStorage(ctx context.Context, stageID for _, existingStg := range existingStagesListCache.GetStageIDs() { if existingStg.IsEqual(stageID) { - desc, err := m.GetFinalStagesStorage().GetStageDescription(ctx, m.ProjectName, stageID.Digest, stageID.UniqueID) + desc, err := m.GetFinalStagesStorage().GetStageDescription(ctx, m.ProjectName, stageID) if err != nil { return nil, fmt.Errorf("unable to get stage %s descriptor from final repo %s: %w", stageID.String(), m.GetFinalStagesStorage().String(), err) } @@ -874,7 +874,7 @@ func getStageDescriptionFromLocalManifestCache(ctx context.Context, projectName logboek.Context(ctx).Info().LogF("Got image %s info from the manifest cache (CACHE HIT)\n", stageImageName) return &image.StageDescription{ - StageID: &image.StageID{Digest: stageID.Digest, UniqueID: stageID.UniqueID}, + StageID: image.NewStageID(stageID.Digest, stageID.UniqueID), Info: imgInfo, }, nil } else { @@ -886,10 +886,7 @@ func getStageDescriptionFromLocalManifestCache(ctx context.Context, projectName func ConvertStageDescriptionForStagesStorage(stageDesc *image.StageDescription, stagesStorage storage.StagesStorage) *image.StageDescription { return &image.StageDescription{ - StageID: &image.StageID{ - Digest: stageDesc.StageID.Digest, - UniqueID: stageDesc.StageID.UniqueID, - }, + StageID: image.NewStageID(stageDesc.StageID.Digest, stageDesc.StageID.UniqueID), Info: &image.Info{ Name: fmt.Sprintf("%s:%s-%d", stagesStorage.Address(), stageDesc.StageID.Digest, stageDesc.StageID.UniqueID), Repository: stagesStorage.Address(), @@ -932,7 +929,7 @@ func getStageDescription(ctx context.Context, projectName string, stageID image. err := logboek.Context(ctx).Info().LogProcess("Get stage %s description from cache stages storage %s", stageID.String(), cacheStagesStorage.String()). DoError(func() error { var err error - stageDesc, err = cacheStagesStorage.GetStageDescription(ctx, projectName, stageID.Digest, stageID.UniqueID) + stageDesc, err = cacheStagesStorage.GetStageDescription(ctx, projectName, stageID) logboek.Context(ctx).Debug().LogF("Got stage description: %#v\n", stageDesc) return err @@ -954,7 +951,7 @@ func getStageDescription(ctx context.Context, projectName string, stageID image. } logboek.Context(ctx).Debug().LogF("Getting digest %q uniqueID %d stage info from %s...\n", stageID.Digest, stageID.UniqueID, stagesStorage.String()) - stageDesc, err := stagesStorage.GetStageDescription(ctx, projectName, stageID.Digest, stageID.UniqueID) + stageDesc, err := stagesStorage.GetStageDescription(ctx, projectName, stageID) switch { case storage.IsErrBrokenImage(err): return nil, nil diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index fda76b1b30..898fa6a2d3 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -112,7 +112,7 @@ func (storage *RepoStagesStorage) GetStagesIDs(ctx context.Context, _ string, op } return nil, err } else { - res = append(res, image.StageID{Digest: digest, UniqueID: uniqueID}) + res = append(res, *image.NewStageID(digest, uniqueID)) logboek.Context(ctx).Debug().LogF("Selected stage by digest %q uniqueID %d\n", digest, uniqueID) } @@ -216,7 +216,7 @@ func (storage *RepoStagesStorage) GetStagesIDsByDigest(ctx context.Context, _, d return nil, fmt.Errorf("unable to get digest and uniqueID from rejected stage tag %q: %w", tag, err) } else { logboek.Context(ctx).Info().LogF("Found rejected stage %q\n", tag) - rejectedStages = append(rejectedStages, image.StageID{Digest: digest, UniqueID: uniqueID}) + rejectedStages = append(rejectedStages, *image.NewStageID(digest, uniqueID)) } } @@ -238,7 +238,7 @@ func (storage *RepoStagesStorage) GetStagesIDsByDigest(ctx context.Context, _, d } return nil, fmt.Errorf("unable to get digest and uniqueID from tag %q: %w", tag, err) } else { - stageID := image.StageID{Digest: digest, UniqueID: uniqueID} + stageID := image.NewStageID(digest, uniqueID) for _, rejectedStage := range rejectedStages { if rejectedStage.Digest == stageID.Digest && rejectedStage.UniqueID == stageID.UniqueID { @@ -248,7 +248,7 @@ func (storage *RepoStagesStorage) GetStagesIDsByDigest(ctx context.Context, _, d } logboek.Context(ctx).Debug().LogF("Stage %q is suitable for digest %q\n", tag, digest) - res = append(res, stageID) + res = append(res, *stageID) } } } @@ -262,10 +262,10 @@ func (storage *RepoStagesStorage) GetStagesIDsByDigest(ctx context.Context, _, d // NOTE: go-containerregistry has a special method to get an v1.ImageIndex manifest, instead of v1.Image, // NOTE: should we utilize this method in GetStageDescription also? // NOTE: Current version always tries to get v1.Image manifest and it works without errors though. -func (storage *RepoStagesStorage) GetStageDescription(ctx context.Context, projectName, digest string, uniqueID int64) (*image.StageDescription, error) { - stageImageName := storage.ConstructStageImageName(projectName, digest, uniqueID) +func (storage *RepoStagesStorage) GetStageDescription(ctx context.Context, projectName string, stageID image.StageID) (*image.StageDescription, error) { + stageImageName := storage.ConstructStageImageName(projectName, stageID.Digest, stageID.UniqueID) - logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage GetStageDescription %s %s %d\n", projectName, digest, uniqueID) + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage GetStageDescription %s %s %d\n", projectName, stageID.Digest, stageID.UniqueID) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage stageImageName = %q\n", stageImageName) imgInfo, err := storage.DockerRegistry.GetRepoImage(ctx, stageImageName) @@ -282,18 +282,18 @@ func (storage *RepoStagesStorage) GetStageDescription(ctx context.Context, proje return nil, fmt.Errorf("unable to inspect repo image %s: %w", stageImageName, err) } - rejectedImageName := makeRepoRejectedStageImageRecord(storage.RepoAddress, digest, uniqueID) + rejectedImageName := makeRepoRejectedStageImageRecord(storage.RepoAddress, stageID.Digest, stageID.UniqueID) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetStageDescription check rejected image name: %s\n", rejectedImageName) if rejectedImgInfo, err := storage.DockerRegistry.TryGetRepoImage(ctx, rejectedImageName); err != nil { return nil, fmt.Errorf("unable to get repo image %q: %w", rejectedImageName, err) } else if rejectedImgInfo != nil { - logboek.Context(ctx).Info().LogF("Stage digest %s uniqueID %d image is rejected: ignore stage image\n", digest, uniqueID) + logboek.Context(ctx).Info().LogF("Stage digest %s uniqueID %d image is rejected: ignore stage image\n", stageID.Digest, stageID.UniqueID) return nil, nil } return &image.StageDescription{ - StageID: &image.StageID{Digest: digest, UniqueID: uniqueID}, + StageID: image.NewStageID(stageID.Digest, stageID.UniqueID), Info: imgInfo, }, nil } @@ -904,7 +904,7 @@ func (storage *RepoStagesStorage) PostMultiplatformImage(ctx context.Context, pr } func (storage *RepoStagesStorage) CopyFromStorage(ctx context.Context, src StagesStorage, projectName string, stageID image.StageID, opts CopyFromStorageOptions) (*image.StageDescription, error) { - desc, err := storage.GetStageDescription(ctx, projectName, stageID.Digest, stageID.UniqueID) + desc, err := storage.GetStageDescription(ctx, projectName, stageID) if err != nil { return nil, fmt.Errorf("unable to get stage %s description: %w", stageID, err) } @@ -914,13 +914,11 @@ func (storage *RepoStagesStorage) CopyFromStorage(ctx context.Context, src Stage srcRef := src.ConstructStageImageName(projectName, stageID.Digest, stageID.UniqueID) dstRef := storage.ConstructStageImageName(projectName, stageID.Digest, stageID.UniqueID) - if err := storage.DockerRegistry.CopyImage(ctx, srcRef, dstRef, docker_registry.CopyImageOptions{ - IsImageIndex: opts.IsMultiplatformImage, - }); err != nil { + if err := storage.DockerRegistry.CopyImage(ctx, srcRef, dstRef, docker_registry.CopyImageOptions{}); err != nil { return nil, fmt.Errorf("unable to copy image into registry: %w", err) } - desc, err = storage.GetStageDescription(ctx, projectName, stageID.Digest, stageID.UniqueID) + desc, err = storage.GetStageDescription(ctx, projectName, stageID) if err != nil { return nil, fmt.Errorf("unable to get stage %s description: %w", stageID, err) } diff --git a/pkg/storage/stages_storage.go b/pkg/storage/stages_storage.go index 81584c603a..7e8439ddb7 100644 --- a/pkg/storage/stages_storage.go +++ b/pkg/storage/stages_storage.go @@ -34,7 +34,7 @@ type FilterStagesAndProcessRelatedDataOptions struct { type StagesStorage interface { GetStagesIDs(ctx context.Context, projectName string, opts ...Option) ([]image.StageID, error) GetStagesIDsByDigest(ctx context.Context, projectName, digest string, opts ...Option) ([]image.StageID, error) - GetStageDescription(ctx context.Context, projectName, digest string, uniqueID int64) (*image.StageDescription, error) + GetStageDescription(ctx context.Context, projectName string, stageID image.StageID) (*image.StageDescription, error) ExportStage(ctx context.Context, stageDescription *image.StageDescription, destinationReference string, mutateConfigFunc func(config v1.Config) (v1.Config, error)) error DeleteStage(ctx context.Context, stageDescription *image.StageDescription, options DeleteImageOptions) error diff --git a/pkg/storage/synchronization_server/stages_storage_cache_http_handler_legacy.go b/pkg/storage/synchronization_server/stages_storage_cache_http_handler_legacy.go index ef68d9a794..3f026a3485 100644 --- a/pkg/storage/synchronization_server/stages_storage_cache_http_handler_legacy.go +++ b/pkg/storage/synchronization_server/stages_storage_cache_http_handler_legacy.go @@ -127,7 +127,7 @@ func (handler *StagesStorageCacheHttpHandlerLegacy) handleStoreStagesBySignature logboek.Debug().LogF("StagesStorageCacheHttpHandlerLegacy -- StoreStagesBySignature request %#v\n", request) var stages []image.StageID for _, s := range request.Stages { - stages = append(stages, image.StageID{Digest: s.Signature, UniqueID: s.UniqueID}) + stages = append(stages, *image.NewStageID(s.Signature, s.UniqueID)) } response.Err.Error = handler.StagesStorageCache.StoreStagesByDigest(context.Background(), request.ProjectName, request.Signature, stages) logboek.Debug().LogF("StagesStorageCacheHttpHandlerLegacy -- StoreStagesBySignature response %#v\n", response)