From 7ea0a4cd92427567b34734e036c66821b37d2a14 Mon Sep 17 00:00:00 2001 From: Alexey Igrychev Date: Mon, 14 Mar 2022 18:50:34 +0000 Subject: [PATCH] feat(build): speeding up with runtime caching for meta images - docker registry - ability to reuse cached tags from previous requests - updating cached tags when a tag is added/removed - stages storage: ability to work with cached meta image data - build/cleanup: using caching when working with meta images Signed-off-by: Alexey Igrychev --- pkg/build/build_phase.go | 19 ++- pkg/cleaning/cleanup.go | 2 +- pkg/cleaning/purge.go | 6 +- pkg/cleaning/stage_manager/manager.go | 4 +- pkg/docker_registry/api.go | 6 +- pkg/docker_registry/api_test.go | 2 +- pkg/docker_registry/default.go | 10 +- pkg/docker_registry/docker_registry.go | 10 ++ .../docker_registry_with_cache.go | 154 ++++++++++++++++++ pkg/docker_registry/generic_api.go | 2 +- pkg/docker_registry/harbor.go | 10 +- pkg/docker_registry/interface.go | 7 +- pkg/docker_registry/options.go | 26 +++ pkg/storage/docker_server_stages_storage.go | 16 +- pkg/storage/options.go | 24 +++ pkg/storage/repo_stages_storage.go | 103 ++++++------ pkg/storage/stages_storage.go | 13 +- 17 files changed, 322 insertions(+), 92 deletions(-) create mode 100644 pkg/docker_registry/docker_registry_with_cache.go create mode 100644 pkg/docker_registry/options.go create mode 100644 pkg/storage/options.go diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index df79edcb0b..0207138768 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -258,7 +258,17 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error func (phase *BuildPhase) addManagedImage(ctx context.Context, img *Image) error { if phase.ShouldAddManagedImageRecord { - if err := phase.Conveyor.StorageManager.GetStagesStorage().AddManagedImage(ctx, phase.Conveyor.projectName(), img.GetName()); err != nil { + stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage() + exist, err := stagesStorage.IsManagedImageExist(ctx, phase.Conveyor.projectName(), img.GetName(), storage.WithCache()) + if err != nil { + return fmt.Errorf("unable to check existence of managed image: %w", err) + } + + if exist { + return nil + } + + if err := stagesStorage.AddManagedImage(ctx, phase.Conveyor.projectName(), img.GetName()); err != nil { return fmt.Errorf("unable to add image %q to the managed images of project %q: %s", img.GetName(), phase.Conveyor.projectName(), err) } } @@ -286,13 +296,14 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, img *Image) e } for _, commit := range commits { - exists, err := phase.Conveyor.StorageManager.GetStagesStorage().IsImageMetadataExist(ctx, phase.Conveyor.projectName(), img.GetName(), commit, img.GetStageID()) + stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage() + exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.projectName(), img.GetName(), commit, img.GetStageID(), storage.WithCache()) if err != nil { return fmt.Errorf("unable to get image %s metadata by commit %s and stage ID %s: %s", img.GetName(), commit, img.GetStageID(), err) } - if !exists { - if err := phase.Conveyor.StorageManager.GetStagesStorage().PutImageMetadata(ctx, phase.Conveyor.projectName(), img.GetName(), commit, img.GetStageID()); err != nil { + if !exist { + if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.projectName(), img.GetName(), commit, img.GetStageID()); err != nil { return fmt.Errorf("unable to put image %s metadata by commit %s and stage ID %s: %s", img.GetName(), commit, img.GetStageID(), err) } } diff --git a/pkg/cleaning/cleanup.go b/pkg/cleaning/cleanup.go index 17e98beffb..60c4398dff 100644 --- a/pkg/cleaning/cleanup.go +++ b/pkg/cleaning/cleanup.go @@ -701,7 +701,7 @@ FilterOutFinalStages: } func (m *cleanupManager) initImportsMetadata(ctx context.Context, stageDescriptionList []*image.StageDescription) error { - importMetadataIDs, err := m.StorageManager.GetStagesStorage().GetImportMetadataIDs(ctx, m.ProjectName) + importMetadataIDs, err := m.StorageManager.GetStagesStorage().GetImportMetadataIDs(ctx, m.ProjectName, storage.WithCache()) if err != nil { return err } diff --git a/pkg/cleaning/purge.go b/pkg/cleaning/purge.go index 5e54eabb25..1f8d672f78 100644 --- a/pkg/cleaning/purge.go +++ b/pkg/cleaning/purge.go @@ -49,7 +49,7 @@ func (m *purgeManager) run(ctx context.Context) error { } if err := logboek.Context(ctx).Default().LogProcess("Deleting imports metadata").DoError(func() error { - importMetadataIDs, err := m.StorageManager.GetStagesStorage().GetImportMetadataIDs(ctx, m.ProjectName) + importMetadataIDs, err := m.StorageManager.GetStagesStorage().GetImportMetadataIDs(ctx, m.ProjectName, storage.WithCache()) if err != nil { return err } @@ -60,7 +60,7 @@ func (m *purgeManager) run(ctx context.Context) error { } if err := logboek.Context(ctx).Default().LogProcess("Deleting managed images").DoError(func() error { - managedImages, err := m.StorageManager.GetStagesStorage().GetManagedImages(ctx, m.ProjectName) + managedImages, err := m.StorageManager.GetStagesStorage().GetManagedImages(ctx, m.ProjectName, storage.WithCache()) if err != nil { return err } @@ -75,7 +75,7 @@ func (m *purgeManager) run(ctx context.Context) error { } if err := logboek.Context(ctx).Default().LogProcess("Deleting images metadata").DoError(func() error { - _, imageMetadataByImageName, err := m.StorageManager.GetStagesStorage().GetAllAndGroupImageMetadataByImageName(ctx, m.ProjectName, []string{}) + _, imageMetadataByImageName, err := m.StorageManager.GetStagesStorage().GetAllAndGroupImageMetadataByImageName(ctx, m.ProjectName, []string{}, storage.WithCache()) if err != nil { return err } diff --git a/pkg/cleaning/stage_manager/manager.go b/pkg/cleaning/stage_manager/manager.go index 839b264139..e96d058e79 100644 --- a/pkg/cleaning/stage_manager/manager.go +++ b/pkg/cleaning/stage_manager/manager.go @@ -95,7 +95,7 @@ type GitRepo interface { } func (m *Manager) InitImagesMetadata(ctx context.Context, storageManager manager.StorageManagerInterface, localGit GitRepo, projectName string, imageNameList []string) error { - imageMetadataByImageName, imageMetadataByNotManagedImageName, err := storageManager.GetStagesStorage().GetAllAndGroupImageMetadataByImageName(ctx, projectName, imageNameList) + imageMetadataByImageName, imageMetadataByNotManagedImageName, err := storageManager.GetStagesStorage().GetAllAndGroupImageMetadataByImageName(ctx, projectName, imageNameList, storage.WithCache()) if err != nil { return err } @@ -140,7 +140,7 @@ func (m *Manager) InitCustomTagsMetadata(ctx context.Context, storageManager man } func GetCustomTagsMetadata(ctx context.Context, storageManager manager.StorageManagerInterface) (stageIDCustomTagList map[string][]string, err error) { - stageCustomTagMetadataIDs, err := storageManager.GetStagesStorage().GetStageCustomTagMetadataIDs(ctx) + stageCustomTagMetadataIDs, err := storageManager.GetStagesStorage().GetStageCustomTagMetadataIDs(ctx, storage.WithCache()) if err != nil { return nil, fmt.Errorf("unable to get stage custom tag metadata IDs: %s", err) } diff --git a/pkg/docker_registry/api.go b/pkg/docker_registry/api.go index 84e99a2692..bc5ae553df 100644 --- a/pkg/docker_registry/api.go +++ b/pkg/docker_registry/api.go @@ -39,7 +39,7 @@ func newAPI(options apiOptions) *api { } } -func (api *api) Tags(ctx context.Context, reference string) ([]string, error) { +func (api *api) Tags(ctx context.Context, reference string, _ ...Option) ([]string, error) { return api.tags(ctx, reference) } @@ -134,7 +134,7 @@ func (api *api) GetRepoImage(_ context.Context, reference string) (*image.Info, } } - referenceParts, err := api.ParseReferenceParts(reference) + referenceParts, err := api.parseReferenceParts(reference) if err != nil { return nil, fmt.Errorf("unable to parse reference %q: %s", reference, err) } @@ -349,7 +349,7 @@ type referenceParts struct { digest string } -func (api *api) ParseReferenceParts(reference string) (referenceParts, error) { +func (api *api) parseReferenceParts(reference string) (referenceParts, error) { // validate reference parsedReference, err := name.ParseReference(reference, api.parseReferenceOptions()...) if err != nil { diff --git a/pkg/docker_registry/api_test.go b/pkg/docker_registry/api_test.go index 8659de495c..778e14a83a 100644 --- a/pkg/docker_registry/api_test.go +++ b/pkg/docker_registry/api_test.go @@ -12,7 +12,7 @@ type ParseReferencePartsEntry struct { } var _ = DescribeTable("Api_ParseReferenceParts", func(entry ParseReferencePartsEntry) { - parts, err := (&api{}).ParseReferenceParts(entry.reference) + parts, err := (&api{}).parseReferenceParts(entry.reference) Ω(err).ShouldNot(HaveOccurred()) Ω(parts).Should(Equal(entry.expectation)) }, diff --git a/pkg/docker_registry/default.go b/pkg/docker_registry/default.go index 98e0c88363..8c555546fd 100644 --- a/pkg/docker_registry/default.go +++ b/pkg/docker_registry/default.go @@ -31,7 +31,7 @@ func newDefaultAPIForImplementation(implementation string, options defaultImplem return d, nil } -func (r *defaultImplementation) Tags(ctx context.Context, reference string) ([]string, error) { +func (r *defaultImplementation) Tags(ctx context.Context, reference string, _ ...Option) ([]string, error) { tags, err := r.api.Tags(ctx, reference) if (IsHarbor404Error(err) || IsHarborNotFoundError(err)) && r.Implementation != HarborImplementationName { @@ -49,12 +49,8 @@ func (r *defaultImplementation) Tags(ctx context.Context, reference string) ([]s return tags, err } -func (r *defaultImplementation) IsRepoImageExists(ctx context.Context, reference string) (bool, error) { - if imgInfo, err := r.TryGetRepoImage(ctx, reference); err != nil { - return false, err - } else { - return imgInfo != nil, nil - } +func (r *defaultImplementation) IsTagExist(_ context.Context, _ string, _ ...Option) (bool, error) { + panic("not implemented") } func (r *defaultImplementation) TryGetRepoImage(ctx context.Context, reference string) (*image.Info, error) { diff --git a/pkg/docker_registry/docker_registry.go b/pkg/docker_registry/docker_registry.go index d6a03104c4..04af25631e 100644 --- a/pkg/docker_registry/docker_registry.go +++ b/pkg/docker_registry/docker_registry.go @@ -92,6 +92,16 @@ func (o *DockerRegistryOptions) defaultOptions() defaultImplementationOptions { } func NewDockerRegistry(repositoryAddress string, implementation string, options DockerRegistryOptions) (Interface, error) { + dockerRegistry, err := newDockerRegistry(repositoryAddress, implementation, options) + if err != nil { + return nil, err + } + + dockerRegistryWithCache := newDockerRegistryWithCache(dockerRegistry) + return dockerRegistryWithCache, nil +} + +func newDockerRegistry(repositoryAddress string, implementation string, options DockerRegistryOptions) (Interface, error) { switch implementation { case AwsEcrImplementationName: return newAwsEcr(options.awsEcrOptions()) diff --git a/pkg/docker_registry/docker_registry_with_cache.go b/pkg/docker_registry/docker_registry_with_cache.go new file mode 100644 index 0000000000..381d65cc5d --- /dev/null +++ b/pkg/docker_registry/docker_registry_with_cache.go @@ -0,0 +1,154 @@ +package docker_registry + +import ( + "context" + "fmt" + "strings" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/werf/werf/pkg/image" + "github.com/werf/werf/pkg/util" +) + +type DockerRegistryWithCache struct { + Interface + cachedTagsMap *sync.Map + cachedTagsMutexMap *sync.Map +} + +func newDockerRegistryWithCache(dockerRegistry Interface) *DockerRegistryWithCache { + return &DockerRegistryWithCache{ + Interface: dockerRegistry, + cachedTagsMap: &sync.Map{}, + cachedTagsMutexMap: &sync.Map{}, + } +} + +func (r *DockerRegistryWithCache) Tags(ctx context.Context, reference string, opts ...Option) ([]string, error) { + o := makeOptions(opts...) + return r.withCachedTags(reference, func(cachedTags []string, isExist bool) ([]string, error) { + if isExist && o.cachedTags { + return cachedTags, nil + } + + return r.Interface.Tags(ctx, reference, opts...) + }) +} + +func (r *DockerRegistryWithCache) IsTagExist(ctx context.Context, reference string, opts ...Option) (bool, error) { + referenceParts, err := r.parseReferenceParts(reference) + if err != nil { + return false, err + } + + referenceTag := referenceParts.tag + if referenceTag == "" { + panic(fmt.Sprintf("unexpected reference %q: tag required", reference)) + } + + repositoryAddress := strings.Join([]string{referenceParts.registry, referenceParts.repository}, "/") + tags, err := r.Tags(ctx, repositoryAddress, opts...) + if err != nil { + return false, err + } + + for _, tag := range tags { + if referenceTag == tag { + return true, nil + } + } + + return false, nil +} + +func (r *DockerRegistryWithCache) TagRepoImage(ctx context.Context, repoImage *image.Info, tag string) error { + defer r.mustAddTagToCachedTags(repoImage.Name) + return r.Interface.TagRepoImage(ctx, repoImage, tag) +} + +func (r *DockerRegistryWithCache) PushImage(ctx context.Context, reference string, opts *PushImageOptions) error { + defer r.mustAddTagToCachedTags(reference) + return r.Interface.PushImage(ctx, reference, opts) +} + +func (r *DockerRegistryWithCache) MutateAndPushImage(ctx context.Context, sourceReference, destinationReference string, mutateConfigFunc func(v1.Config) (v1.Config, error)) error { + defer r.mustAddTagToCachedTags(destinationReference) + return r.Interface.MutateAndPushImage(ctx, sourceReference, destinationReference, mutateConfigFunc) +} + +func (r *DockerRegistryWithCache) DeleteRepoImage(ctx context.Context, repoImage *image.Info) error { + defer r.mustDeleteTagFromCachedTags(repoImage.Name) + return r.Interface.DeleteRepoImage(ctx, repoImage) +} + +func (r *DockerRegistryWithCache) mustAddTagToCachedTags(reference string) { + _, err := r.withCachedTags(reference, func(tags []string, isExist bool) ([]string, error) { + referenceParts, err := r.parseReferenceParts(reference) + if err != nil { + return nil, fmt.Errorf("unable to parse reference parts %q: %w", reference, err) + } + + if !isExist { + return nil, nil + } + + tags = append(tags, referenceParts.tag) + return tags, nil + }) + if err != nil { + panic(fmt.Sprintf("unexpected err: %s", err)) + } +} + +func (r *DockerRegistryWithCache) mustDeleteTagFromCachedTags(reference string) { + _, err := r.withCachedTags(reference, func(tags []string, isExist bool) ([]string, error) { + referenceParts, err := r.parseReferenceParts(reference) + if err != nil { + return nil, fmt.Errorf("unable to parse reference parts %q: %w", reference, err) + } + + if !isExist { + return nil, nil + } + + tags = util.ExcludeFromStringArray(tags, referenceParts.tag) + return tags, nil + }) + if err != nil { + panic(fmt.Sprintf("unexpected err: %s", err)) + } +} + +func (r *DockerRegistryWithCache) withCachedTags(reference string, f func([]string, bool) ([]string, error)) ([]string, error) { + cachedTagsID := r.mustGetCachedTagsID(reference) + + mutex := util.MapLoadOrCreateMutex(r.cachedTagsMutexMap, cachedTagsID) + mutex.Lock() + defer mutex.Unlock() + + value, isExist := r.cachedTagsMap.Load(cachedTagsID) + var tags []string + if isExist { + tags = value.([]string) + } + + newTags, err := f(tags, isExist) + if err != nil { + return nil, err + } + + r.cachedTagsMap.Store(cachedTagsID, newTags) + return newTags, nil +} + +func (r *DockerRegistryWithCache) mustGetCachedTagsID(reference string) string { + referenceParts, err := r.parseReferenceParts(reference) + if err != nil { + panic(fmt.Sprintf("unexpected reference %q: %s", reference, err)) + } + + repositoryAddress := strings.Join([]string{referenceParts.registry, referenceParts.repository}, "/") + return repositoryAddress +} diff --git a/pkg/docker_registry/generic_api.go b/pkg/docker_registry/generic_api.go index 5f8e660e7b..84563924ce 100644 --- a/pkg/docker_registry/generic_api.go +++ b/pkg/docker_registry/generic_api.go @@ -94,7 +94,7 @@ func (api *genericApi) GetRepoImage(ctx context.Context, reference string) (*ima func (api *genericApi) mirrorReferenceList(reference string) ([]string, error) { var referenceList []string - referenceParts, err := api.commonApi.ParseReferenceParts(reference) + referenceParts, err := api.commonApi.parseReferenceParts(reference) if err != nil { return nil, err } diff --git a/pkg/docker_registry/harbor.go b/pkg/docker_registry/harbor.go index 07cc1084b2..fe564dc1f0 100644 --- a/pkg/docker_registry/harbor.go +++ b/pkg/docker_registry/harbor.go @@ -61,7 +61,7 @@ func newHarbor(options harborOptions) (*harbor, error) { return harbor, nil } -func (r *harbor) Tags(ctx context.Context, reference string) ([]string, error) { +func (r *harbor) Tags(ctx context.Context, reference string, _ ...Option) ([]string, error) { tags, err := r.defaultImplementation.Tags(ctx, reference) if err != nil { if IsHarborNotFoundError(err) { @@ -73,14 +73,6 @@ func (r *harbor) Tags(ctx context.Context, reference string) ([]string, error) { return tags, nil } -func (r *harbor) IsRepoImageExists(ctx context.Context, reference string) (bool, error) { - if imgInfo, err := r.TryGetRepoImage(ctx, reference); err != nil { - return false, err - } else { - return imgInfo != nil, nil - } -} - func (r *harbor) TryGetRepoImage(ctx context.Context, reference string) (*image.Info, error) { res, err := r.api.TryGetRepoImage(ctx, reference) if err != nil { diff --git a/pkg/docker_registry/interface.go b/pkg/docker_registry/interface.go index 515ca71339..937901cc3a 100644 --- a/pkg/docker_registry/interface.go +++ b/pkg/docker_registry/interface.go @@ -11,16 +11,17 @@ import ( type Interface interface { CreateRepo(ctx context.Context, reference string) error DeleteRepo(ctx context.Context, reference string) error - Tags(ctx context.Context, reference string) ([]string, error) + Tags(ctx context.Context, reference string, opts ...Option) ([]string, error) + IsTagExist(ctx context.Context, reference string, opts ...Option) (bool, error) TagRepoImage(ctx context.Context, repoImage *image.Info, tag string) error GetRepoImage(ctx context.Context, reference string) (*image.Info, error) TryGetRepoImage(ctx context.Context, reference string) (*image.Info, error) - IsRepoImageExists(ctx context.Context, reference string) (bool, error) DeleteRepoImage(ctx context.Context, repoImage *image.Info) error PushImage(ctx context.Context, reference string, opts *PushImageOptions) error MutateAndPushImage(ctx context.Context, sourceReference, destinationReference string, mutateConfigFunc func(v1.Config) (v1.Config, error)) error - String() string + + parseReferenceParts(reference string) (referenceParts, error) } type ApiInterface interface { diff --git a/pkg/docker_registry/options.go b/pkg/docker_registry/options.go new file mode 100644 index 0000000000..57e4100007 --- /dev/null +++ b/pkg/docker_registry/options.go @@ -0,0 +1,26 @@ +package docker_registry + +const OptionCachedTagsDefault = false + +type Options struct { + cachedTags bool +} + +func makeOptions(opts ...Option) Options { + opt := Options{ + cachedTags: OptionCachedTagsDefault, + } + for _, o := range opts { + o(&opt) + } + + return opt +} + +type Option func(*Options) + +func WithCachedTags() Option { + return func(o *Options) { + o.cachedTags = true + } +} diff --git a/pkg/storage/docker_server_stages_storage.go b/pkg/storage/docker_server_stages_storage.go index 40233fddfd..33f0fa9295 100644 --- a/pkg/storage/docker_server_stages_storage.go +++ b/pkg/storage/docker_server_stages_storage.go @@ -143,7 +143,7 @@ func (storage *DockerServerStagesStorage) GetStageCustomTagMetadata(_ context.Co return nil, fmt.Errorf("not implemented") } -func (storage *DockerServerStagesStorage) GetStageCustomTagMetadataIDs(_ context.Context) ([]string, error) { +func (storage *DockerServerStagesStorage) GetStageCustomTagMetadataIDs(_ context.Context, _ ...Option) ([]string, error) { return nil, nil } @@ -155,7 +155,11 @@ func (storage *DockerServerStagesStorage) RmManagedImage(_ context.Context, _, _ return nil } -func (storage *DockerServerStagesStorage) GetManagedImages(_ context.Context, _ string) ([]string, error) { +func (storage *DockerServerStagesStorage) IsManagedImageExist(_ context.Context, _, _ string, _ ...Option) (bool, error) { + return false, nil +} + +func (storage *DockerServerStagesStorage) GetManagedImages(_ context.Context, _ string, _ ...Option) ([]string, error) { return []string{}, nil } @@ -197,11 +201,11 @@ func (storage *DockerServerStagesStorage) selectFullImageMetadataName(_ context. return "", nil } -func (storage *DockerServerStagesStorage) IsImageMetadataExist(_ context.Context, _, _, _, _ string) (bool, error) { +func (storage *DockerServerStagesStorage) IsImageMetadataExist(_ context.Context, _, _, _, _ string, _ ...Option) (bool, error) { return false, nil } -func (storage *DockerServerStagesStorage) GetAllAndGroupImageMetadataByImageName(_ context.Context, _ string, _ []string) (map[string]map[string][]string, map[string]map[string][]string, error) { +func (storage *DockerServerStagesStorage) GetAllAndGroupImageMetadataByImageName(_ context.Context, _ string, _ []string, _ ...Option) (map[string]map[string][]string, map[string]map[string][]string, error) { return map[string]map[string][]string{}, map[string]map[string][]string{}, nil } @@ -264,7 +268,7 @@ func (storage *DockerServerStagesStorage) RmImportMetadata(ctx context.Context, return nil } -func (storage *DockerServerStagesStorage) GetImportMetadataIDs(ctx context.Context, projectName string) ([]string, error) { +func (storage *DockerServerStagesStorage) GetImportMetadataIDs(ctx context.Context, projectName string, _ ...Option) ([]string, error) { logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.GetImportMetadataIDs %s\n", projectName) filterSet := filters.NewArgs() @@ -303,7 +307,7 @@ func (storage *DockerServerStagesStorage) Address() string { return LocalStorageAddress } -func (storage *DockerServerStagesStorage) GetClientIDRecords(ctx context.Context, projectName string) ([]*ClientIDRecord, error) { +func (storage *DockerServerStagesStorage) GetClientIDRecords(ctx context.Context, projectName string, _ ...Option) ([]*ClientIDRecord, error) { logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.GetClientID for project %s\n", projectName) filterSet := filters.NewArgs() diff --git a/pkg/storage/options.go b/pkg/storage/options.go new file mode 100644 index 0000000000..9d061bfe13 --- /dev/null +++ b/pkg/storage/options.go @@ -0,0 +1,24 @@ +package storage + +import "github.com/werf/werf/pkg/docker_registry" + +type Options struct { + dockerRegistryOptions []docker_registry.Option +} + +func makeOptions(opts ...Option) Options { + opt := Options{} + for _, o := range opts { + o(&opt) + } + + return opt +} + +type Option func(*Options) + +func WithCache() Option { + return func(o *Options) { + o.dockerRegistryOptions = append(o.dockerRegistryOptions, docker_registry.WithCachedTags()) + } +} diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index 0c3bf43229..c8144f02dc 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -167,7 +167,7 @@ func (storage *RepoStagesStorage) RejectStage(ctx context.Context, projectName, rejectedImageName := makeRepoRejectedStageImageRecord(storage.RepoAddress, digest, uniqueID) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RejectStage full image name: %s\n", rejectedImageName) - if isExists, err := storage.DockerRegistry.IsRepoImageExists(ctx, rejectedImageName); err != nil { + if isExists, err := storage.DockerRegistry.IsTagExist(ctx, rejectedImageName); err != nil { return err } else if isExists { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RejectStage record %q is exists => exiting\n", rejectedImageName) @@ -389,8 +389,9 @@ func (storage *RepoStagesStorage) GetStageCustomTagMetadata(ctx context.Context, return newCustomTagMetadataFromLabels(img.Labels), nil } -func (storage *RepoStagesStorage) GetStageCustomTagMetadataIDs(ctx context.Context) ([]string, error) { - tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress) +func (storage *RepoStagesStorage) GetStageCustomTagMetadataIDs(ctx context.Context, opts ...Option) ([]string, error) { + o := makeOptions(opts...) + tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress, o.dockerRegistryOptions...) if err != nil { return nil, fmt.Errorf("unable to get repo %s tags: %s", storage.RepoAddress, err) } @@ -414,7 +415,7 @@ func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectNa fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageNameOrManagedImageName) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage full image name: %s\n", fullImageName) - if isExists, err := storage.DockerRegistry.IsRepoImageExists(ctx, fullImageName); err != nil { + if isExists, err := storage.DockerRegistry.IsTagExist(ctx, fullImageName); err != nil { return err } else if isExists { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage record %q is exists => exiting\n", fullImageName) @@ -454,22 +455,29 @@ func (storage *RepoStagesStorage) RmManagedImage(ctx context.Context, projectNam return nil } -func (storage *RepoStagesStorage) GetManagedImages(ctx context.Context, projectName string) ([]string, error) { - logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetManagedImages %s\n", projectName) +func (storage *RepoStagesStorage) IsManagedImageExist(ctx context.Context, _, imageNameOrManagedImageName string, opts ...Option) (bool, error) { + fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageNameOrManagedImageName) + o := makeOptions(opts...) + return storage.DockerRegistry.IsTagExist(ctx, fullImageName, o.dockerRegistryOptions...) +} - var res []string +func (storage *RepoStagesStorage) GetManagedImages(ctx context.Context, projectName string, opts ...Option) ([]string, error) { + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetManagedImages %s\n", projectName) - if tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress); err != nil { + o := makeOptions(opts...) + tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress, o.dockerRegistryOptions...) + if err != nil { return nil, fmt.Errorf("unable to get repo %s tags: %s", storage.RepoAddress, err) - } else { - for _, tag := range tags { - if !strings.HasPrefix(tag, RepoManagedImageRecord_ImageTagPrefix) { - continue - } + } - managedImageName := getManagedImageNameFromManagedImageID(strings.TrimPrefix(tag, RepoManagedImageRecord_ImageTagPrefix)) - res = append(res, managedImageName) + var res []string + for _, tag := range tags { + if !strings.HasPrefix(tag, RepoManagedImageRecord_ImageTagPrefix) { + continue } + + managedImageName := getManagedImageNameFromManagedImageID(strings.TrimPrefix(tag, RepoManagedImageRecord_ImageTagPrefix)) + res = append(res, managedImageName) } return res, nil @@ -583,20 +591,21 @@ func (storage *RepoStagesStorage) selectMetadataNameImage(ctx context.Context, i } } -func (storage *RepoStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) (bool, error) { +func (storage *RepoStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string, opts ...Option) (bool, error) { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist %s %s %s %s\n", projectName, imageNameOrManagedImageName, commit, stageID) fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageNameOrManagedImageName, commit, stageID) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist full image name: %s\n", fullImageName) - img, err := storage.DockerRegistry.TryGetRepoImage(ctx, fullImageName) - return img != nil, err + o := makeOptions(opts...) + return storage.DockerRegistry.IsTagExist(ctx, fullImageName, o.dockerRegistryOptions...) } -func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string) (map[string]map[string][]string, map[string]map[string][]string, error) { +func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string, opts ...Option) (map[string]map[string][]string, map[string]map[string][]string, error) { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetImageNameStageIDCommitList %s %s\n", projectName) - tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress) + o := makeOptions(opts...) + tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress, o.dockerRegistryOptions...) if err != nil { return nil, nil, fmt.Errorf("unable to get repo %s tags: %s", storage.RepoAddress, err) } @@ -660,10 +669,11 @@ func (storage *RepoStagesStorage) RmImportMetadata(ctx context.Context, _, id st return nil } -func (storage *RepoStagesStorage) GetImportMetadataIDs(ctx context.Context, _ string) ([]string, error) { +func (storage *RepoStagesStorage) GetImportMetadataIDs(ctx context.Context, _ string, opts ...Option) ([]string, error) { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetImportMetadataIDs\n") - tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress) + o := makeOptions(opts...) + tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress, o.dockerRegistryOptions...) if err != nil { return nil, fmt.Errorf("unable to get repo %s tags: %s", storage.RepoAddress, err) } @@ -821,37 +831,38 @@ func unslugImageName(tag string) string { return res } -func (storage *RepoStagesStorage) GetClientIDRecords(ctx context.Context, projectName string) ([]*ClientIDRecord, error) { +func (storage *RepoStagesStorage) GetClientIDRecords(ctx context.Context, projectName string, opts ...Option) ([]*ClientIDRecord, error) { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetClientIDRecords for project %s\n", projectName) - var res []*ClientIDRecord - - if tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress); err != nil { + o := makeOptions(opts...) + tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress, o.dockerRegistryOptions...) + if err != nil { return nil, fmt.Errorf("unable to get repo %s tags: %s", storage.RepoAddress, err) - } else { - for _, tag := range tags { - if !strings.HasPrefix(tag, RepoClientIDRecord_ImageTagPrefix) { - continue - } - - tagWithoutPrefix := strings.TrimPrefix(tag, RepoClientIDRecord_ImageTagPrefix) - dataParts := strings.SplitN(util.Reverse(tagWithoutPrefix), "-", 2) - if len(dataParts) != 2 { - continue - } + } - clientID, timestampMillisecStr := util.Reverse(dataParts[1]), util.Reverse(dataParts[0]) + var res []*ClientIDRecord + for _, tag := range tags { + if !strings.HasPrefix(tag, RepoClientIDRecord_ImageTagPrefix) { + continue + } - timestampMillisec, err := strconv.ParseInt(timestampMillisecStr, 10, 64) - if err != nil { - continue - } + tagWithoutPrefix := strings.TrimPrefix(tag, RepoClientIDRecord_ImageTagPrefix) + dataParts := strings.SplitN(util.Reverse(tagWithoutPrefix), "-", 2) + if len(dataParts) != 2 { + continue + } - rec := &ClientIDRecord{ClientID: clientID, TimestampMillisec: timestampMillisec} - res = append(res, rec) + clientID, timestampMillisecStr := util.Reverse(dataParts[1]), util.Reverse(dataParts[0]) - logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetClientIDRecords got clientID record: %s\n", rec) + timestampMillisec, err := strconv.ParseInt(timestampMillisecStr, 10, 64) + if err != nil { + continue } + + rec := &ClientIDRecord{ClientID: clientID, TimestampMillisec: timestampMillisec} + res = append(res, rec) + + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetClientIDRecords got clientID record: %s\n", rec) } return res, nil @@ -864,7 +875,7 @@ func (storage *RepoStagesStorage) PostClientIDRecord(ctx context.Context, projec logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PostClientID full image name: %s\n", fullImageName) - if isExists, err := storage.DockerRegistry.IsRepoImageExists(ctx, fullImageName); err != nil { + if isExists, err := storage.DockerRegistry.IsTagExist(ctx, fullImageName); err != nil { return err } else if isExists { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage record %q is exists => exiting\n", fullImageName) diff --git a/pkg/storage/stages_storage.go b/pkg/storage/stages_storage.go index ffd7eb54df..47a9b6eaf0 100644 --- a/pkg/storage/stages_storage.go +++ b/pkg/storage/stages_storage.go @@ -33,7 +33,7 @@ type StagesStorage interface { AddStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error CheckStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error DeleteStageCustomTag(ctx context.Context, tag string) error - GetStageCustomTagMetadataIDs(ctx context.Context) ([]string, error) + GetStageCustomTagMetadataIDs(ctx context.Context, opts ...Option) ([]string, error) GetStageCustomTagMetadata(ctx context.Context, tagOrID string) (*CustomTagMetadata, error) RejectStage(ctx context.Context, projectName, digest string, uniqueID int64) error @@ -56,20 +56,21 @@ type StagesStorage interface { // if the name contains unsupported special characters, or // if the name exceeds the docker tag limit. RmManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error + IsManagedImageExist(ctx context.Context, projectName, imageNameOrManagedImageName string, opts ...Option) (bool, error) // GetManagedImages returns the list of managedImageName. - GetManagedImages(ctx context.Context, projectName string) ([]string, error) + GetManagedImages(ctx context.Context, projectName string, opts ...Option) ([]string, error) PutImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) error RmImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID string) error - IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) (bool, error) - GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string) (map[string]map[string][]string, map[string]map[string][]string, error) + IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string, opts ...Option) (bool, error) + GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string, opts ...Option) (map[string]map[string][]string, map[string]map[string][]string, error) GetImportMetadata(ctx context.Context, projectName, id string) (*ImportMetadata, error) PutImportMetadata(ctx context.Context, projectName string, metadata *ImportMetadata) error RmImportMetadata(ctx context.Context, projectName, id string) error - GetImportMetadataIDs(ctx context.Context, projectName string) ([]string, error) + GetImportMetadataIDs(ctx context.Context, projectName string, opts ...Option) ([]string, error) - GetClientIDRecords(ctx context.Context, projectName string) ([]*ClientIDRecord, error) + GetClientIDRecords(ctx context.Context, projectName string, opts ...Option) ([]*ClientIDRecord, error) PostClientIDRecord(ctx context.Context, projectName string, rec *ClientIDRecord) error String() string