From 04404a3a6b7d1c59cd93fe9e8cfe9e6a8158be16 Mon Sep 17 00:00:00 2001 From: Alexey Igrychev Date: Thu, 17 Feb 2022 17:13:29 +0000 Subject: [PATCH] feat(cleanup): cleaning up artifacts by git history-based policy as well as images Update managed image and image metadata format - support spaces in image name (for artifacts) - support image name of any length - update naming Treat artifacts as images - add artifacts to managed images - put image metadata (commits for git history based cleanup) for artifacts Signed-off-by: Alexey Igrychev --- pkg/build/build_phase.go | 8 +- pkg/slug/slug.go | 14 +++- pkg/slug/slug_test.go | 8 +- pkg/storage/repo_stages_storage.go | 116 +++++++++++++++-------------- pkg/storage/stages_storage.go | 18 +++-- 5 files changed, 93 insertions(+), 71 deletions(-) diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 022e95fd73..df79edcb0b 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -225,10 +225,6 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error img.SetLastNonEmptyStage(phase.StagesIterator.PrevNonEmptyStage) img.SetContentDigest(phase.StagesIterator.PrevNonEmptyStage.GetContentDigest()) - if img.isArtifact { - return nil - } - if err := phase.addManagedImage(ctx, img); err != nil { return err } @@ -237,6 +233,10 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error return err } + if img.isArtifact { + return nil + } + if !phase.ShouldBeBuiltMode { if err := phase.addCustomImageTagsToStagesStorage(ctx, img); err != nil { return fmt.Errorf("unable to add custom image tags to stages storage: %s", err) diff --git a/pkg/slug/slug.go b/pkg/slug/slug.go index a5fc3ec293..856751c26f 100644 --- a/pkg/slug/slug.go +++ b/pkg/slug/slug.go @@ -16,7 +16,7 @@ var ( DefaultSlugMaxSize = 42 // legacy dockerTagRegexp = regexp.MustCompile(`^[\w][\w.-]*$`) - dockerTagMaxSize = 128 + DockerTagMaxSize = 128 projectNameRegex = regexp.MustCompile(`^(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])$`) projectNameMaxSize = 50 @@ -61,13 +61,21 @@ func validateProject(name string) error { func DockerTag(name string) string { if err := ValidateDockerTag(name); err != nil { - return slug(name, dockerTagMaxSize) + return slug(name, DockerTagMaxSize) } return name } +func IsValidDockerTag(name string) bool { + if shouldNotBeSlugged(name, dockerTagRegexp, DockerTagMaxSize) { + return true + } + + return false +} + func ValidateDockerTag(name string) error { - if shouldNotBeSlugged(name, dockerTagRegexp, dockerTagMaxSize) { + if IsValidDockerTag(name) { return nil } diff --git a/pkg/slug/slug_test.go b/pkg/slug/slug_test.go index f916a63fcd..db0a840873 100644 --- a/pkg/slug/slug_test.go +++ b/pkg/slug/slug_test.go @@ -78,8 +78,8 @@ func TestDockerTag(t *testing.T) { }, { name: "maxSizeExceeded", - data: strings.Repeat("x", dockerTagMaxSize+1), - result: strings.Repeat("x", dockerTagMaxSize-servicePartSize) + "-8cca70eb", + data: strings.Repeat("x", DockerTagMaxSize+1), + result: strings.Repeat("x", DockerTagMaxSize-servicePartSize) + "-8cca70eb", }, } @@ -90,8 +90,8 @@ func TestDockerTag(t *testing.T) { t.Errorf("\n[EXPECTED]: %s (%d)\n[GOT]: %s (%d)", test.result, len(test.result), result, len(result)) } - if len(result) > dockerTagMaxSize { - t.Errorf("Max size exceeded: [EXPECTED]: %d [GOT]: %d", dockerTagMaxSize, len(result)) + if len(result) > DockerTagMaxSize { + t.Errorf("Max size exceeded: [EXPECTED]: %d [GOT]: %d", DockerTagMaxSize, len(result)) } }) diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index b5f1ada2f1..c7b1f6c323 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -411,14 +411,10 @@ func (storage *RepoStagesStorage) GetStageCustomTagMetadataIDs(ctx context.Conte return res, nil } -func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectName, imageName string) error { - logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage %s %s\n", projectName, imageName) +func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error { + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage %s %s\n", projectName, imageNameOrManagedImageName) - if validateImageName(imageName) != nil { - return nil - } - - fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageName) + fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageNameOrManagedImageName) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage full image name: %s\n", fullImageName) if isExists, err := storage.DockerRegistry.IsRepoImageExists(ctx, fullImageName); err != nil { @@ -439,10 +435,10 @@ func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectNa return nil } -func (storage *RepoStagesStorage) RmManagedImage(ctx context.Context, projectName, imageName string) error { - logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmManagedImage %s %s\n", projectName, imageName) +func (storage *RepoStagesStorage) RmManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error { + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmManagedImage %s %s\n", projectName, imageNameOrManagedImageName) - fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageName) + fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageNameOrManagedImageName) imgInfo, err := storage.DockerRegistry.TryGetRepoImage(ctx, fullImageName) if err != nil { @@ -474,12 +470,7 @@ func (storage *RepoStagesStorage) GetManagedImages(ctx context.Context, projectN continue } - managedImageName := unslugDockerImageTagAsImageName(strings.TrimPrefix(tag, RepoManagedImageRecord_ImageTagPrefix)) - - if validateImageName(managedImageName) != nil { - continue - } - + managedImageName := getManagedImageNameFromManagedImageID(strings.TrimPrefix(tag, RepoManagedImageRecord_ImageTagPrefix)) res = append(res, managedImageName) } } @@ -522,10 +513,10 @@ func (storage *RepoStagesStorage) ShouldFetchImage(ctx context.Context, img cont return true, nil } -func (storage *RepoStagesStorage) PutImageMetadata(ctx context.Context, projectName, imageName, commit, stageID string) error { - logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata %s %s %s %s\n", projectName, imageName, commit, stageID) +func (storage *RepoStagesStorage) PutImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) error { + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata %s %s %s %s\n", projectName, imageNameOrManagedImageName, commit, stageID) - fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageName, commit, stageID) + fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageNameOrManagedImageName, commit, stageID) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata full image name: %s\n", fullImageName) opts := &docker_registry.PushImageOptions{Labels: map[string]string{image.WerfLabel: projectName}} @@ -533,15 +524,15 @@ func (storage *RepoStagesStorage) PutImageMetadata(ctx context.Context, projectN if err := storage.DockerRegistry.PushImage(ctx, fullImageName, opts); err != nil { return fmt.Errorf("unable to push image %s: %s", fullImageName, err) } - logboek.Context(ctx).Info().LogF("Put image %s commit %s stage ID %s\n", imageName, commit, stageID) + logboek.Context(ctx).Info().LogF("Put image %s commit %s stage ID %s\n", imageNameOrManagedImageName, commit, stageID) return nil } -func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectName, imageNameOrID, commit, stageID string) error { - logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmImageMetadata %s %s %s %s\n", projectName, imageNameOrID, commit, stageID) +func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID string) error { + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmImageMetadata %s %s %s %s\n", projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID) - img, err := storage.selectMetadataNameImage(ctx, imageNameOrID, commit, stageID) + img, err := storage.selectMetadataNameImage(ctx, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID) if err != nil { return err } @@ -555,7 +546,7 @@ func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectNa return fmt.Errorf("unable to remove repo image %s: %s", img.Tag, err) } - logboek.Context(ctx).Info().LogF("Removed image %s commit %s stage ID %s\n", imageNameOrID, commit, stageID) + logboek.Context(ctx).Info().LogF("Removed image %s commit %s stage ID %s\n", imageNameOrManagedImageNameOrImageMetadataID, commit, stageID) return nil } @@ -575,17 +566,17 @@ func (storage *RepoStagesStorage) selectMetadataNameImage(ctx context.Context, i return nil, nil } -func (storage *RepoStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageName, commit, stageID string) (bool, error) { - logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist %s %s %s %s\n", projectName, imageName, commit, stageID) +func (storage *RepoStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) (bool, error) { + logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist %s %s %s %s\n", projectName, imageNameOrManagedImageName, commit, stageID) - fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageName, commit, stageID) + fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageNameOrManagedImageName, commit, stageID) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist full image name: %s\n", fullImageName) img, err := storage.DockerRegistry.TryGetRepoImage(ctx, fullImageName) return img != nil, err } -func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameList []string) (map[string]map[string][]string, map[string]map[string][]string, error) { +func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string) (map[string]map[string][]string, map[string]map[string][]string, error) { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetImageNameStageIDCommitList %s %s\n", projectName) tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress) @@ -593,7 +584,7 @@ func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx con return nil, nil, fmt.Errorf("unable to get repo %s tags: %s", storage.RepoAddress, err) } - return groupImageMetadataTagsByImageName(ctx, imageNameList, tags, RepoImageMetadataByCommitRecord_ImageTagPrefix) + return groupImageMetadataTagsByImageName(ctx, imageNameOrManagedImageList, tags, RepoImageMetadataByCommitRecord_ImageTagPrefix) } func (storage *RepoStagesStorage) GetImportMetadata(ctx context.Context, _, id string) (*ImportMetadata, error) { @@ -680,10 +671,10 @@ func makeRepoImportMetadataName(repoAddress, importSourceID string) string { return fmt.Sprintf(RepoImportMetadata_ImageNameFormat, repoAddress, importSourceID) } -func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []string, tags []string, imageTagPrefix string) (map[string]map[string][]string, map[string]map[string][]string, error) { - imageNameNameByID := map[string]string{} - for _, imageName := range imageNameList { - imageNameNameByID[imageNameID(imageName)] = imageName +func groupImageMetadataTagsByImageName(ctx context.Context, imageNameOrManagedImageList []string, tags []string, imageTagPrefix string) (map[string]map[string][]string, map[string]map[string][]string, error) { + imageMetadataIDImageNameOrManagedImageName := map[string]string{} + for _, imageNameOrManagedImageName := range imageNameOrManagedImageList { + imageMetadataIDImageNameOrManagedImageName[getImageMetadataID(imageNameOrManagedImageName)] = imageNameOrManagedImageName } result := map[string]map[string][]string{} @@ -708,7 +699,7 @@ func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []stri logboek.Context(ctx).Debug().LogF("Found image ID %s commit %s stage ID %s\n", tagImageNameID, tagCommit, tagStageID) - imageName, ok := imageNameNameByID[tagImageNameID] + imageName, ok := imageMetadataIDImageNameOrManagedImageName[tagImageNameID] if !ok { res = resultNotManagedImageName imageName = tagImageNameID @@ -734,15 +725,20 @@ func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []stri return result, resultNotManagedImageName, nil } -func makeRepoImageMetadataName(repoAddress, imageNameOrID, commit, stageID string) string { - return makeRepoImageMetadataNameByImageID(repoAddress, imageNameID(imageNameOrID), commit, stageID) +func makeRepoImageMetadataName(repoAddress, imageNameOrManagedImageName, commit, stageID string) string { + return strings.Join([]string{repoAddress, makeRepoImageMetadataTagName(imageNameOrManagedImageName, commit, stageID)}, ":") +} + +func makeRepoImageMetadataNameByImageMetadataID(repoAddress, imageMetadataID, commit, stageID string) string { + return strings.Join([]string{repoAddress, makeRepoImageMetadataTagNameByImageID(imageMetadataID, commit, stageID)}, ":") } -func makeRepoImageMetadataNameByImageID(repoAddress, imageID, commit, stageID string) string { - return strings.Join([]string{ - repoAddress, - slug.DockerTag(fmt.Sprintf(RepoImageMetadataByCommitRecord_TagFormat, imageID, commit, stageID)), - }, ":") +func makeRepoImageMetadataTagName(imageNameOrManagedImageName, commit, stageID string) string { + return makeRepoImageMetadataTagNameByImageID(getImageMetadataID(imageNameOrManagedImageName), commit, stageID) +} + +func makeRepoImageMetadataTagNameByImageID(imageMetadataID, commit, stageID string) string { + return fmt.Sprintf(RepoImageMetadataByCommitRecord_TagFormat, imageMetadataID, commit, stageID) } func (storage *RepoStagesStorage) String() string { @@ -757,16 +753,34 @@ func makeRepoCustomTagMetadataRecord(repoAddress, tag string) string { return fmt.Sprintf(RepoCustomTagMetadata_ImageNameFormat, repoAddress, slug.LimitedSlug(tag, 48)) } -func makeRepoManagedImageRecord(repoAddress, imageName string) string { - return fmt.Sprintf(RepoManagedImageRecord_ImageNameFormat, repoAddress, slugImageNameAsDockerImageTag(imageName)) +func makeRepoManagedImageRecord(repoAddress, imageNameOrManagedImageName string) string { + return fmt.Sprintf(RepoManagedImageRecord_ImageNameFormat, repoAddress, getManagedImageID(imageNameOrManagedImageName)) } -func imageNameID(imageName string) string { - return util.MurmurHash(imageName) +func getManagedImageID(imageNameOrManagedImageName string) string { + imageNameOrManagedImageName = slugImageName(imageNameOrManagedImageName) + if !slug.IsValidDockerTag(imageNameOrManagedImageName) { + imageNameOrManagedImageName = slug.LimitedSlug(imageNameOrManagedImageName, slug.DockerTagMaxSize-len(RepoManagedImageRecord_ImageTagPrefix)) + } + + return imageNameOrManagedImageName +} + +func getManagedImageNameFromManagedImageID(managedImageID string) string { + return unslugImageName(managedImageID) } -func slugImageNameAsDockerImageTag(imageName string) string { +func getImageMetadataID(imageNameOrManagedImageName string) string { + return util.MurmurHash(getManagedImageNameByImageNameOrManagedImage(imageNameOrManagedImageName)) +} + +func getManagedImageNameByImageNameOrManagedImage(imageNameOrManagedImageName string) string { + return getManagedImageNameFromManagedImageID(getManagedImageID(imageNameOrManagedImageName)) +} + +func slugImageName(imageName string) string { res := imageName + res = strings.ReplaceAll(res, " ", "__space__") res = strings.ReplaceAll(res, "/", "__slash__") res = strings.ReplaceAll(res, "+", "__plus__") @@ -777,8 +791,9 @@ func slugImageNameAsDockerImageTag(imageName string) string { return res } -func unslugDockerImageTagAsImageName(tag string) string { +func unslugImageName(tag string) string { res := tag + res = strings.ReplaceAll(res, "__space__", " ") res = strings.ReplaceAll(res, "__slash__", "/") res = strings.ReplaceAll(res, "__plus__", "+") @@ -789,13 +804,6 @@ func unslugDockerImageTagAsImageName(tag string) string { return res } -func validateImageName(name string) error { - if strings.ToLower(name) != name { - return fmt.Errorf("no upcase symbols allowed") - } - return nil -} - func (storage *RepoStagesStorage) GetClientIDRecords(ctx context.Context, projectName string) ([]*ClientIDRecord, error) { logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetClientIDRecords for project %s\n", projectName) diff --git a/pkg/storage/stages_storage.go b/pkg/storage/stages_storage.go index 6ce1d77e9e..a8ca63adae 100644 --- a/pkg/storage/stages_storage.go +++ b/pkg/storage/stages_storage.go @@ -44,14 +44,20 @@ type StagesStorage interface { CreateRepo(ctx context.Context) error DeleteRepo(ctx context.Context) error - AddManagedImage(ctx context.Context, projectName, imageName string) error - RmManagedImage(ctx context.Context, projectName, imageName string) error + // AddManagedImage adds managed image. + AddManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error + // RmManagedImage removes managed image by imageName or managedImageName. + // Typically, the managedImageName is the same as the imageName, but may be different + // if the name contains unsupported special characters, or + // if the name exceeds the docker tag limit. + RmManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error + // GetManagedImages returns the list of managedImageName. GetManagedImages(ctx context.Context, projectName string) ([]string, error) - PutImageMetadata(ctx context.Context, projectName, imageName, commit, stageID string) error - RmImageMetadata(ctx context.Context, projectName, imageNameOrID, commit, stageID string) error - IsImageMetadataExist(ctx context.Context, projectName, imageName, commit, stageID string) (bool, error) - GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameList []string) (map[string]map[string][]string, map[string]map[string][]string, error) + PutImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) error + RmImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID string) error + IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) (bool, error) + GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string) (map[string]map[string][]string, map[string]map[string][]string, error) GetImportMetadata(ctx context.Context, projectName, id string) (*ImportMetadata, error) PutImportMetadata(ctx context.Context, projectName string, metadata *ImportMetadata) error