Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat(cleanup): cleaning up artifacts by git history-based policy as w…
…ell as images

Update managed image and image metadata format

- support spaces in image name (for artifacts)
- support image name of any length
- update naming

Treat artifacts as images

- add artifacts to managed images
- put image metadata (commits for git history based cleanup) for artifacts

Signed-off-by: Alexey Igrychev <alexey.igrychev@flant.com>
  • Loading branch information
alexey-igrychev committed Feb 17, 2022
1 parent b59f07c commit 04404a3
Show file tree
Hide file tree
Showing 5 changed files with 93 additions and 71 deletions.
8 changes: 4 additions & 4 deletions pkg/build/build_phase.go
Expand Up @@ -225,10 +225,6 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
img.SetLastNonEmptyStage(phase.StagesIterator.PrevNonEmptyStage)
img.SetContentDigest(phase.StagesIterator.PrevNonEmptyStage.GetContentDigest())

if img.isArtifact {
return nil
}

if err := phase.addManagedImage(ctx, img); err != nil {
return err
}
Expand All @@ -237,6 +233,10 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
return err
}

if img.isArtifact {
return nil
}

if !phase.ShouldBeBuiltMode {
if err := phase.addCustomImageTagsToStagesStorage(ctx, img); err != nil {
return fmt.Errorf("unable to add custom image tags to stages storage: %s", err)
Expand Down
14 changes: 11 additions & 3 deletions pkg/slug/slug.go
Expand Up @@ -16,7 +16,7 @@ var (
DefaultSlugMaxSize = 42 // legacy

dockerTagRegexp = regexp.MustCompile(`^[\w][\w.-]*$`)
dockerTagMaxSize = 128
DockerTagMaxSize = 128

projectNameRegex = regexp.MustCompile(`^(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])$`)
projectNameMaxSize = 50
Expand Down Expand Up @@ -61,13 +61,21 @@ func validateProject(name string) error {

func DockerTag(name string) string {
if err := ValidateDockerTag(name); err != nil {
return slug(name, dockerTagMaxSize)
return slug(name, DockerTagMaxSize)
}
return name
}

func IsValidDockerTag(name string) bool {
if shouldNotBeSlugged(name, dockerTagRegexp, DockerTagMaxSize) {
return true
}

return false
}

func ValidateDockerTag(name string) error {
if shouldNotBeSlugged(name, dockerTagRegexp, dockerTagMaxSize) {
if IsValidDockerTag(name) {
return nil
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/slug/slug_test.go
Expand Up @@ -78,8 +78,8 @@ func TestDockerTag(t *testing.T) {
},
{
name: "maxSizeExceeded",
data: strings.Repeat("x", dockerTagMaxSize+1),
result: strings.Repeat("x", dockerTagMaxSize-servicePartSize) + "-8cca70eb",
data: strings.Repeat("x", DockerTagMaxSize+1),
result: strings.Repeat("x", DockerTagMaxSize-servicePartSize) + "-8cca70eb",
},
}

Expand All @@ -90,8 +90,8 @@ func TestDockerTag(t *testing.T) {
t.Errorf("\n[EXPECTED]: %s (%d)\n[GOT]: %s (%d)", test.result, len(test.result), result, len(result))
}

if len(result) > dockerTagMaxSize {
t.Errorf("Max size exceeded: [EXPECTED]: %d [GOT]: %d", dockerTagMaxSize, len(result))
if len(result) > DockerTagMaxSize {
t.Errorf("Max size exceeded: [EXPECTED]: %d [GOT]: %d", DockerTagMaxSize, len(result))
}
})

Expand Down
116 changes: 62 additions & 54 deletions pkg/storage/repo_stages_storage.go
Expand Up @@ -411,14 +411,10 @@ func (storage *RepoStagesStorage) GetStageCustomTagMetadataIDs(ctx context.Conte
return res, nil
}

func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectName, imageName string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage %s %s\n", projectName, imageName)
func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage %s %s\n", projectName, imageNameOrManagedImageName)

if validateImageName(imageName) != nil {
return nil
}

fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageName)
fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageNameOrManagedImageName)
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.AddManagedImage full image name: %s\n", fullImageName)

if isExists, err := storage.DockerRegistry.IsRepoImageExists(ctx, fullImageName); err != nil {
Expand All @@ -439,10 +435,10 @@ func (storage *RepoStagesStorage) AddManagedImage(ctx context.Context, projectNa
return nil
}

func (storage *RepoStagesStorage) RmManagedImage(ctx context.Context, projectName, imageName string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmManagedImage %s %s\n", projectName, imageName)
func (storage *RepoStagesStorage) RmManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmManagedImage %s %s\n", projectName, imageNameOrManagedImageName)

fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageName)
fullImageName := makeRepoManagedImageRecord(storage.RepoAddress, imageNameOrManagedImageName)

imgInfo, err := storage.DockerRegistry.TryGetRepoImage(ctx, fullImageName)
if err != nil {
Expand Down Expand Up @@ -474,12 +470,7 @@ func (storage *RepoStagesStorage) GetManagedImages(ctx context.Context, projectN
continue
}

managedImageName := unslugDockerImageTagAsImageName(strings.TrimPrefix(tag, RepoManagedImageRecord_ImageTagPrefix))

if validateImageName(managedImageName) != nil {
continue
}

managedImageName := getManagedImageNameFromManagedImageID(strings.TrimPrefix(tag, RepoManagedImageRecord_ImageTagPrefix))
res = append(res, managedImageName)
}
}
Expand Down Expand Up @@ -522,26 +513,26 @@ func (storage *RepoStagesStorage) ShouldFetchImage(ctx context.Context, img cont
return true, nil
}

func (storage *RepoStagesStorage) PutImageMetadata(ctx context.Context, projectName, imageName, commit, stageID string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata %s %s %s %s\n", projectName, imageName, commit, stageID)
func (storage *RepoStagesStorage) PutImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata %s %s %s %s\n", projectName, imageNameOrManagedImageName, commit, stageID)

fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageName, commit, stageID)
fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageNameOrManagedImageName, commit, stageID)
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImageMetadata full image name: %s\n", fullImageName)

opts := &docker_registry.PushImageOptions{Labels: map[string]string{image.WerfLabel: projectName}}

if err := storage.DockerRegistry.PushImage(ctx, fullImageName, opts); err != nil {
return fmt.Errorf("unable to push image %s: %s", fullImageName, err)
}
logboek.Context(ctx).Info().LogF("Put image %s commit %s stage ID %s\n", imageName, commit, stageID)
logboek.Context(ctx).Info().LogF("Put image %s commit %s stage ID %s\n", imageNameOrManagedImageName, commit, stageID)

return nil
}

func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectName, imageNameOrID, commit, stageID string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmImageMetadata %s %s %s %s\n", projectName, imageNameOrID, commit, stageID)
func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID string) error {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.RmImageMetadata %s %s %s %s\n", projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID)

img, err := storage.selectMetadataNameImage(ctx, imageNameOrID, commit, stageID)
img, err := storage.selectMetadataNameImage(ctx, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID)
if err != nil {
return err
}
Expand All @@ -555,7 +546,7 @@ func (storage *RepoStagesStorage) RmImageMetadata(ctx context.Context, projectNa
return fmt.Errorf("unable to remove repo image %s: %s", img.Tag, err)
}

logboek.Context(ctx).Info().LogF("Removed image %s commit %s stage ID %s\n", imageNameOrID, commit, stageID)
logboek.Context(ctx).Info().LogF("Removed image %s commit %s stage ID %s\n", imageNameOrManagedImageNameOrImageMetadataID, commit, stageID)

return nil
}
Expand All @@ -575,25 +566,25 @@ func (storage *RepoStagesStorage) selectMetadataNameImage(ctx context.Context, i
return nil, nil
}

func (storage *RepoStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageName, commit, stageID string) (bool, error) {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist %s %s %s %s\n", projectName, imageName, commit, stageID)
func (storage *RepoStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) (bool, error) {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist %s %s %s %s\n", projectName, imageNameOrManagedImageName, commit, stageID)

fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageName, commit, stageID)
fullImageName := makeRepoImageMetadataName(storage.RepoAddress, imageNameOrManagedImageName, commit, stageID)
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.IsImageMetadataExist full image name: %s\n", fullImageName)

img, err := storage.DockerRegistry.TryGetRepoImage(ctx, fullImageName)
return img != nil, err
}

func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameList []string) (map[string]map[string][]string, map[string]map[string][]string, error) {
func (storage *RepoStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string) (map[string]map[string][]string, map[string]map[string][]string, error) {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetImageNameStageIDCommitList %s %s\n", projectName)

tags, err := storage.DockerRegistry.Tags(ctx, storage.RepoAddress)
if err != nil {
return nil, nil, fmt.Errorf("unable to get repo %s tags: %s", storage.RepoAddress, err)
}

return groupImageMetadataTagsByImageName(ctx, imageNameList, tags, RepoImageMetadataByCommitRecord_ImageTagPrefix)
return groupImageMetadataTagsByImageName(ctx, imageNameOrManagedImageList, tags, RepoImageMetadataByCommitRecord_ImageTagPrefix)
}

func (storage *RepoStagesStorage) GetImportMetadata(ctx context.Context, _, id string) (*ImportMetadata, error) {
Expand Down Expand Up @@ -680,10 +671,10 @@ func makeRepoImportMetadataName(repoAddress, importSourceID string) string {
return fmt.Sprintf(RepoImportMetadata_ImageNameFormat, repoAddress, importSourceID)
}

func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []string, tags []string, imageTagPrefix string) (map[string]map[string][]string, map[string]map[string][]string, error) {
imageNameNameByID := map[string]string{}
for _, imageName := range imageNameList {
imageNameNameByID[imageNameID(imageName)] = imageName
func groupImageMetadataTagsByImageName(ctx context.Context, imageNameOrManagedImageList []string, tags []string, imageTagPrefix string) (map[string]map[string][]string, map[string]map[string][]string, error) {
imageMetadataIDImageNameOrManagedImageName := map[string]string{}
for _, imageNameOrManagedImageName := range imageNameOrManagedImageList {
imageMetadataIDImageNameOrManagedImageName[getImageMetadataID(imageNameOrManagedImageName)] = imageNameOrManagedImageName
}

result := map[string]map[string][]string{}
Expand All @@ -708,7 +699,7 @@ func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []stri

logboek.Context(ctx).Debug().LogF("Found image ID %s commit %s stage ID %s\n", tagImageNameID, tagCommit, tagStageID)

imageName, ok := imageNameNameByID[tagImageNameID]
imageName, ok := imageMetadataIDImageNameOrManagedImageName[tagImageNameID]
if !ok {
res = resultNotManagedImageName
imageName = tagImageNameID
Expand All @@ -734,15 +725,20 @@ func groupImageMetadataTagsByImageName(ctx context.Context, imageNameList []stri
return result, resultNotManagedImageName, nil
}

func makeRepoImageMetadataName(repoAddress, imageNameOrID, commit, stageID string) string {
return makeRepoImageMetadataNameByImageID(repoAddress, imageNameID(imageNameOrID), commit, stageID)
func makeRepoImageMetadataName(repoAddress, imageNameOrManagedImageName, commit, stageID string) string {
return strings.Join([]string{repoAddress, makeRepoImageMetadataTagName(imageNameOrManagedImageName, commit, stageID)}, ":")
}

func makeRepoImageMetadataNameByImageMetadataID(repoAddress, imageMetadataID, commit, stageID string) string {
return strings.Join([]string{repoAddress, makeRepoImageMetadataTagNameByImageID(imageMetadataID, commit, stageID)}, ":")
}

func makeRepoImageMetadataNameByImageID(repoAddress, imageID, commit, stageID string) string {
return strings.Join([]string{
repoAddress,
slug.DockerTag(fmt.Sprintf(RepoImageMetadataByCommitRecord_TagFormat, imageID, commit, stageID)),
}, ":")
func makeRepoImageMetadataTagName(imageNameOrManagedImageName, commit, stageID string) string {
return makeRepoImageMetadataTagNameByImageID(getImageMetadataID(imageNameOrManagedImageName), commit, stageID)
}

func makeRepoImageMetadataTagNameByImageID(imageMetadataID, commit, stageID string) string {
return fmt.Sprintf(RepoImageMetadataByCommitRecord_TagFormat, imageMetadataID, commit, stageID)
}

func (storage *RepoStagesStorage) String() string {
Expand All @@ -757,16 +753,34 @@ func makeRepoCustomTagMetadataRecord(repoAddress, tag string) string {
return fmt.Sprintf(RepoCustomTagMetadata_ImageNameFormat, repoAddress, slug.LimitedSlug(tag, 48))
}

func makeRepoManagedImageRecord(repoAddress, imageName string) string {
return fmt.Sprintf(RepoManagedImageRecord_ImageNameFormat, repoAddress, slugImageNameAsDockerImageTag(imageName))
func makeRepoManagedImageRecord(repoAddress, imageNameOrManagedImageName string) string {
return fmt.Sprintf(RepoManagedImageRecord_ImageNameFormat, repoAddress, getManagedImageID(imageNameOrManagedImageName))
}

func imageNameID(imageName string) string {
return util.MurmurHash(imageName)
func getManagedImageID(imageNameOrManagedImageName string) string {
imageNameOrManagedImageName = slugImageName(imageNameOrManagedImageName)
if !slug.IsValidDockerTag(imageNameOrManagedImageName) {
imageNameOrManagedImageName = slug.LimitedSlug(imageNameOrManagedImageName, slug.DockerTagMaxSize-len(RepoManagedImageRecord_ImageTagPrefix))
}

return imageNameOrManagedImageName
}

func getManagedImageNameFromManagedImageID(managedImageID string) string {
return unslugImageName(managedImageID)
}

func slugImageNameAsDockerImageTag(imageName string) string {
func getImageMetadataID(imageNameOrManagedImageName string) string {
return util.MurmurHash(getManagedImageNameByImageNameOrManagedImage(imageNameOrManagedImageName))
}

func getManagedImageNameByImageNameOrManagedImage(imageNameOrManagedImageName string) string {
return getManagedImageNameFromManagedImageID(getManagedImageID(imageNameOrManagedImageName))
}

func slugImageName(imageName string) string {
res := imageName
res = strings.ReplaceAll(res, " ", "__space__")
res = strings.ReplaceAll(res, "/", "__slash__")
res = strings.ReplaceAll(res, "+", "__plus__")

Expand All @@ -777,8 +791,9 @@ func slugImageNameAsDockerImageTag(imageName string) string {
return res
}

func unslugDockerImageTagAsImageName(tag string) string {
func unslugImageName(tag string) string {
res := tag
res = strings.ReplaceAll(res, "__space__", " ")
res = strings.ReplaceAll(res, "__slash__", "/")
res = strings.ReplaceAll(res, "__plus__", "+")

Expand All @@ -789,13 +804,6 @@ func unslugDockerImageTagAsImageName(tag string) string {
return res
}

func validateImageName(name string) error {
if strings.ToLower(name) != name {
return fmt.Errorf("no upcase symbols allowed")
}
return nil
}

func (storage *RepoStagesStorage) GetClientIDRecords(ctx context.Context, projectName string) ([]*ClientIDRecord, error) {
logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.GetClientIDRecords for project %s\n", projectName)

Expand Down
18 changes: 12 additions & 6 deletions pkg/storage/stages_storage.go
Expand Up @@ -44,14 +44,20 @@ type StagesStorage interface {
CreateRepo(ctx context.Context) error
DeleteRepo(ctx context.Context) error

AddManagedImage(ctx context.Context, projectName, imageName string) error
RmManagedImage(ctx context.Context, projectName, imageName string) error
// AddManagedImage adds managed image.
AddManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error
// RmManagedImage removes managed image by imageName or managedImageName.
// Typically, the managedImageName is the same as the imageName, but may be different
// if the name contains unsupported special characters, or
// if the name exceeds the docker tag limit.
RmManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error
// GetManagedImages returns the list of managedImageName.
GetManagedImages(ctx context.Context, projectName string) ([]string, error)

PutImageMetadata(ctx context.Context, projectName, imageName, commit, stageID string) error
RmImageMetadata(ctx context.Context, projectName, imageNameOrID, commit, stageID string) error
IsImageMetadataExist(ctx context.Context, projectName, imageName, commit, stageID string) (bool, error)
GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameList []string) (map[string]map[string][]string, map[string]map[string][]string, error)
PutImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) error
RmImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID string) error
IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) (bool, error)
GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string) (map[string]map[string][]string, map[string]map[string][]string, error)

GetImportMetadata(ctx context.Context, projectName, id string) (*ImportMetadata, error)
PutImportMetadata(ctx context.Context, projectName string, metadata *ImportMetadata) error
Expand Down

0 comments on commit 04404a3

Please sign in to comment.