Skip to content

Commit

Permalink
Merge pull request #4196 from werf/feat_cleanup_use_git_history_based…
Browse files Browse the repository at this point in the history 
…_cleanup_policy_for_artifacts

feat(cleanup): cleaning up artifacts by git history-based policy as well as images
  • Loading branch information
@ilya-lesikov
ilya-lesikov committed Feb 18, 2022
2 parents dfa159c + b9d0440 commit bcf8205
Show file tree
Hide file tree
Showing 6 changed files with 121 additions and 195 deletions.
108 changes: 0 additions & 108 deletions integration/suites/cleanup/cleanup_test.go
View file
Expand Up @@ -594,114 +594,6 @@ var _ = Describe("cleanup command", func() {
Ω(StagesCount()).Should(Equal(count))
})

Context("imports metadata", func() {
It("should keep used artifact", func() {
utils.RunSucceedCommand(
SuiteData.TestDirPath,
SuiteData.WerfBinPath,
"build",
)

utils.RunSucceedCommand(
SuiteData.TestDirPath,
"git",
"push", "--set-upstream", "origin", branchName,
)

countAfterFirstBuild := StagesCount()
Ω(countAfterFirstBuild).Should(Equal(4))

utils.RunSucceedCommand(
SuiteData.TestDirPath,
SuiteData.WerfBinPath,
"cleanup",
)

countAfterCleanup := StagesCount()
Ω(countAfterCleanup).Should(Equal(4))
Ω(len(ImportMetadataIDs())).Should(BeEquivalentTo(1))
})

It("should keep both artifacts by identical import checksum", func() {
utils.RunSucceedCommand(
SuiteData.TestDirPath,
SuiteData.WerfBinPath,
"build",
)

countAfterFirstBuild := StagesCount()
Ω(countAfterFirstBuild).Should(Equal(4))

utils.RunSucceedCommand(
SuiteData.TestDirPath,
"git",
"commit", "--allow-empty", "-m", "test",
)

SuiteData.Stubs.SetEnv("WERF_CONFIG", "werf_2b.yaml") // full artifact rebuild
utils.RunSucceedCommand(
SuiteData.TestDirPath,
SuiteData.WerfBinPath,
"build",
)

utils.RunSucceedCommand(
SuiteData.TestDirPath,
"git",
"push", "--set-upstream", "origin", branchName,
)

countAfterSecondBuild := StagesCount()
if SuiteData.TestImplementation != docker_registry.QuayImplementationName {
Ω(countAfterSecondBuild).Should(Equal(6))
}

utils.RunSucceedCommand(
SuiteData.TestDirPath,
SuiteData.WerfBinPath,
"cleanup",
)

if SuiteData.TestImplementation != docker_registry.QuayImplementationName {
countAfterCleanup := StagesCount()
Ω(countAfterCleanup).Should(Equal(countAfterSecondBuild))
Ω(len(ImportMetadataIDs())).Should(BeEquivalentTo(2))
}
})

It("should remove unused artifact which does not have related import metadata", func() {
utils.RunSucceedCommand(
SuiteData.TestDirPath,
SuiteData.WerfBinPath,
"build",
)

utils.RunSucceedCommand(
SuiteData.TestDirPath,
"git",
"push", "--set-upstream", "origin", branchName,
)

countAfterFirstBuild := StagesCount()
Ω(countAfterFirstBuild).Should(Equal(4))
Ω(len(ImportMetadataIDs())).Should(BeEquivalentTo(1))

RmImportMetadata(ImportMetadataIDs()[0])

utils.RunSucceedCommand(
SuiteData.TestDirPath,
SuiteData.WerfBinPath,
"cleanup",
)

if SuiteData.TestImplementation != docker_registry.QuayImplementationName {
countAfterCleanup := StagesCount()
Ω(countAfterCleanup).Should(Equal(3))
Ω(len(ImportMetadataIDs())).Should(BeEquivalentTo(0))
}
})
})

Context("custom tags", func() {
It("should remove custom tag associated with the deleted stage", func() {
customTag1 := "tag1"
Expand Down
8 changes: 4 additions & 4 deletions pkg/build/build_phase.go
View file
Expand Up @@ -225,10 +225,6 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
img.SetLastNonEmptyStage(phase.StagesIterator.PrevNonEmptyStage)
img.SetContentDigest(phase.StagesIterator.PrevNonEmptyStage.GetContentDigest())

if img.isArtifact {
return nil
}

if err := phase.addManagedImage(ctx, img); err != nil {
return err
}
Expand All @@ -237,6 +233,10 @@ func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *Image) error
return err
}

if img.isArtifact {
return nil
}

if !phase.ShouldBeBuiltMode {
if err := phase.addCustomImageTagsToStagesStorage(ctx, img); err != nil {
return fmt.Errorf("unable to add custom image tags to stages storage: %s", err)
Expand Down
14 changes: 11 additions & 3 deletions pkg/slug/slug.go
View file
Expand Up @@ -16,7 +16,7 @@ var (
DefaultSlugMaxSize = 42 // legacy

dockerTagRegexp = regexp.MustCompile(`^[\w][\w.-]*$`)
dockerTagMaxSize = 128
DockerTagMaxSize = 128

projectNameRegex = regexp.MustCompile(`^(?:[a-z0-9]|[a-z0-9][a-z0-9-]*[a-z0-9])$`)
projectNameMaxSize = 50
Expand Down Expand Up @@ -61,13 +61,21 @@ func validateProject(name string) error {

func DockerTag(name string) string {
if err := ValidateDockerTag(name); err != nil {
return slug(name, dockerTagMaxSize)
return slug(name, DockerTagMaxSize)
}
return name
}

func IsValidDockerTag(name string) bool {
if shouldNotBeSlugged(name, dockerTagRegexp, DockerTagMaxSize) {
return true
}

return false
}

func ValidateDockerTag(name string) error {
if shouldNotBeSlugged(name, dockerTagRegexp, dockerTagMaxSize) {
if IsValidDockerTag(name) {
return nil
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/slug/slug_test.go
View file
Expand Up @@ -78,8 +78,8 @@ func TestDockerTag(t *testing.T) {
},
{
name: "maxSizeExceeded",
data: strings.Repeat("x", dockerTagMaxSize+1),
result: strings.Repeat("x", dockerTagMaxSize-servicePartSize) + "-8cca70eb",
data: strings.Repeat("x", DockerTagMaxSize+1),
result: strings.Repeat("x", DockerTagMaxSize-servicePartSize) + "-8cca70eb",
},
}

Expand All @@ -90,8 +90,8 @@ func TestDockerTag(t *testing.T) {
t.Errorf("\n[EXPECTED]: %s (%d)\n[GOT]: %s (%d)", test.result, len(test.result), result, len(result))
}

if len(result) > dockerTagMaxSize {
t.Errorf("Max size exceeded: [EXPECTED]: %d [GOT]: %d", dockerTagMaxSize, len(result))
if len(result) > DockerTagMaxSize {
t.Errorf("Max size exceeded: [EXPECTED]: %d [GOT]: %d", DockerTagMaxSize, len(result))
}
})

Expand Down

0 comments on commit bcf8205

Please sign in to comment.