feat(cleanup): disable cleanup policies in werf.yaml

Signed-off-by: Alexey Igrychev <alexey.igrychev@flant.com>
werf · Apr 27, 2022 · c293f3d · c293f3d
1 parent eab4f2f
commit c293f3d
Show file tree

Hide file tree

Showing 5 changed files with 65 additions and 36 deletions.
diff --git a/cmd/werf/cleanup/cleanup.go b/cmd/werf/cleanup/cleanup.go
@@ -6,6 +6,7 @@ import (
 
 	"github.com/spf13/cobra"
 
+	"github.com/werf/kubedog/pkg/kube"
 	"github.com/werf/logboek"
 	"github.com/werf/werf/cmd/werf/common"
 	"github.com/werf/werf/pkg/cleaning"
@@ -233,18 +234,24 @@ It is worth noting that auto-cleaning is enabled by default, and manual use is u
 	}
 	logboek.Debug().LogF("Managed images names: %v\n", imagesNames)
 
-	kubernetesContextClients, err := common.GetKubernetesContextClients(&commonCmdData)
-	if err != nil {
-		return fmt.Errorf("unable to get Kubernetes clusters connections: %w", err)
+	var kubernetesContextClients []*kube.ContextClient
+	var kubernetesNamespaceRestrictionByContext map[string]string
+	if !(*commonCmdData.WithoutKube || werfConfig.Meta.Cleanup.DisableKubernetesBasedPolicy) {
+		kubernetesContextClients, err = common.GetKubernetesContextClients(&commonCmdData)
+		if err != nil {
+			return fmt.Errorf("unable to get Kubernetes clusters connections: %w", err)
+		}
+
+		kubernetesNamespaceRestrictionByContext = common.GetKubernetesNamespaceRestrictionByContext(&commonCmdData, kubernetesContextClients)
 	}
 
 	cleanupOptions := cleaning.CleanupOptions{
 		ImageNameList:                           imagesNames,
 		LocalGit:                                giterminismManager.LocalGitRepo().(*git_repo.Local),
 		KubernetesContextClients:                kubernetesContextClients,
-		KubernetesNamespaceRestrictionByContext: common.GetKubernetesNamespaceRestrictionByContext(&commonCmdData, kubernetesContextClients),
+		KubernetesNamespaceRestrictionByContext: kubernetesNamespaceRestrictionByContext,
 		WithoutKube:                             *commonCmdData.WithoutKube,
-		GitHistoryBasedCleanupOptions:           werfConfig.Meta.Cleanup,
+		ConfigMetaCleanup:                       werfConfig.Meta.Cleanup,
 		KeepStagesBuiltWithinLastNHours:         *commonCmdData.KeepStagesBuiltWithinLastNHours,
 		DryRun:                                  *commonCmdData.DryRun,
 	}

diff --git a/docs/_data/werf_yaml.yml b/docs/_data/werf_yaml.yml
@@ -81,9 +81,24 @@ sections:
         collapsible: true
         isCollapsedByDefault: false
         directives:
+          - name: disableKubernetesBasedPolicy
+            value: "bool"
+            description:
+              en: Disable a cleanup policy that allows not to remove images deployed in Kubernetes from the container registry
+              ru: Отключить политику очистки, которая позволяет не удалять запущенные в Kubernetes образы из container registry
+          - name: disableGitHistoryBasedPolicy
+            value: "bool"
+            description:
+              en: Disable a cleanup policy that allows not to remove images taking into account user-defined policies by the Git history (keepPolicies)
+              ru: Отключить политику очистки, которая позволяет не удалять образы с учётом пользовательских политик по истории Git (keepPolicies)
+          - name: disableBuiltWithinLastNHoursPolicy
+            value: "bool"
+            description:
+              en: Disable a cleanup policy that allows not to remove images built in last hours (keepImagesBuiltWithinLastNHours)
+              ru: Отключить политику очистки, которая позволяет не удалять образы собранные в рамках заданного периода времени (keepImagesBuiltWithinLastNHours)
           - name: keepPolicies
             description:
-              en: Set of policies to select relevant images using the git history
+              en: Set of policies to select relevant images using the Git history
               ru: Набор политик для выборки актуальных образов, используя историю Git
             detailsAnchor:
               en: "#configuring-cleanup-policies"

diff --git a/pkg/cleaning/cleanup.go b/pkg/cleaning/cleanup.go
@@ -30,8 +30,8 @@ type CleanupOptions struct {
 	LocalGit                                GitRepo
 	KubernetesContextClients                []*kube.ContextClient
 	KubernetesNamespaceRestrictionByContext map[string]string
-	WithoutKube                             bool
-	GitHistoryBasedCleanupOptions           config.MetaCleanup
+	WithoutKube                             bool // legacy
+	ConfigMetaCleanup                       config.MetaCleanup
 	KeepStagesBuiltWithinLastNHours         uint64
 	DryRun                                  bool
 }
@@ -51,7 +51,7 @@ func newCleanupManager(projectName string, storageManager *manager.StorageManage
 		KubernetesContextClients:                options.KubernetesContextClients,
 		KubernetesNamespaceRestrictionByContext: options.KubernetesNamespaceRestrictionByContext,
 		WithoutKube:                             options.WithoutKube,
-		GitHistoryBasedCleanupOptions:           options.GitHistoryBasedCleanupOptions,
+		ConfigMetaCleanup:                       options.ConfigMetaCleanup,
 		KeepStagesBuiltWithinLastNHours:         options.KeepStagesBuiltWithinLastNHours,
 	}
 }
@@ -68,7 +68,7 @@ type cleanupManager struct {
 	KubernetesContextClients                []*kube.ContextClient
 	KubernetesNamespaceRestrictionByContext map[string]string
 	WithoutKube                             bool
-	GitHistoryBasedCleanupOptions           config.MetaCleanup
+	ConfigMetaCleanup                       config.MetaCleanup
 	KeepStagesBuiltWithinLastNHours         uint64
 	DryRun                                  bool
 }
@@ -117,38 +117,35 @@ func (m *cleanupManager) run(ctx context.Context) error {
 		return err
 	}
 
-	if m.LocalGit != nil {
-		if !m.WithoutKube {
-			if len(m.KubernetesContextClients) == 0 {
-				return fmt.Errorf("no kubernetes configs found to skip images being used in the Kubernetes, pass --without-kube option (or WERF_WITHOUT_KUBE env var) to suppress this error")
-			}
+	if !(m.WithoutKube || m.ConfigMetaCleanup.DisableKubernetesBasedPolicy) {
+		if len(m.KubernetesContextClients) == 0 {
+			return fmt.Errorf("no kubernetes configs found to skip images being used in the Kubernetes, pass --without-kube option (or WERF_WITHOUT_KUBE env var) to suppress this error")
+		}
 
-			deployedDockerImagesNames, err := m.deployedDockerImagesNames(ctx)
-			if err != nil {
-				return fmt.Errorf("error getting deployed docker images names from Kubernetes: %w", err)
-			}
+		deployedDockerImagesNames, err := m.deployedDockerImagesNames(ctx)
+		if err != nil {
+			return fmt.Errorf("error getting deployed docker images names from Kubernetes: %w", err)
+		}
 
-			if err := logboek.Context(ctx).LogProcess("Skipping repo tags that are being used in Kubernetes").DoError(func() error {
-				return m.skipStageIDsThatAreUsedInKubernetes(ctx, deployedDockerImagesNames)
-			}); err != nil {
-				return err
-			}
+		if err := logboek.Context(ctx).LogProcess("Skipping repo tags that are being used in Kubernetes").DoError(func() error {
+			return m.skipStageIDsThatAreUsedInKubernetes(ctx, deployedDockerImagesNames)
+		}); err != nil {
+			return err
+		}
 
-			if err := logboek.Context(ctx).LogProcess("Skipping final repo tags that are being used in Kubernetes").DoError(func() error {
-				return m.skipFinalStageIDsThatAreUsedInKubernetes(ctx, deployedDockerImagesNames)
-			}); err != nil {
-				return err
-			}
+		if err := logboek.Context(ctx).LogProcess("Skipping final repo tags that are being used in Kubernetes").DoError(func() error {
+			return m.skipFinalStageIDsThatAreUsedInKubernetes(ctx, deployedDockerImagesNames)
+		}); err != nil {
+			return err
 		}
+	}
 
+	if !m.ConfigMetaCleanup.DisableGitHistoryBasedPolicy {
 		if err := logboek.Context(ctx).LogProcess("Git history-based cleanup").DoError(func() error {
 			return m.gitHistoryBasedCleanup(ctx)
 		}); err != nil {
 			return err
 		}
-	} else {
-		logboek.Context(ctx).Warn().LogLn("WARNING: Git history-based cleanup skipped due to local git repository was not detected")
-		logboek.Context(ctx).Default().LogOptionalLn()
 	}
 
 	if err := logboek.Context(ctx).LogProcess("Cleanup unused stages").DoError(func() error {
@@ -263,7 +260,7 @@ func (m *cleanupManager) gitHistoryBasedCleanup(ctx context.Context) error {
 
 	var referencesToScan []*git_history_based_cleanup.ReferenceToScan
 	if err := logboek.Context(ctx).Default().LogProcess("Preparing references to scan").DoError(func() error {
-		referencesToScan, err = git_history_based_cleanup.ReferencesToScan(ctx, gitRepository, m.GitHistoryBasedCleanupOptions.KeepPolicies)
+		referencesToScan, err = git_history_based_cleanup.ReferencesToScan(ctx, gitRepository, m.ConfigMetaCleanup.KeepPolicies)
 		return err
 	}); err != nil {
 		return err
@@ -610,8 +607,8 @@ func (m *cleanupManager) cleanupUnusedStages(ctx context.Context) error {
 		return fmt.Errorf("unable to init imports metadata: %w", err)
 	}
 
+	// skip stages and their relatives covered by Kubernetes- or git history-based cleanup policies
 	stageDescriptionListToDelete := stageDescriptionList
-	// skip stages and their relatives based on deployed images in k8s and git history based cleanup policies
 	{
 		var excludedSDList []*image.StageDescription
 		for _, sd := range m.stageManager.GetProtectedStageDescriptionList() {

diff --git a/pkg/config/meta_cleanup.go b/pkg/config/meta_cleanup.go
@@ -8,7 +8,10 @@ import (
 )
 
 type MetaCleanup struct {
-	KeepPolicies []*MetaCleanupKeepPolicy
+	DisableKubernetesBasedPolicy       bool
+	DisableGitHistoryBasedPolicy       bool
+	DisableBuiltWithinLastNHoursPolicy bool
+	KeepPolicies                       []*MetaCleanupKeepPolicy
 }
 
 type MetaCleanupKeepPolicy struct {

diff --git a/pkg/config/raw_meta_cleanup.go b/pkg/config/raw_meta_cleanup.go
@@ -8,7 +8,10 @@ import (
 )
 
 type rawMetaCleanup struct {
-	KeepPolicies []*rawMetaCleanupKeepPolicy `yaml:"keepPolicies,omitempty"`
+	DisableKubernetesBasedPolicy       bool                        `yaml:"disableKubernetesBasedPolicy,omitempty"`
+	DisableGitHistoryBasedPolicy       bool                        `yaml:"disableGitHistoryBasedPolicy,omitempty"`
+	DisableBuiltWithinLastNHoursPolicy bool                        `yaml:"disableBuiltWithinLastNHoursPolicy,omitempty"`
+	KeepPolicies                       []*rawMetaCleanupKeepPolicy `yaml:"keepPolicies,omitempty"`
 
 	rawMeta               *rawMeta
 	UnsupportedAttributes map[string]interface{} `yaml:",inline"`
@@ -181,6 +184,10 @@ func (c *rawMetaCleanupKeepPolicyReferences) processRegexpString(name, configVal
 func (c *rawMetaCleanup) toMetaCleanup() MetaCleanup {
 	metaCleanup := MetaCleanup{}
 
+	metaCleanup.DisableKubernetesBasedPolicy = c.DisableKubernetesBasedPolicy
+	metaCleanup.DisableBuiltWithinLastNHoursPolicy = c.DisableBuiltWithinLastNHoursPolicy
+	metaCleanup.DisableGitHistoryBasedPolicy = c.DisableGitHistoryBasedPolicy
+
 	for _, policy := range c.KeepPolicies {
 		metaCleanup.KeepPolicies = append(metaCleanup.KeepPolicies, policy.toMetaCleanupKeepPolicy())
 	}