cleanup.go

package cleaning

import (
	"context"
	"fmt"
	"strings"
	"sync"
	"time"

	"github.com/go-git/go-git/v5"
	"github.com/gookit/color"
	"github.com/rodaine/table"

	"github.com/werf/kubedog/pkg/kube"
	"github.com/werf/logboek"
	"github.com/werf/werf/pkg/cleaning/allow_list"
	"github.com/werf/werf/pkg/cleaning/git_history_based_cleanup"
	"github.com/werf/werf/pkg/cleaning/stage_manager"
	"github.com/werf/werf/pkg/config"
	"github.com/werf/werf/pkg/docker_registry"
	"github.com/werf/werf/pkg/image"
	"github.com/werf/werf/pkg/logging"
	"github.com/werf/werf/pkg/storage"
	"github.com/werf/werf/pkg/storage/manager"
	"github.com/werf/werf/pkg/util"
	"github.com/werf/werf/pkg/werf/global_warnings"
)

type CleanupOptions struct {
	ImageNameList                           []string
	LocalGit                                GitRepo
	KubernetesContextClients                []*kube.ContextClient
	KubernetesNamespaceRestrictionByContext map[string]string
	WithoutKube                             bool
	GitHistoryBasedCleanupOptions           config.MetaCleanup
	KeepStagesBuiltWithinLastNHours         uint64
	DryRun                                  bool
}

func Cleanup(ctx context.Context, projectName string, storageManager *manager.StorageManager, options CleanupOptions) error {
	return newCleanupManager(projectName, storageManager, options).run(ctx)
}

func newCleanupManager(projectName string, storageManager *manager.StorageManager, options CleanupOptions) *cleanupManager {
	return &cleanupManager{
		stageManager:                            stage_manager.NewManager(),
		ProjectName:                             projectName,
		StorageManager:                          storageManager,
		ImageNameList:                           options.ImageNameList,
		DryRun:                                  options.DryRun,
		LocalGit:                                options.LocalGit,
		KubernetesContextClients:                options.KubernetesContextClients,
		KubernetesNamespaceRestrictionByContext: options.KubernetesNamespaceRestrictionByContext,
		WithoutKube:                             options.WithoutKube,
		GitHistoryBasedCleanupOptions:           options.GitHistoryBasedCleanupOptions,
		KeepStagesBuiltWithinLastNHours:         options.KeepStagesBuiltWithinLastNHours,
	}
}

type cleanupManager struct {
	stageManager stage_manager.Manager

	nonexistentImportMetadataIDs []string

	ProjectName                             string
	StorageManager                          manager.StorageManagerInterface
	ImageNameList                           []string
	LocalGit                                GitRepo
	KubernetesContextClients                []*kube.ContextClient
	KubernetesNamespaceRestrictionByContext map[string]string
	WithoutKube                             bool
	GitHistoryBasedCleanupOptions           config.MetaCleanup
	KeepStagesBuiltWithinLastNHours         uint64
	DryRun                                  bool
}

type GitRepo interface {
	PlainOpen() (*git.Repository, error)
	IsCommitExists(ctx context.Context, commit string) (bool, error)
}

func (m *cleanupManager) init(ctx context.Context) error {
	if err := logboek.Context(ctx).Info().LogProcess("Fetching manifests").DoError(func() error {
		return m.stageManager.InitStages(ctx, m.StorageManager)
	}); err != nil {
		return err
	}

	if m.StorageManager.GetFinalStagesStorage() != nil {
		if err := logboek.Context(ctx).Info().LogProcess("Fetching final repo manifests").DoError(func() error {
			return m.stageManager.InitFinalStages(ctx, m.StorageManager)
		}); err != nil {
			return err
		}
	}

	if err := logboek.Context(ctx).Info().LogProcess("Fetching metadata").DoError(func() error {
		if err := m.stageManager.InitImagesMetadata(ctx, m.StorageManager, m.LocalGit, m.ProjectName, m.ImageNameList); err != nil {
			return fmt.Errorf("unable to init images metadata: %s", err)
		}

		if err := m.stageManager.InitCustomTagsMetadata(ctx, m.StorageManager); err != nil {
			return fmt.Errorf("unable to init custom tags metadata: %s", err)
		}

		return nil
	}); err != nil {
		return err
	}

	return nil
}

func (m *cleanupManager) run(ctx context.Context) error {
	if err := logboek.Context(ctx).LogProcess("Fetching manifests and metadata").DoError(func() error {
		return m.init(ctx)
	}); err != nil {
		return err
	}

	if m.LocalGit != nil {
		if !m.WithoutKube {
			if len(m.KubernetesContextClients) == 0 {
				global_warnings.GlobalWarningLn(ctx, "No kubernetes configs found to skip images being used in the Kubernetes, pass --without-kube option (or WERF_WITHOUT_KUBE env var) to silence this warning")
			}

			deployedDockerImagesNames, err := m.deployedDockerImagesNames(ctx)
			if err != nil {
				return fmt.Errorf("error getting deployed docker images names from Kubernetes: %s", err)
			}

			if err := logboek.Context(ctx).LogProcess("Skipping repo tags that are being used in Kubernetes").DoError(func() error {
				return m.skipStageIDsThatAreUsedInKubernetes(ctx, deployedDockerImagesNames)
			}); err != nil {
				return err
			}

			if err := logboek.Context(ctx).LogProcess("Skipping final repo tags that are being used in Kubernetes").DoError(func() error {
				return m.skipFinalStageIDsThatAreUsedInKubernetes(ctx, deployedDockerImagesNames)
			}); err != nil {
				return err
			}
		}

		if err := logboek.Context(ctx).LogProcess("Git history-based cleanup").DoError(func() error {
			return m.gitHistoryBasedCleanup(ctx)
		}); err != nil {
			return err
		}
	} else {
		logboek.Context(ctx).Warn().LogLn("WARNING: Git history-based cleanup skipped due to local git repository was not detected")
		logboek.Context(ctx).Default().LogOptionalLn()
	}

	if err := logboek.Context(ctx).LogProcess("Cleanup unused stages").DoError(func() error {
		return m.cleanupUnusedStages(ctx)
	}); err != nil {
		return err
	}

	if m.StorageManager.GetFinalStagesStorage() != nil {
		if err := logboek.Context(ctx).LogProcess("Cleanup final stages").DoError(func() error {
			return m.cleanupFinalStages(ctx)
		}); err != nil {
			return err
		}
	}

	return nil
}

func (m *cleanupManager) skipStageIDsThatAreUsedInKubernetes(ctx context.Context, deployedDockerImagesNames []string) error {
	handledDeployedStages := map[string]bool{}
	handleTagFunc := func(tag, stageID string) {
		dockerImageName := fmt.Sprintf("%s:%s", m.StorageManager.GetStagesStorage().Address(), tag)
		for _, deployedDockerImageName := range deployedDockerImagesNames {
			if deployedDockerImageName == dockerImageName {
				if !handledDeployedStages[stageID] {
					m.stageManager.MarkStageAsProtected(stageID)

					logboek.Context(ctx).Default().LogFDetails("  tag: %s\n", tag)
					logboek.Context(ctx).LogOptionalLn()
					handledDeployedStages[stageID] = true
				}

				break
			}
		}
	}

	for _, stageID := range m.stageManager.GetStageIDList() {
		handleTagFunc(stageID, stageID)
	}

	for stageID, customTagList := range m.stageManager.GetCustomTagsMetadata() {
		for _, customTag := range customTagList {
			handleTagFunc(customTag, stageID)
		}
	}

	return nil
}

func (m *cleanupManager) skipFinalStageIDsThatAreUsedInKubernetes(ctx context.Context, deployedDockerImagesNames []string) error {
	handledDeployedFinalStages := map[string]bool{}
Loop:
	for _, stageID := range m.stageManager.GetFinalStageIDList() {
		dockerImageName := fmt.Sprintf("%s:%s", m.StorageManager.GetFinalStagesStorage().Address(), stageID)

		for _, deployedDockerImageName := range deployedDockerImagesNames {
			if deployedDockerImageName == dockerImageName {
				if !handledDeployedFinalStages[stageID] {
					m.stageManager.MarkFinalStageAsProtected(stageID)

					logboek.Context(ctx).Default().LogFDetails("  tag: %s\n", stageID)
					logboek.Context(ctx).LogOptionalLn()
					handledDeployedFinalStages[stageID] = true
				}

				continue Loop
			}
		}
	}

	return nil
}

func (m *cleanupManager) deployedDockerImagesNames(ctx context.Context) ([]string, error) {
	var deployedDockerImagesNames []string
	for _, contextClient := range m.KubernetesContextClients {
		if err := logboek.Context(ctx).LogProcessInline("Getting deployed docker images (context %s)", contextClient.ContextName).
			DoError(func() error {
				kubernetesClientDeployedDockerImagesNames, err := allow_list.DeployedDockerImages(contextClient.Client, m.KubernetesNamespaceRestrictionByContext[contextClient.ContextName])
				if err != nil {
					return fmt.Errorf("cannot get deployed imagesStageList: %s", err)
				}

				deployedDockerImagesNames = append(deployedDockerImagesNames, kubernetesClientDeployedDockerImagesNames...)

				return nil
			}); err != nil {
			return nil, err
		}
	}

	return deployedDockerImagesNames, nil
}

func (m *cleanupManager) gitHistoryBasedCleanup(ctx context.Context) error {
	gitRepository, err := m.LocalGit.PlainOpen()
	if err != nil {
		return fmt.Errorf("git plain open failed: %s", err)
	}

	var referencesToScan []*git_history_based_cleanup.ReferenceToScan
	if err := logboek.Context(ctx).Default().LogProcess("Preparing references to scan").DoError(func() error {
		referencesToScan, err = git_history_based_cleanup.ReferencesToScan(ctx, gitRepository, m.GitHistoryBasedCleanupOptions.KeepPolicies)
		return err
	}); err != nil {
		return err
	}

	for imageName, stageIDCommitList := range m.stageManager.GetImageStageIDCommitListToCleanup() {
		var reachedStageIDs []string
		var hitStageIDCommitList map[string][]string
		if err := logboek.Context(ctx).LogProcess(logging.ImageLogProcessName(imageName, false)).DoError(func() error {
			if logboek.Context(ctx).Streams().Width() > 120 {
				m.printStageIDCommitListTable(ctx, imageName)
				m.printStageIDCustomTagListTable(ctx)
			}

			if err := logboek.Context(ctx).LogProcess("Scanning git references history").DoError(func() error {
				if countStageIDCommitList(stageIDCommitList) != 0 {
					reachedStageIDs, hitStageIDCommitList, err = git_history_based_cleanup.ScanReferencesHistory(ctx, gitRepository, referencesToScan, stageIDCommitList)
				} else {
					logboek.Context(ctx).LogLn("Scanning stopped due to nothing to seek")
				}

				return nil
			}); err != nil {
				return err
			}

			var stageIDToUnlink []string
		outerLoop:
			for stageID := range stageIDCommitList {
				for _, reachedStageID := range reachedStageIDs {
					if stageID == reachedStageID {
						continue outerLoop
					}
				}

				stageIDToUnlink = append(stageIDToUnlink, stageID)
			}

			if len(reachedStageIDs) != 0 {
				m.handleSavedStageIDs(ctx, reachedStageIDs)
			}

			if err := logboek.Context(ctx).LogProcess("Cleaning image metadata").DoError(func() error {
				return m.cleanupImageMetadata(ctx, imageName, hitStageIDCommitList, stageIDToUnlink)
			}); err != nil {
				return err
			}

			return nil
		}); err != nil {
			return err
		}
	}

	if err := m.cleanupNonexistentImageMetadata(ctx); err != nil {
		return fmt.Errorf("ubable to cleanup nonexistent image metadata: %s", err)
	}

	return nil
}

func (m *cleanupManager) printStageIDCommitListTable(ctx context.Context, imageName string) {
	stageIDCommitList := m.stageManager.GetStageIDCommitListToCleanup(imageName)
	if countStageIDCommitList(stageIDCommitList) == 0 {
		return
	}

	rows := m.prepareStageIDTableRows(ctx, stageIDCommitList)
	if len(rows) != 0 {
		tbl := table.New("Tag", "Commits")
		tbl.WithWriter(logboek.Context(ctx).OutStream())
		tbl.WithHeaderFormatter(func(format string, a ...interface{}) string {
			return logboek.ColorizeF(color.New(color.OpUnderscore), format, a...)
		})
		for _, row := range rows {
			tbl.AddRow(row...)
		}
		tbl.Print()

		logboek.Context(ctx).LogOptionalLn()
	}
}

func (m *cleanupManager) printStageIDCustomTagListTable(ctx context.Context) {
	stageIDCustomTagList := m.stageManager.GetCustomTagsMetadata()
	if len(stageIDCustomTagList) == 0 {
		return
	}

	rows := m.prepareStageIDTableRows(ctx, stageIDCustomTagList)
	if len(rows) != 0 {
		tbl := table.New("Tag", "Custom tags")
		tbl.WithWriter(logboek.Context(ctx).OutStream())
		tbl.WithHeaderFormatter(func(format string, a ...interface{}) string {
			return logboek.ColorizeF(color.New(color.OpUnderscore), format, a...)
		})
		for _, row := range rows {
			tbl.AddRow(row...)
		}
		tbl.Print()

		logboek.Context(ctx).LogOptionalLn()
	}
}

func (m *cleanupManager) prepareStageIDTableRows(ctx context.Context, stageIDAnyList map[string][]string) [][]interface{} {
	var rows [][]interface{}
	for stageID, anyList := range stageIDAnyList {
		for ind, any := range anyList {
			var columns []interface{}
			if ind == 0 {
				stageIDColumn := stageID

				space := len(stageID) - len(any) - 1
				if logboek.Context(ctx).Streams().ContentWidth() < space {
					stageIDColumn = fmt.Sprintf("%s..%s", stageID[:space-5], stageID[space-3:])
				}

				columns = append(columns, stageIDColumn)
			} else {
				columns = append(columns, "")
			}

			columns = append(columns, any)
			rows = append(rows, columns)
		}
	}

	return rows
}

func (m *cleanupManager) handleSavedStageIDs(ctx context.Context, savedStageIDs []string) {
	logboek.Context(ctx).Default().LogBlock("Saved tags").Do(func() {
		for _, stageID := range savedStageIDs {
			m.stageManager.MarkStageAsProtected(stageID)
			logboek.Context(ctx).Default().LogFDetails("  tag: %s\n", stageID)
			logboek.Context(ctx).LogOptionalLn()
		}
	})
}

func (m *cleanupManager) deleteStages(ctx context.Context, stages []*image.StageDescription, isFinal bool) error {
	deleteStageOptions := manager.ForEachDeleteStageOptions{
		DeleteImageOptions: storage.DeleteImageOptions{
			RmiForce: false,
		},
		FilterStagesAndProcessRelatedDataOptions: storage.FilterStagesAndProcessRelatedDataOptions{
			SkipUsedImage:            true,
			RmForce:                  false,
			RmContainersThatUseImage: false,
		},
	}

	return deleteStages(ctx, m.StorageManager, m.DryRun, deleteStageOptions, stages, isFinal)
}

func deleteStages(ctx context.Context, storageManager manager.StorageManagerInterface, dryRun bool, deleteStageOptions manager.ForEachDeleteStageOptions, stages []*image.StageDescription, isFinal bool) error {
	if dryRun {
		for _, stageDesc := range stages {
			logboek.Context(ctx).Default().LogFDetails("  tag: %s\n", stageDesc.Info.Tag)
			logboek.Context(ctx).LogOptionalLn()
		}
		return nil
	}

	onDeleteFunc := func(ctx context.Context, stageDesc *image.StageDescription, err error) error {
		if err != nil {
			if err := handleDeletionError(err); err != nil {
				return err
			}

			logboek.Context(ctx).Warn().LogF("WARNING: Image %s deletion failed: %s\n", stageDesc.Info.Name, err)

			return nil
		}

		logboek.Context(ctx).Default().LogFDetails("  tag: %s\n", stageDesc.Info.Tag)

		return nil
	}

	if isFinal {
		return storageManager.ForEachDeleteFinalStage(ctx, deleteStageOptions, stages, onDeleteFunc)
	}

	return storageManager.ForEachDeleteStage(ctx, deleteStageOptions, stages, onDeleteFunc)
}

func (m *cleanupManager) cleanupImageMetadata(ctx context.Context, imageName string, hitStageIDCommitList map[string][]string, stageIDsToUnlink []string) error {
	if countStageIDCommitList(hitStageIDCommitList) != 0 || len(stageIDsToUnlink) != 0 {
		stageIDCommitListToDelete := map[string][]string{}
		stageIDCommitListToCleanup := m.stageManager.GetStageIDCommitListToCleanup(imageName)
	stageIDCommitListLoop:
		for stageID, commitList := range stageIDCommitListToCleanup {
			commitListToCheck := commitList
			for _, stageIDToUnlink := range stageIDsToUnlink {
				if stageIDToUnlink == stageID {
					stageIDCommitListToDelete[stageID] = commitListToCheck
					continue stageIDCommitListLoop
				}
			}

			hitCommitList, ok := hitStageIDCommitList[stageID]
			if ok {
				commitListToCheck = util.ExcludeFromStringArray(commitListToCheck, hitCommitList...)
			}

			stageIDCommitListToDelete[stageID] = commitListToCheck
		}

		if countStageIDCommitList(stageIDCommitListToDelete) != 0 {
			header := fmt.Sprintf("Cleaning up metadata (%d/%d)", countStageIDCommitList(stageIDCommitListToDelete), countStageIDCommitList(stageIDCommitListToCleanup))

			logProcessDoError := logboek.Context(ctx).Default().LogProcess(header).DoError
			if logboek.Context(ctx).Info().IsAccepted() {
				logProcessDoError = logboek.Context(ctx).Info().LogProcess(header).DoError
			}

			if err := logProcessDoError(func() error {
				return m.deleteImageMetadata(ctx, imageName, stageIDCommitListToDelete)
			}); err != nil {
				return err
			}
		}
	}

	nonexistentStageIDCommitList := m.stageManager.GetNonexistentStageIDCommitList(imageName)
	if countStageIDCommitList(nonexistentStageIDCommitList) != 0 {
		header := fmt.Sprintf("Deleting metadata for nonexistent stageIDs (%d)", countStageIDCommitList(nonexistentStageIDCommitList))

		logProcessDoError := logboek.Context(ctx).Default().LogProcessInline(header).DoError
		if logboek.Context(ctx).Info().IsAccepted() {
			logProcessDoError = logboek.Context(ctx).Info().LogProcess(header).DoError
		}

		if err := logProcessDoError(func() error {
			return m.deleteImageMetadata(ctx, imageName, nonexistentStageIDCommitList)
		}); err != nil {
			return err
		}
	}

	stageIDNonexistentCommitList := m.stageManager.GetStageIDNonexistentCommitList(imageName)
	if countStageIDCommitList(stageIDNonexistentCommitList) != 0 {
		header := fmt.Sprintf("Deleting metadata for nonexistent commits (%d)", countStageIDCommitList(stageIDNonexistentCommitList))

		logProcessDoError := logboek.Context(ctx).Default().LogProcessInline(header).DoError
		if logboek.Context(ctx).Info().IsAccepted() {
			logProcessDoError = logboek.Context(ctx).Info().LogProcess(header).DoError
		}

		if err := logProcessDoError(func() error {
			return m.deleteImageMetadata(ctx, imageName, stageIDNonexistentCommitList)
		}); err != nil {
			return err
		}
	}

	return nil
}

func countStageIDCommitList(stageIDCommitList map[string][]string) int {
	var result int
	for _, commitList := range stageIDCommitList {
		for range commitList {
			result++
		}
	}

	return result
}

func (m *cleanupManager) cleanupNonexistentImageMetadata(ctx context.Context) error {
	var counter int
	stageIDCommitListByNonexistentImage := m.stageManager.GetStageIDCommitListByNonexistentImage()
	for _, stageIDCommitList := range stageIDCommitListByNonexistentImage {
		counter += countStageIDCommitList(stageIDCommitList)
	}

	if counter == 0 {
		return nil
	}

	return logboek.Context(ctx).Default().LogProcess("Deleting metadata for nonexistent images (%d)", counter).DoError(func() error {
		for imageName, stageIDCommitList := range stageIDCommitListByNonexistentImage {
			if err := m.deleteImageMetadata(ctx, imageName, stageIDCommitList); err != nil {
				return err
			}
		}

		return nil
	})
}

func (m *cleanupManager) deleteImageMetadata(ctx context.Context, imageName string, stageIDCommitList map[string][]string) error {
	if err := deleteImageMetadata(ctx, m.ProjectName, m.StorageManager, imageName, stageIDCommitList, m.DryRun); err != nil {
		return err
	}

	return nil
}

func deleteImageMetadata(ctx context.Context, projectName string, storageManager manager.StorageManagerInterface, imageNameOrID string, stageIDCommitList map[string][]string, dryRun bool) error {
	if dryRun {
		for stageID, commitList := range stageIDCommitList {
			if len(commitList) == 0 {
				continue
			}

			logboek.Context(ctx).Info().LogFDetails("  imageName: %s\n", imageNameOrID)
			logboek.Context(ctx).Info().LogFDetails("  stageID: %s\n", stageID)
			logboek.Context(ctx).Info().LogFDetails("  commits: %d\n", len(commitList))
			logboek.Context(ctx).Info().LogOptionalLn()
		}
		return nil
	}

	return storageManager.ForEachRmImageMetadata(ctx, projectName, imageNameOrID, stageIDCommitList, func(ctx context.Context, commit, stageID string, err error) error {
		if err != nil {
			if err := handleDeletionError(err); err != nil {
				return err
			}

			logboek.Context(ctx).Warn().LogF("WARNING: Image metadata %q commit %s stage ID %s deletion failed: %s\n", imageNameOrID, commit, stageID, err)

			return nil
		}

		logboek.Context(ctx).Info().LogFDetails("  imageName: %s\n", imageNameOrID)
		logboek.Context(ctx).Info().LogFDetails("  stageID: %s\n", stageID)
		logboek.Context(ctx).Info().LogFDetails("  commit: %s\n", commit)

		return nil
	})
}

func (m *cleanupManager) cleanupUnusedStages(ctx context.Context) error {
	stageDescriptionList := m.stageManager.GetStageDescriptionList(stage_manager.StageDescriptionListOptions{})
	stageDescriptionListCount := len(stageDescriptionList)

	if err := logboek.Context(ctx).Info().LogProcess("Fetching imports metadata").DoError(func() error {
		return m.initImportsMetadata(ctx, stageDescriptionList)
	}); err != nil {
		return fmt.Errorf("unable to init imports metadata: %s", err)
	}

	stageDescriptionListToDelete := stageDescriptionList
	// skip stages and their relatives based on deployed images in k8s and git history based cleanup policies
	{
		var excludedSDList []*image.StageDescription
		for _, sd := range m.stageManager.GetProtectedStageDescriptionList() {
			var excludedSDListBySD []*image.StageDescription
			stageDescriptionListToDelete, excludedSDListBySD = m.excludeStageAndRelativesByImageID(stageDescriptionListToDelete, sd.Info.ID)
			excludedSDList = append(excludedSDList, excludedSDListBySD...)
		}

		logboek.Context(ctx).Default().LogBlock("Saved stages (%d/%d)", len(excludedSDList), len(stageDescriptionList)).Do(func() {
			for _, excludedSD := range excludedSDList {
				logboek.Context(ctx).Default().LogFDetails("  tag: %s\n", excludedSD.Info.Tag)
				logboek.Context(ctx).Default().LogOptionalLn()
			}
		})
	}

	// skip stages and their relatives based on KeepStagesBuiltWithinLastNHours policy
	{
		if m.KeepStagesBuiltWithinLastNHours != 0 {
			var excludedSDList []*image.StageDescription
			for _, sd := range stageDescriptionListToDelete {
				if (time.Since(sd.Info.GetCreatedAt()).Hours()) <= float64(m.KeepStagesBuiltWithinLastNHours) {
					var excludedRelativesSDList []*image.StageDescription
					stageDescriptionListToDelete, excludedRelativesSDList = m.excludeStageAndRelativesByImageID(stageDescriptionListToDelete, sd.Info.ID)
					excludedSDList = append(excludedSDList, excludedRelativesSDList...)
				}
			}

			if len(excludedSDList) != 0 {
				logboek.Context(ctx).Default().LogBlock("Saved stages that were built within last %d hours (%d/%d)", m.KeepStagesBuiltWithinLastNHours, len(excludedSDList), len(stageDescriptionList)).Do(func() {
					for _, stage := range excludedSDList {
						logboek.Context(ctx).Default().LogFDetails("  tag: %s\n", stage.Info.Tag)
						logboek.Context(ctx).LogOptionalLn()
					}
				})
			}
		}
	}

	if len(stageDescriptionListToDelete) != 0 {
		if err := logboek.Context(ctx).Default().LogProcess("Deleting stages tags (%d/%d)", len(stageDescriptionListToDelete), stageDescriptionListCount).DoError(func() error {
			return m.deleteStages(ctx, stageDescriptionListToDelete, false)
		}); err != nil {
			return err
		}

		m.stageManager.ForgetDeletedStages(stageDescriptionListToDelete)
	}

	if err := m.deleteUnusedCustomTags(ctx); err != nil {
		return fmt.Errorf("unable to cleanup custom tags metadata: %s", err)
	}

	if len(m.nonexistentImportMetadataIDs) != 0 {
		if err := logboek.Context(ctx).Default().LogProcess("Cleaning imports metadata (%d)", len(m.nonexistentImportMetadataIDs)).DoError(func() error {
			return m.deleteImportsMetadata(ctx, m.nonexistentImportMetadataIDs)
		}); err != nil {
			return err
		}
	}

	return nil
}

func (m *cleanupManager) cleanupFinalStages(ctx context.Context) error {
	finalStagesDescriptionListFull := m.stageManager.GetFinalStageDescriptionList(stage_manager.StageDescriptionListOptions{})
	finalStageDescriptionListFullCount := len(finalStagesDescriptionListFull)

	finalStagesDescriptionList := m.stageManager.GetFinalStageDescriptionList(stage_manager.StageDescriptionListOptions{ExcludeProtected: true})
	stagesDescriptionList := m.stageManager.GetStageDescriptionList(stage_manager.StageDescriptionListOptions{})

	var finalStagesDescriptionListToDelete []*image.StageDescription

FilterOutFinalStages:
	for _, finalStg := range finalStagesDescriptionList {
		for _, stg := range stagesDescriptionList {
			if stg.StageID.IsEqual(*finalStg.StageID) {
				continue FilterOutFinalStages
			}
		}

		finalStagesDescriptionListToDelete = append(finalStagesDescriptionListToDelete, finalStg)
	}

	if len(finalStagesDescriptionListToDelete) != 0 {
		if err := logboek.Context(ctx).Default().LogProcess("Deleting final stages tags (%d/%d)", len(finalStagesDescriptionListToDelete), finalStageDescriptionListFullCount).DoError(func() error {
			return m.deleteStages(ctx, finalStagesDescriptionListToDelete, true)
		}); err != nil {
			return err
		}

		m.stageManager.ForgetDeletedFinalStages(finalStagesDescriptionListToDelete)
	}

	return nil
}

func (m *cleanupManager) initImportsMetadata(ctx context.Context, stageDescriptionList []*image.StageDescription) error {
	importMetadataIDs, err := m.StorageManager.GetStagesStorage().GetImportMetadataIDs(ctx, m.ProjectName, storage.WithCache())
	if err != nil {
		return err
	}

	var mutex sync.Mutex
	return m.StorageManager.ForEachGetImportMetadata(ctx, m.ProjectName, importMetadataIDs, func(ctx context.Context, metadataID string, metadata *storage.ImportMetadata, err error) error {
		if err != nil {
			return err
		}

		if metadata == nil {
			if err := logboek.Context(ctx).Warn().LogProcess("Deleting invalid import metadata %s", metadataID).
				DoError(func() error {
					return m.deleteImportsMetadata(ctx, []string{metadataID})
				}); err != nil {
				return fmt.Errorf("unable to delete import metadata %s: %s", metadataID, err)
			}

			return nil
		}

		mutex.Lock()
		defer mutex.Unlock()

		sourceImageID := metadata.SourceImageID
		importSourceID := metadata.ImportSourceID
		stage := findStageByImageID(stageDescriptionList, sourceImageID)
		if stage == nil {
			m.nonexistentImportMetadataIDs = append(m.nonexistentImportMetadataIDs, importSourceID)
		}

		return nil
	})
}

func (m *cleanupManager) deleteImportsMetadata(ctx context.Context, importMetadataIDs []string) error {
	return deleteImportsMetadata(ctx, m.ProjectName, m.StorageManager, importMetadataIDs, m.DryRun)
}

func deleteImportsMetadata(ctx context.Context, projectName string, storageManager manager.StorageManagerInterface, importMetadataIDs []string, dryRun bool) error {
	if dryRun {
		for _, importMetadataID := range importMetadataIDs {
			logboek.Context(ctx).Info().LogFDetails("  importMetadataID: %s\n", importMetadataID)
			logboek.Context(ctx).Info().LogOptionalLn()
		}
		return nil
	}

	return storageManager.ForEachRmImportMetadata(ctx, projectName, importMetadataIDs, func(ctx context.Context, importMetadataID string, err error) error {
		if err != nil {
			if err := handleDeletionError(err); err != nil {
				return err
			}

			logboek.Context(ctx).Warn().LogF("WARNING: Import metadata ID %s deletion failed: %s\n", importMetadataID, err)

			return nil
		}

		logboek.Context(ctx).Info().LogFDetails("  importMetadataID: %s\n", importMetadataID)

		return nil
	})
}

func (m *cleanupManager) excludeStageAndRelativesByImageID(stages []*image.StageDescription, imageID string) ([]*image.StageDescription, []*image.StageDescription) {
	stage := findStageByImageID(stages, imageID)
	if stage == nil {
		return stages, []*image.StageDescription{}
	}

	return m.excludeStageAndRelativesByStage(stages, stage)
}

func findStageByImageID(stages []*image.StageDescription, imageID string) *image.StageDescription {
	for _, stage := range stages {
		if stage.Info.ID == imageID {
			return stage
		}
	}

	return nil
}

func (m *cleanupManager) excludeStageAndRelativesByStage(stages []*image.StageDescription, stage *image.StageDescription) ([]*image.StageDescription, []*image.StageDescription) {
	var excludedStages []*image.StageDescription
	currentStage := stage
	for {
		stages = excludeStages(stages, currentStage)
		excludedStages = append(excludedStages, currentStage)

		currentStage = findStageByImageID(stages, currentStage.Info.ParentID)
		if currentStage == nil {
			break
		}
	}

	return stages, excludedStages
}

func (m *cleanupManager) deleteUnusedCustomTags(ctx context.Context) error {
	stageIDCustomTagList := m.stageManager.GetCustomTagsMetadata()
	if len(stageIDCustomTagList) == 0 {
		return nil
	}

	var customTagListToDelete []string
	var customTagListToKeep []string
	var numberOfCustomTags int
	for stageID, customTagList := range stageIDCustomTagList {
		numberOfCustomTags += len(customTagList)
		if !m.stageManager.IsStageExist(stageID) {
			customTagListToDelete = append(customTagListToDelete, customTagList...)
		} else {
			customTagListToKeep = append(customTagListToKeep, customTagList...)
		}
	}

	if len(customTagListToKeep) != 0 {
		header := fmt.Sprintf("Saved custom tags (%d/%d)", len(customTagListToKeep), numberOfCustomTags)
		logboek.Context(ctx).Default().LogBlock(header).Do(func() {
			for _, customTag := range customTagListToKeep {
				logboek.Context(ctx).Default().LogFDetails("  tag: %s\n", customTag)
				logboek.Context(ctx).LogOptionalLn()
			}
		})
	}

	if len(customTagListToDelete) != 0 {
		header := fmt.Sprintf("Deleting unused custom tags (%d/%d)", len(customTagListToDelete), numberOfCustomTags)
		if err := logboek.Context(ctx).LogProcess(header).DoError(func() error {
			for _, customTag := range customTagListToDelete {
				if err := deleteCustomTag(ctx, m.StorageManager.GetStagesStorage(), customTag); err != nil {
					return err
				}
			}

			return nil
		}); err != nil {
			return err
		}
	}

	return nil
}

func excludeStages(stages []*image.StageDescription, stagesToExclude ...*image.StageDescription) []*image.StageDescription {
	var updatedStageList []*image.StageDescription

loop:
	for _, stage := range stages {
		for _, stageToExclude := range stagesToExclude {
			if stage == stageToExclude {
				continue loop
			}
		}

		updatedStageList = append(updatedStageList, stage)
	}

	return updatedStageList
}

func handleDeletionError(err error) error {
	switch {
	case docker_registry.IsDockerHubUnauthorizedErr(err):
		return fmt.Errorf(`%s

You should specify Docker Hub token or username and password to remove tags with Docker Hub API.
Check --repo-docker-hub-token, --repo-docker-hub-username and --repo-docker-hub-password options.
Be aware that access to the resource is forbidden with personal access token.
Read more details here https://werf.io/documentation/v1.2/advanced/supported_container_registries.html#docker-hub`, err)
	case docker_registry.IsGitHubPackagesUnauthorizedErr(err), docker_registry.IsGitHubPackagesForbiddenErr(err):
		return fmt.Errorf(`%s

You should specify a token with delete:packages and read:packages scopes to remove package versions.
Check --repo-github-token option.
Be aware that the token provided to GitHub Actions workflow is not enough to remove package versions.
Read more details here https://werf.io/documentation/v1.2/advanced/supported_container_registries.html#github-packages`, err)
	default:
		if storage.IsImageDeletionFailedDueToUsingByContainerErr(err) {
			return err
		} else if strings.Contains(err.Error(), "UNAUTHORIZED") || strings.Contains(err.Error(), "UNSUPPORTED") {
			return err
		}

		return nil
	}
}