From dd43b6839b2a28ce2b4273f77c47f2fa0994969e Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Tue, 5 Oct 2021 16:02:48 +0300 Subject: [PATCH] fix(cleanup): fix "should reset storage cache" error during werf-cleanup and werf-purge Automatically reset stages storage cache in the werf-cleanup and werf-purge commands as in werf-build. --- cmd/werf/cleanup/cleanup.go | 10 ++-- cmd/werf/purge/purge.go | 14 +++--- .../_includes/reference/cli/werf_purge.md | 2 +- pkg/build/conveyor_with_retry.go | 46 ++++++------------- pkg/storage/manager/storage_manager.go | 17 +++++++ 5 files changed, 43 insertions(+), 46 deletions(-) diff --git a/cmd/werf/cleanup/cleanup.go b/cmd/werf/cleanup/cleanup.go index 81ab89534f..f2187775e9 100644 --- a/cmd/werf/cleanup/cleanup.go +++ b/cmd/werf/cleanup/cleanup.go @@ -258,10 +258,8 @@ It is worth noting that auto-cleaning is enabled by default, and manual use is u DryRun: *commonCmdData.DryRun, } - logboek.LogOptionalLn() - if err := cleaning.Cleanup(ctx, projectName, storageManager, storageLockManager, cleanupOptions); err != nil { - return err - } - - return nil + return manager.RetryOnStagesStorageCacheResetError(ctx, storageManager, func() error { + logboek.LogOptionalLn() + return cleaning.Cleanup(ctx, projectName, storageManager, storageLockManager, cleanupOptions) + }) } diff --git a/cmd/werf/purge/purge.go b/cmd/werf/purge/purge.go index 467d25cef5..c97f10e5a3 100644 --- a/cmd/werf/purge/purge.go +++ b/cmd/werf/purge/purge.go @@ -28,7 +28,7 @@ func NewCmd() *cobra.Command { Use: "purge", DisableFlagsInUseLine: true, Short: "Purge all project images in the container registry", - Long: common.GetLongCommandDescription(`Purge all project images in the container registry. + Long: common.GetLongCommandDescription(`Purge all project images in the container registry. WARNING: Images that are being used in the Kubernetes cluster will also be deleted.`), RunE: func(cmd *cobra.Command, args []string) error { @@ -150,7 +150,7 @@ func runPurge() error { stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData) if err != nil { - logboek.Context(ctx).Default().LogLnDetails(`The "werf purge" command is only used to cleaning the container registry. In case you need to clean the runner or the localhost, use the commands of the "werf host" group. + logboek.Context(ctx).Default().LogLnDetails(`The "werf purge" command is only used to cleaning the container registry. In case you need to clean the runner or the localhost, use the commands of the "werf host" group. It is worth noting that auto-cleaning is enabled by default, and manual use is usually not required (if not, we would appreciate feedback and creating an issue https://github.com/werf/werf/issues/new).`) return err @@ -194,10 +194,8 @@ It is worth noting that auto-cleaning is enabled by default, and manual use is u DryRun: *commonCmdData.DryRun, } - logboek.LogOptionalLn() - if err := cleaning.Purge(ctx, projectName, storageManager, storageLockManager, purgeOptions); err != nil { - return err - } - - return nil + return manager.RetryOnStagesStorageCacheResetError(ctx, storageManager, func() error { + logboek.LogOptionalLn() + return cleaning.Purge(ctx, projectName, storageManager, storageLockManager, purgeOptions) + }) } diff --git a/docs/documentation/_includes/reference/cli/werf_purge.md b/docs/documentation/_includes/reference/cli/werf_purge.md index 2630d1fd75..afbe3c8a83 100644 --- a/docs/documentation/_includes/reference/cli/werf_purge.md +++ b/docs/documentation/_includes/reference/cli/werf_purge.md @@ -3,7 +3,7 @@ {% else %} {% assign header = "###" %} {% endif %} -Purge all project images in the container registry. +Purge all project images in the container registry. WARNING: Images that are being used in the Kubernetes cluster will also be deleted. diff --git a/pkg/build/conveyor_with_retry.go b/pkg/build/conveyor_with_retry.go index 8c21499d41..97c4aa458b 100644 --- a/pkg/build/conveyor_with_retry.go +++ b/pkg/build/conveyor_with_retry.go @@ -3,7 +3,6 @@ package build import ( "context" - "github.com/werf/logboek" "github.com/werf/werf/pkg/config" "github.com/werf/werf/pkg/container_runtime" "github.com/werf/werf/pkg/giterminism_manager" @@ -45,37 +44,22 @@ func (wrapper *ConveyorWithRetryWrapper) Terminate() error { } func (wrapper *ConveyorWithRetryWrapper) WithRetryBlock(ctx context.Context, f func(c *Conveyor) error) error { -Retry: - newConveyor := NewConveyor( - wrapper.WerfConfig, - wrapper.GiterminismManager, - wrapper.ImageNamesToProcess, - wrapper.ProjectDir, - wrapper.BaseTmpDir, - wrapper.SshAuthSock, - wrapper.ContainerRuntime, - wrapper.StorageManager, - wrapper.StorageLockManager, - wrapper.ConveyorOptions, - ) + return manager.RetryOnStagesStorageCacheResetError(ctx, wrapper.StorageManager, func() error { + newConveyor := NewConveyor( + wrapper.WerfConfig, + wrapper.GiterminismManager, + wrapper.ImageNamesToProcess, + wrapper.ProjectDir, + wrapper.BaseTmpDir, + wrapper.SshAuthSock, + wrapper.ContainerRuntime, + wrapper.StorageManager, + wrapper.StorageLockManager, + wrapper.ConveyorOptions, + ) - if shouldRetry, err := func() (bool, error) { defer newConveyor.Terminate(ctx) - if err := f(newConveyor); manager.ShouldResetStagesStorageCache(err) { - logboek.Context(ctx).Error().LogF("Will reset stages storage cache due to error: %s\n", err) - - if err := newConveyor.StorageManager.ResetStagesStorageCache(ctx); err != nil { - return false, err - } - return true, nil - } else { - return false, err - } - }(); err != nil { - return err - } else if shouldRetry { - goto Retry - } - return nil + return f(newConveyor) + }) } diff --git a/pkg/storage/manager/storage_manager.go b/pkg/storage/manager/storage_manager.go index 58cee6034a..b1d1c7de90 100644 --- a/pkg/storage/manager/storage_manager.go +++ b/pkg/storage/manager/storage_manager.go @@ -80,6 +80,23 @@ func ShouldResetStagesStorageCache(err error) bool { return false } +func RetryOnStagesStorageCacheResetError(ctx context.Context, manager StorageManagerInterface, f func() error) error { +Retry: + err := f() + + if ShouldResetStagesStorageCache(err) { + logboek.Context(ctx).Error().LogF("Will reset stages storage cache due to error: %s\n", err) + + if err := manager.ResetStagesStorageCache(ctx); err != nil { + return fmt.Errorf("unable to reset stages storage cache: %s", err) + } + + goto Retry + } + + return err +} + func NewStorageManager(projectName string, stagesStorage storage.StagesStorage, finalStagesStorage storage.StagesStorage, secondaryStagesStorageList []storage.StagesStorage, cacheStagesStorageList []storage.StagesStorage, storageLockManager storage.LockManager, stagesStorageCache storage.StagesStorageCache) *StorageManager { return &StorageManager{ ProjectName: projectName,