feat(gc): run host garbage collection in background

cmd/werf/build/main.go

@@ -51,7 +51,7 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image
 			common.CmdEnvAnno: common.EnvsDescription(common.WerfDebugAnsibleArgs),
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			defer global_warnings.PrintGlobalWarnings(ctx)
 

cmd/werf/bundle/apply/apply.go

@@ -44,7 +44,7 @@ func NewCmd() *cobra.Command {
 			common.CmdEnvAnno: common.EnvsDescription(),
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			defer global_warnings.PrintGlobalWarnings(common.BackgroundContext())
+			defer global_warnings.PrintGlobalWarnings(common.GetContext())
 
 			if err := common.ProcessLogOptions(&commonCmdData); err != nil {
 				common.PrintHelp(cmd)
@@ -101,7 +101,7 @@ func NewCmd() *cobra.Command {
 }
 
 func runApply() error {
-	ctx := common.BackgroundContext()
+	ctx := common.GetContext()
 
 	global_warnings.PostponeMultiwerfNotUpToDateWarning()
 

cmd/werf/bundle/download/download.go

@@ -30,7 +30,7 @@ func NewCmd() *cobra.Command {
 			common.CmdEnvAnno: common.EnvsDescription(),
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			defer global_warnings.PrintGlobalWarnings(common.BackgroundContext())
+			defer global_warnings.PrintGlobalWarnings(common.GetContext())
 
 			if err := common.ProcessLogOptions(&commonCmdData); err != nil {
 				common.PrintHelp(cmd)
@@ -67,7 +67,7 @@ func NewCmd() *cobra.Command {
 }
 
 func runDownload() error {
-	ctx := common.BackgroundContext()
+	ctx := common.GetContext()
 
 	if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
 		return fmt.Errorf("initialization error: %s", err)

cmd/werf/bundle/export/export.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
-	helm_v3 "helm.sh/helm/v3/cmd/helm"
+	"helm.sh/helm/v3/cmd/helm"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/cli/values"
@@ -47,7 +47,7 @@ func NewCmd() *cobra.Command {
 			common.CmdEnvAnno: common.EnvsDescription(),
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			defer global_warnings.PrintGlobalWarnings(ctx)
 

cmd/werf/bundle/publish/publish.go

@@ -10,7 +10,7 @@ import (
 	"github.com/Masterminds/semver"
 	uuid "github.com/satori/go.uuid"
 	"github.com/spf13/cobra"
-	helm_v3 "helm.sh/helm/v3/cmd/helm"
+	"helm.sh/helm/v3/cmd/helm"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/cli/values"
@@ -55,7 +55,7 @@ Published into container registry bundle can be rolled out by the "werf bundle"
 			common.CmdEnvAnno: common.EnvsDescription(),
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			defer global_warnings.PrintGlobalWarnings(ctx)
 

cmd/werf/ci_env/ci_env.go

@@ -87,7 +87,7 @@ Currently supported only GitLab (gitlab) and GitHub (github) CI systems`,
 func runCIEnv(cmd *cobra.Command, args []string) error {
 	logboek.SetAcceptedLevel(level.Error)
 
-	ctx := common.BackgroundContext()
+	ctx := common.GetContext()
 
 	if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
 		return fmt.Errorf("initialization error: %s", err)

cmd/werf/cleanup/cleanup.go

@@ -34,7 +34,7 @@ The command works according to special rules called cleanup policies, which the
 It is safe to run this command periodically (daily is enough) by automated cleanup job in parallel with other werf commands such as build, converge and host cleanup.`),
 		Example: `  $ werf cleanup --repo registry.mydomain.com/myproject/werf`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			defer global_warnings.PrintGlobalWarnings(ctx)
 

cmd/werf/common/common.go

@@ -28,6 +28,7 @@ import (
 	"github.com/werf/werf/pkg/true_git"
 	"github.com/werf/werf/pkg/util"
 	"github.com/werf/werf/pkg/werf"
+	"github.com/werf/werf/pkg/werf/global_warnings"
 )
 
 type CmdData struct {
@@ -1182,17 +1183,17 @@ func GetGiterminismManager(ctx context.Context, cmdData *CmdData) (giterminism_m
 		openLocalRepoOptions.ServiceBranchOptions.GlobExcludeList = GetDevIgnore(cmdData)
 	}
 
-	localGitRepo, err := git_repo.OpenLocalRepo(BackgroundContext(), "own", gitWorkTree, openLocalRepoOptions)
+	localGitRepo, err := git_repo.OpenLocalRepo(GetContext(), "own", gitWorkTree, openLocalRepoOptions)
 	if err != nil {
 		return nil, err
 	}
 
-	headCommit, err := localGitRepo.HeadCommitHash(BackgroundContext())
+	headCommit, err := localGitRepo.HeadCommitHash(GetContext())
 	if err != nil {
 		return nil, err
 	}
 
-	return giterminism_manager.NewManager(BackgroundContext(), workingDir, localGitRepo, headCommit, giterminism_manager.NewManagerOptions{
+	return giterminism_manager.NewManager(GetContext(), workingDir, localGitRepo, headCommit, giterminism_manager.NewManagerOptions{
 		LooseGiterminism: *cmdData.LooseGiterminism,
 		Dev:              *cmdData.Dev,
 	})
@@ -1520,6 +1521,70 @@ func SetupPlatform(cmdData *CmdData, cmd *cobra.Command) {
 	cmd.Flags().StringVarP(cmdData.Platform, "platform", "", defaultValue, "Enable platform emulation when building images with werf. The only supported option for now is linux/amd64.")
 }
 
-func BackgroundContext() context.Context {
+func GetContext() context.Context {
 	return logboek.NewContext(context.Background(), logboek.DefaultLogger())
 }
+
+func WithContext(allowBackgroundMode bool, f func(ctx context.Context) error) error {
+	var ctx context.Context
+
+	if allowBackgroundMode && IsBackgroundModeEnabled() {
+		out, err := os.OpenFile(GetBackgroundOutputFile(), os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0o644)
+		if err != nil {
+			return fmt.Errorf("unable to open background output file %q: %s", GetBackgroundOutputFile(), err)
+		}
+		defer out.Close()
+
+		ctx = logboek.NewContext(context.Background(), logboek.NewLogger(out, out))
+
+		if err := f(ctx); err != nil {
+			if err := os.WriteFile(GetLastBackgroundErrorFile(), []byte(err.Error()+"\n"), 0o644); err != nil {
+				logboek.Context(ctx).Warn().LogF("ERROR: unable to write %q: %s\n", GetLastBackgroundErrorFile(), err)
+			}
+			return err
+		}
+
+		return nil
+	} else {
+		ctx = logboek.NewContext(context.Background(), logboek.DefaultLogger())
+
+		if allowBackgroundMode {
+			if backgroundErr, err := GetAndRemoveLastBackgroundError(); err != nil {
+				return fmt.Errorf("unable to get last background error: %s", err)
+			} else if backgroundErr != nil {
+				global_warnings.GlobalWarningLn(ctx, fmt.Sprintf("last background error: %s", backgroundErr))
+			}
+		}
+
+		return f(ctx)
+	}
+}
+
+func GetAndRemoveLastBackgroundError() (error, error) {
+	data, err := os.ReadFile(GetLastBackgroundErrorFile())
+	if err != nil {
+		return nil, nil
+	}
+
+	if err := os.RemoveAll(GetLastBackgroundErrorFile()); err != nil {
+		return nil, fmt.Errorf("unable to remove %q: %s", GetLastBackgroundErrorFile(), err)
+	}
+
+	if len(data) != 0 {
+		return fmt.Errorf("%s", string(data)), nil
+	}
+
+	return nil, nil
+}
+
+func IsBackgroundModeEnabled() bool {
+	return os.Getenv("_WERF_BACKGROUND_MODE_ENABLED") == "1"
+}
+
+func GetBackgroundOutputFile() string {
+	return filepath.Join(werf.GetServiceDir(), "background_output.log")
+}
+
+func GetLastBackgroundErrorFile() string {
+	return filepath.Join(werf.GetServiceDir(), "last_background_error")
+}

cmd/werf/common/host_cleanup.go

@@ -23,14 +23,16 @@ func RunAutoHostCleanup(ctx context.Context, cmdData *CmdData) error {
 		return fmt.Errorf("incompatible params --allowed-local-cache-volume-usage=%d and --allowed-local-cache-volume-usage-margin=%d: margin percentage should be less than allowed volume usage level percentage", *cmdData.AllowedLocalCacheVolumeUsage, *cmdData.AllowedLocalCacheVolumeUsageMargin)
 	}
 
-	return host_cleaning.RunAutoHostCleanup(ctx, host_cleaning.HostCleanupOptions{
-		DryRun: false,
-		Force:  false,
-		AllowedDockerStorageVolumeUsagePercentage:       cmdData.AllowedDockerStorageVolumeUsage,
-		AllowedDockerStorageVolumeUsageMarginPercentage: cmdData.AllowedDockerStorageVolumeUsageMargin,
-		AllowedLocalCacheVolumeUsagePercentage:          cmdData.AllowedLocalCacheVolumeUsage,
-		AllowedLocalCacheVolumeUsageMarginPercentage:    cmdData.AllowedLocalCacheVolumeUsageMargin,
-		DockerServerStoragePath:                         *cmdData.DockerServerStoragePath,
+	return host_cleaning.RunAutoHostCleanup(ctx, host_cleaning.AutoHostCleanupOptions{
+		HostCleanupOptions: host_cleaning.HostCleanupOptions{
+			DryRun: false,
+			Force:  false,
+			AllowedDockerStorageVolumeUsagePercentage:       cmdData.AllowedDockerStorageVolumeUsage,
+			AllowedDockerStorageVolumeUsageMarginPercentage: cmdData.AllowedDockerStorageVolumeUsageMargin,
+			AllowedLocalCacheVolumeUsagePercentage:          cmdData.AllowedLocalCacheVolumeUsage,
+			AllowedLocalCacheVolumeUsageMarginPercentage:    cmdData.AllowedLocalCacheVolumeUsageMargin,
+			DockerServerStoragePath:                         cmdData.DockerServerStoragePath,
+		},
 	})
 }
 

cmd/werf/common/storage_repo_data.go

@@ -31,7 +31,7 @@ func (d *RepoData) GetContainerRegistry() string {
 	case *d.ContainerRegistry != "":
 		return *d.ContainerRegistry
 	case *d.Implementation != "":
-		logboek.Context(BackgroundContext()).Warn().LogLn("DEPRECATION WARNING: The option --repo-implementation ($WERF_REPO_IMPLEMENTATION) is renamed to --repo-container-registry ($WERF_REPO_CONTAINER_REGISTRY) and will be removed in v1.3!")
+		logboek.Context(GetContext()).Warn().LogLn("DEPRECATION WARNING: The option --repo-implementation ($WERF_REPO_IMPLEMENTATION) is renamed to --repo-container-registry ($WERF_REPO_CONTAINER_REGISTRY) and will be removed in v1.3!")
 		return *d.Implementation
 	default:
 		return ""

cmd/werf/common/synchronization.go

@@ -54,7 +54,7 @@ func checkSynchronizationKubernetesParamsForWarnings(cmdData *CmdData) {
 		return
 	}
 
-	ctx := BackgroundContext()
+	ctx := GetContext()
 	doPrintWarning := false
 	kubeConfigEnv := os.Getenv("KUBECONFIG")
 	switch {

cmd/werf/compose/main.go

@@ -164,7 +164,7 @@ services:
 		},
 		Example: options.Example,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 			defer global_warnings.PrintGlobalWarnings(ctx)
 
 			if err := common.ProcessLogOptions(&commonCmdData); err != nil {

cmd/werf/config/list/list.go

@@ -53,7 +53,7 @@ func NewCmd() *cobra.Command {
 }
 
 func run() error {
-	ctx := common.BackgroundContext()
+	ctx := common.GetContext()
 
 	if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil {
 		return fmt.Errorf("initialization error: %s", err)
@@ -77,7 +77,7 @@ func run() error {
 		return err
 	}
 
-	_, werfConfig, err := common.GetRequiredWerfConfig(common.BackgroundContext(), &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, false))
+	_, werfConfig, err := common.GetRequiredWerfConfig(common.GetContext(), &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, false))
 	if err != nil {
 		return err
 	}

cmd/werf/config/render/render.go

@@ -21,7 +21,7 @@ func NewCmd() *cobra.Command {
 		DisableFlagsInUseLine: true,
 		Short:                 "Render werf.yaml",
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			if err := common.ProcessLogOptions(&commonCmdData); err != nil {
 				common.PrintHelp(cmd)
@@ -62,7 +62,7 @@ func NewCmd() *cobra.Command {
 				return err
 			}
 
-			return config.RenderWerfConfig(common.BackgroundContext(), customWerfConfigRelPath, customWerfConfigTemplatesDirRelPath, args, giterminismManager, configOpts)
+			return config.RenderWerfConfig(common.GetContext(), customWerfConfigRelPath, customWerfConfigTemplatesDirRelPath, args, giterminismManager, configOpts)
 		},
 	}
 

cmd/werf/converge/converge.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/spf13/cobra"
-	helm_v3 "helm.sh/helm/v3/cmd/helm"
+	"helm.sh/helm/v3/cmd/helm"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
@@ -65,7 +65,7 @@ werf converge --repo registry.mydomain.com/web --env production`,
 			common.CmdEnvAnno: common.EnvsDescription(common.WerfDebugAnsibleArgs, common.WerfSecretKey),
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			defer global_warnings.PrintGlobalWarnings(ctx)
 

cmd/werf/cr/login/login.go

@@ -41,7 +41,7 @@ werf cr login -p token registry.example.com
 werf cr login --insecure-registry registry.example.com`,
 		DisableFlagsInUseLine: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			defer global_warnings.PrintGlobalWarnings(ctx)
 

cmd/werf/cr/logout/logout.go

@@ -23,7 +23,7 @@ func NewCmd() *cobra.Command {
 		Long:                  common.GetLongCommandDescription(`Logout from a remote registry`),
 		DisableFlagsInUseLine: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			defer global_warnings.PrintGlobalWarnings(ctx)
 

cmd/werf/dismiss/dismiss.go

@@ -53,7 +53,7 @@ Read more info about Helm Release name, Kubernetes Namespace and how to change i
   $ werf dismiss --release myrelease --namespace myns`,
 		DisableFlagsInUseLine: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			defer global_warnings.PrintGlobalWarnings(ctx)
 

cmd/werf/export/export.go

@@ -45,7 +45,7 @@ All meta-information related to werf is removed from the exported images, and th
 			common.DisableOptionsInUseLineAnno: "1",
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 			defer global_warnings.PrintGlobalWarnings(ctx)
 
 			if err := common.ProcessLogOptions(&commonCmdData); err != nil {

cmd/werf/helm/get_autogenerated_values.go

@@ -37,7 +37,7 @@ These values includes project name, docker images ids and other`),
 			common.CmdEnvAnno: common.EnvsDescription(common.WerfSecretKey),
 		},
 		RunE: func(cmd *cobra.Command, args []string) error {
-			ctx := common.BackgroundContext()
+			ctx := common.GetContext()
 
 			if err := common.ProcessLogOptions(&getAutogeneratedValuedCmdData); err != nil {
 				common.PrintHelp(cmd)

cmd/werf/helm/get_namespace.go

@@ -47,7 +47,7 @@ func NewGetNamespaceCmd() *cobra.Command {
 }
 
 func runGetNamespace() error {
-	ctx := common.BackgroundContext()
+	ctx := common.GetContext()
 
 	if err := werf.Init(*getNamespaceCmdData.TmpDir, *getNamespaceCmdData.HomeDir); err != nil {
 		return fmt.Errorf("initialization error: %s", err)
@@ -71,7 +71,7 @@ func runGetNamespace() error {
 		return err
 	}
 
-	_, werfConfig, err := common.GetRequiredWerfConfig(common.BackgroundContext(), &getNamespaceCmdData, giterminismManager, common.GetWerfConfigOptions(&getNamespaceCmdData, false))
+	_, werfConfig, err := common.GetRequiredWerfConfig(common.GetContext(), &getNamespaceCmdData, giterminismManager, common.GetWerfConfigOptions(&getNamespaceCmdData, false))
 	if err != nil {
 		return fmt.Errorf("unable to load werf config: %s", err)
 	}

cmd/werf/helm/get_release.go

@@ -47,7 +47,7 @@ func NewGetReleaseCmd() *cobra.Command {
 }
 
 func runGetRelease() error {
-	ctx := common.BackgroundContext()
+	ctx := common.GetContext()
 
 	if err := werf.Init(*getReleaseCmdData.TmpDir, *getReleaseCmdData.HomeDir); err != nil {
 		return fmt.Errorf("initialization error: %s", err)
@@ -71,7 +71,7 @@ func runGetRelease() error {
 		return err
 	}
 
-	_, werfConfig, err := common.GetRequiredWerfConfig(common.BackgroundContext(), &getReleaseCmdData, giterminismManager, common.GetWerfConfigOptions(&getReleaseCmdData, false))
+	_, werfConfig, err := common.GetRequiredWerfConfig(common.GetContext(), &getReleaseCmdData, giterminismManager, common.GetWerfConfigOptions(&getReleaseCmdData, false))
 	if err != nil {
 		return fmt.Errorf("unable to load werf config: %s", err)
 	}