feat: new experimental deploy engine

Signed-off-by: Ilya Lesikov <ilya@lesikov.com>

cmd/werf/bundle/apply/apply.go

@@ -152,8 +152,14 @@ func runApply(ctx context.Context) error {
 		return err
 	}
 
+	namespace := common.GetNamespace(&commonCmdData)
+	releaseName, err := common.GetRequiredRelease(&commonCmdData)
+	if err != nil {
+		return err
+	}
+
 	actionConfig := new(action.Configuration)
-	if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), *commonCmdData.Namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
+	if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
 		StatusProgressPeriod:      time.Duration(*commonCmdData.StatusProgressPeriodSeconds) * time.Second,
 		HooksStatusProgressPeriod: time.Duration(*commonCmdData.HooksStatusProgressPeriodSeconds) * time.Second,
 		KubeConfigOptions: kube.KubeConfigOptions{
@@ -164,7 +170,7 @@ func runApply(ctx context.Context) error {
 		},
 		ReleasesHistoryMax: *commonCmdData.ReleasesHistoryMax,
 		RegistryClient:     helmRegistryClient,
-	}); err != nil {
+	}, nil); err != nil {
 		return err
 	}
 
@@ -175,12 +181,6 @@ func runApply(ctx context.Context) error {
 		return fmt.Errorf("unable to pull bundle: %w", err)
 	}
 
-	namespace := common.GetNamespace(&commonCmdData)
-	releaseName, err := common.GetRequiredRelease(&commonCmdData)
-	if err != nil {
-		return err
-	}
-
 	var lockManager *lock_manager.LockManager
 	if m, err := lock_manager.NewLockManager(namespace); err != nil {
 		return fmt.Errorf("unable to create lock manager: %w", err)

cmd/werf/bundle/render/render.go

@@ -150,8 +150,11 @@ func runRender(ctx context.Context) error {
 		return fmt.Errorf("unable to create helm registry client: %w", err)
 	}
 
+	namespace := common.GetNamespace(&commonCmdData)
+	releaseName := common.GetOptionalRelease(&commonCmdData)
+
 	actionConfig := new(action.Configuration)
-	if err := helm.InitActionConfig(ctx, nil, *commonCmdData.Namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{RegistryClient: helmRegistryClient}); err != nil {
+	if err := helm.InitActionConfig(ctx, nil, releaseName, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{RegistryClient: helmRegistryClient}, nil); err != nil {
 		return err
 	}
 
@@ -181,9 +184,6 @@ func runRender(ctx context.Context) error {
 		}
 	}
 
-	namespace := common.GetNamespace(&commonCmdData)
-	releaseName := common.GetOptionalRelease(&commonCmdData)
-
 	if *commonCmdData.Environment != "" {
 		userExtraAnnotations["project.werf.io/env"] = *commonCmdData.Environment
 	}

cmd/werf/common/helm.go

@@ -7,6 +7,7 @@ import (
 	helm_v3 "helm.sh/helm/v3/cmd/helm"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/registry"
+	"helm.sh/helm/v3/pkg/werf/mutator"
 
 	"github.com/werf/kubedog/pkg/kube"
 	"github.com/werf/logboek"
@@ -54,10 +55,10 @@ func NewBundlesRegistryClient(ctx context.Context, commonCmdData *CmdData) (*bun
 	)
 }
 
-func NewActionConfig(ctx context.Context, kubeInitializer helm.KubeInitializer, namespace string, commonCmdData *CmdData, registryClient *registry.Client) (*action.Configuration, error) {
+func NewActionConfig(ctx context.Context, kubeInitializer helm.KubeInitializer, releaseName, namespace string, commonCmdData *CmdData, registryClient *registry.Client, extraMutators []mutator.RuntimeResourceMutator) (*action.Configuration, error) {
 	actionConfig := new(action.Configuration)
 
-	if err := helm.InitActionConfig(ctx, kubeInitializer, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
+	if err := helm.InitActionConfig(ctx, kubeInitializer, releaseName, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
 		StatusProgressPeriod:      time.Duration(*commonCmdData.StatusProgressPeriodSeconds) * time.Second,
 		HooksStatusProgressPeriod: time.Duration(*commonCmdData.HooksStatusProgressPeriodSeconds) * time.Second,
 		KubeConfigOptions: kube.KubeConfigOptions{
@@ -68,7 +69,7 @@ func NewActionConfig(ctx context.Context, kubeInitializer helm.KubeInitializer,
 		},
 		ReleasesHistoryMax: *commonCmdData.ReleasesHistoryMax,
 		RegistryClient:     registryClient,
-	}); err != nil {
+	}, extraMutators); err != nil {
 		return nil, err
 	}
 

cmd/werf/converge/converge.go

@@ -16,6 +16,7 @@ import (
 	"helm.sh/helm/v3/pkg/cli/values"
 	"helm.sh/helm/v3/pkg/postrender"
 	"helm.sh/helm/v3/pkg/registry"
+	"helm.sh/helm/v3/pkg/werf/mutator"
 
 	"github.com/werf/kubedog/pkg/kube"
 	"github.com/werf/logboek"
@@ -469,51 +470,95 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken
 		FileValues:   common.GetSetFile(&commonCmdData),
 	}
 
-	actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, &commonCmdData, helmRegistryClient)
+	var extraRuntimeResourceMutators []mutator.RuntimeResourceMutator
+	if util.GetBoolEnvironmentDefaultFalse("WERF_EXPERIMENTAL_DEPLOY_ENGINE") {
+		extraRuntimeResourceMutators = []mutator.RuntimeResourceMutator{
+			helm.NewExtraAnnotationsMutator(userExtraAnnotations),
+			helm.NewExtraLabelsMutator(userExtraLabels),
+			helm.NewServiceAnnotationsMutator(*commonCmdData.Environment, werfConfig.Meta.Project),
+		}
+	}
+
+	actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, &commonCmdData, helmRegistryClient, extraRuntimeResourceMutators)
 	if err != nil {
 		return err
 	}
 	maintenanceHelper := createMaintenanceHelper(ctx, actionConfig, kubeConfigOptions)
 
-	if err := migrateHelm2ToHelm3(ctx, releaseName, namespace, maintenanceHelper, wc.ChainPostRenderer, valueOpts, filepath.Join(giterminismManager.ProjectDir(), chartDir), helmRegistryClient); err != nil {
+	if err := migrateHelm2ToHelm3(ctx, releaseName, namespace, maintenanceHelper, wc.ChainPostRenderer, valueOpts, filepath.Join(giterminismManager.ProjectDir(), chartDir), helmRegistryClient, extraRuntimeResourceMutators); err != nil {
 		return err
 	}
 
-	actionConfig, err = common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, &commonCmdData, helmRegistryClient)
+	actionConfig, err = common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, &commonCmdData, helmRegistryClient, extraRuntimeResourceMutators)
 	if err != nil {
 		return err
 	}
 
-	var deployReportPath *string
-	if common.GetSaveDeployReport(&commonCmdData) {
-		if path, err := common.GetDeployReportPath(&commonCmdData); err != nil {
-			return fmt.Errorf("unable to get deploy report path: %w", err)
-		} else {
-			deployReportPath = &path
+	if util.GetBoolEnvironmentDefaultFalse("WERF_EXPERIMENTAL_DEPLOY_ENGINE") {
+		// FIXME(ilya-lesikov): implement rollback
+		autoRollback := common.NewBool(cmdData.AutoRollback)
+		if *autoRollback {
+			return fmt.Errorf("--auto-rollback and --atomic not yet supported with new deploy engine")
 		}
-	}
 
-	helmUpgradeCmd, _ := helm_v3.NewUpgradeCmd(actionConfig, logboek.OutStream(), helm_v3.UpgradeCmdOptions{
-		StagesSplitter:              helm.NewStagesSplitter(),
-		StagesExternalDepsGenerator: helm.NewStagesExternalDepsGenerator(&actionConfig.RESTClientGetter, &namespace),
-		ChainPostRenderer:           wc.ChainPostRenderer,
-		ValueOpts:                   valueOpts,
-		CreateNamespace:             common.NewBool(true),
-		Install:                     common.NewBool(true),
-		Wait:                        common.NewBool(true),
-		Atomic:                      common.NewBool(cmdData.AutoRollback),
-		Timeout:                     common.NewDuration(time.Duration(cmdData.Timeout) * time.Second),
-		IgnorePending:               common.NewBool(true),
-		CleanupOnFail:               common.NewBool(true),
-		DeployReportPath:            deployReportPath,
-	})
+		var deployReportPath string
+		if common.GetSaveDeployReport(&commonCmdData) {
+			deployReportPath, err = common.GetDeployReportPath(&commonCmdData)
+			if err != nil {
+				return fmt.Errorf("unable to get deploy report path: %w", err)
+			}
+		}
+
+		helmDeployCmd := action.NewDeploy(releaseName, chartDir, actionConfig, helm_v3.Settings, action.DeployOptions{
+			OutStream:         logboek.OutStream(),
+			ErrStream:         logboek.ErrStream(),
+			ValueOptions:      valueOpts,
+			ReleaseNamespace:  namespace,
+			RollbackOnFailure: *autoRollback,
+			TrackTimeout:      *common.NewDuration(time.Duration(cmdData.Timeout) * time.Second),
+			KeepHistoryLimit:  *commonCmdData.ReleasesHistoryMax,
+			DeployReportPath:  deployReportPath,
+		})
 
-	return command_helpers.LockReleaseWrapper(ctx, releaseName, lockManager, func() error {
-		if err := helmUpgradeCmd.RunE(helmUpgradeCmd, []string{releaseName, filepath.Join(giterminismManager.ProjectDir(), chartDir)}); err != nil {
-			return fmt.Errorf("helm upgrade have failed: %w", err)
+		return command_helpers.LockReleaseWrapper(ctx, releaseName, lockManager, func() error {
+			if err := helmDeployCmd.Run(ctx); err != nil {
+				return fmt.Errorf("helm deploy failed: %w", err)
+			}
+
+			return nil
+		})
+	} else {
+		var deployReportPath *string
+		if common.GetSaveDeployReport(&commonCmdData) {
+			if path, err := common.GetDeployReportPath(&commonCmdData); err != nil {
+				return fmt.Errorf("unable to get deploy report path: %w", err)
+			} else {
+				deployReportPath = &path
+			}
 		}
-		return nil
-	})
+
+		helmUpgradeCmd, _ := helm_v3.NewUpgradeCmd(actionConfig, logboek.OutStream(), helm_v3.UpgradeCmdOptions{
+			StagesSplitter:              helm.NewStagesSplitter(),
+			StagesExternalDepsGenerator: helm.NewStagesExternalDepsGenerator(&actionConfig.RESTClientGetter, &namespace),
+			ChainPostRenderer:           wc.ChainPostRenderer,
+			ValueOpts:                   valueOpts,
+			CreateNamespace:             common.NewBool(true),
+			Install:                     common.NewBool(true),
+			Wait:                        common.NewBool(true),
+			Atomic:                      common.NewBool(cmdData.AutoRollback),
+			Timeout:                     common.NewDuration(time.Duration(cmdData.Timeout) * time.Second),
+			IgnorePending:               common.NewBool(true),
+			CleanupOnFail:               common.NewBool(true),
+			DeployReportPath:            deployReportPath,
+		})
+
+		return command_helpers.LockReleaseWrapper(ctx, releaseName, lockManager, func() error {
+			if err := helmUpgradeCmd.RunE(helmUpgradeCmd, []string{releaseName, filepath.Join(giterminismManager.ProjectDir(), chartDir)}); err != nil {
+				return fmt.Errorf("helm upgrade have failed: %w", err)
+			}
+			return nil
+		})
+	}
 }
 
 func createMaintenanceHelper(ctx context.Context, actionConfig *action.Configuration, kubeConfigOptions kube.KubeConfigOptions) *maintenance_helper.MaintenanceHelper {
@@ -545,7 +590,7 @@ func createMaintenanceHelper(ctx context.Context, actionConfig *action.Configura
 	return maintenance_helper.NewMaintenanceHelper(actionConfig, maintenanceOpts)
 }
 
-func migrateHelm2ToHelm3(ctx context.Context, releaseName, namespace string, maintenanceHelper *maintenance_helper.MaintenanceHelper, chainPostRenderer func(postrender.PostRenderer) postrender.PostRenderer, valueOpts *values.Options, fullChartDir string, helmRegistryClient *registry.Client) error {
+func migrateHelm2ToHelm3(ctx context.Context, releaseName, namespace string, maintenanceHelper *maintenance_helper.MaintenanceHelper, chainPostRenderer func(postrender.PostRenderer) postrender.PostRenderer, valueOpts *values.Options, fullChartDir string, helmRegistryClient *registry.Client, extraMutators []mutator.RuntimeResourceMutator) error {
 	if helm2Exists, err := checkHelm2AvailableAndReleaseExists(ctx, releaseName, namespace, maintenanceHelper); err != nil {
 		return fmt.Errorf("error checking availability of helm 2 and existence of helm 2 release %q: %w", releaseName, err)
 	} else if !helm2Exists {
@@ -572,7 +617,7 @@ func migrateHelm2ToHelm3(ctx context.Context, releaseName, namespace string, mai
 
 	logboek.Context(ctx).Default().LogOptionalLn()
 	if err := logboek.Context(ctx).LogProcess("Rendering helm 3 templates for the current project state").DoError(func() error {
-		actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, &commonCmdData, helmRegistryClient)
+		actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, &commonCmdData, helmRegistryClient, extraMutators)
 		if err != nil {
 			return err
 		}

cmd/werf/dismiss/dismiss.go

@@ -209,7 +209,7 @@ func runDismiss(ctx context.Context) error {
 	}
 
 	actionConfig := new(action.Configuration)
-	if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
+	if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), release, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
 		StatusProgressPeriod:      time.Duration(*commonCmdData.StatusProgressPeriodSeconds) * time.Second,
 		HooksStatusProgressPeriod: time.Duration(*commonCmdData.HooksStatusProgressPeriodSeconds) * time.Second,
 		KubeConfigOptions: kube.KubeConfigOptions{
@@ -219,7 +219,7 @@ func runDismiss(ctx context.Context) error {
 		},
 		ReleasesHistoryMax: *commonCmdData.ReleasesHistoryMax,
 		RegistryClient:     helmRegistryClient,
-	}); err != nil {
+	}, nil); err != nil {
 		return err
 	}
 

cmd/werf/helm/helm.go

@@ -200,7 +200,7 @@ func NewCmd(ctx context.Context) (*cobra.Command, error) {
 
 				common.SetupOndemandKubeInitializer(*_commonCmdData.KubeContext, *_commonCmdData.KubeConfig, *_commonCmdData.KubeConfigBase64, *_commonCmdData.KubeConfigPathMergeList)
 
-				helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
+				helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), "", namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
 					StatusProgressPeriod:      time.Duration(*_commonCmdData.StatusProgressPeriodSeconds) * time.Second,
 					HooksStatusProgressPeriod: time.Duration(*_commonCmdData.HooksStatusProgressPeriodSeconds) * time.Second,
 					KubeConfigOptions: kube.KubeConfigOptions{
@@ -210,7 +210,7 @@ func NewCmd(ctx context.Context) (*cobra.Command, error) {
 						ConfigDataBase64:    *_commonCmdData.KubeConfigBase64,
 					},
 					ReleasesHistoryMax: *_commonCmdData.ReleasesHistoryMax,
-				})
+				}, nil)
 
 				if oldRunE != nil {
 					return oldRunE(cmd, args)

cmd/werf/helm/migrate2to3.go

@@ -140,10 +140,10 @@ func runMigrate2To3(ctx context.Context) error {
 	}
 
 	actionConfig := new(action.Configuration)
-	if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), targetNamespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
+	if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), targetReleaseName, targetNamespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{
 		KubeConfigOptions: kubeConfigOptions,
 		RegistryClient:    helmRegistryClient,
-	}); err != nil {
+	}, nil); err != nil {
 		return err
 	}
 

cmd/werf/render/render.go

@@ -423,7 +423,7 @@ func runRender(ctx context.Context, imagesToProcess build.ImagesToProcess) error
 		wc.SetServiceValues(vals)
 	}
 
-	actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, &commonCmdData, helmRegistryClient)
+	actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, &commonCmdData, helmRegistryClient, nil)
 	if err != nil {
 		return err
 	}

go.mod

@@ -347,6 +347,6 @@ replace (
 	github.com/helm/helm-2to3 => github.com/werf/3p-helm-2to3 v0.0.0-20230313155428-cf9dd655c0e3 // switch back to upstream when merged: https://github.com/helm/helm-2to3/pull/224
 	github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305 // upstream not maintained
 	github.com/maorfr/helm-plugin-utils => github.com/werf/3p-helm-plugin-utils v0.6.1-0.20230313152239-057595ce9d57 // switch back to upstream when merged: https://github.com/maorfr/helm-plugin-utils/pull/17
-	helm.sh/helm/v3 => github.com/werf/3p-helm/v3 v3.0.0-20230309133321-09d8a1e63ebd
+	helm.sh/helm/v3 => github.com/werf/3p-helm/v3 v3.0.0-20230518112226-7a0b661fec7e
 	k8s.io/helm => github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f
 )

go.sum

@@ -1738,8 +1738,8 @@ github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln
 github.com/weppos/publicsuffix-go v0.20.1-0.20221209102050-40d9c30084b3 h1:ypyhoprZWFzU0ydOBv3I5SS7/jLFJ+ujPAU+BD/EVFM=
 github.com/werf/3p-helm-2to3 v0.0.0-20230313155428-cf9dd655c0e3 h1:pdKLiuiUoOFswc0HV6GX1+JHgP14HqE035xl2DQJi8A=
 github.com/werf/3p-helm-2to3 v0.0.0-20230313155428-cf9dd655c0e3/go.mod h1:lTSQBLomjl67m4HQMbg3BHKWes3pOmjNBg8yygXYugk=
-github.com/werf/3p-helm/v3 v3.0.0-20230309133321-09d8a1e63ebd h1:wkuBgBCZsQ5XwjGIJugEwUvVarcDdn49GAJxG6EMokg=
-github.com/werf/3p-helm/v3 v3.0.0-20230309133321-09d8a1e63ebd/go.mod h1:6n2FO7dTVoTlSkrU+oceTIXZrd8GbPiEshqXHdVIckc=
+github.com/werf/3p-helm/v3 v3.0.0-20230518112226-7a0b661fec7e h1:dyyHwYMYuY0UAzFGGPGM1MBYvoD2EcSiw11VTtxoLlY=
+github.com/werf/3p-helm/v3 v3.0.0-20230518112226-7a0b661fec7e/go.mod h1:vH6ckglZT6mhJefVLbUM9759Zt/wQnqqRe0BY4jmmHM=
 github.com/werf/copy-recurse v0.2.7 h1:3FTOarbJ9uhFLi75oeUCioK9zxZwuV7o28kuUBPDZPM=
 github.com/werf/copy-recurse v0.2.7/go.mod h1:6Ypb+qN+hRBJgoCgEkX1vpbqcQ+8q69BQ3hi8s8Y6Qc=
 github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f h1:81YscYTF9mmTf0ULOsCmm42YWQp+qWDzWi1HjWniZrg=

pkg/deploy/helm/extra_annotations_mutator.go

@@ -0,0 +1,42 @@
+package helm
+
+import (
+	"fmt"
+
+	"helm.sh/helm/v3/pkg/werf/common"
+	"helm.sh/helm/v3/pkg/werf/mutator"
+	"helm.sh/helm/v3/pkg/werf/resource"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+)
+
+var _ mutator.RuntimeResourceMutator = (*ExtraAnnotationsMutator)(nil)
+
+func NewExtraAnnotationsMutator(extraAnnos map[string]string) *ExtraAnnotationsMutator {
+	return &ExtraAnnotationsMutator{
+		extraAnnos: extraAnnos,
+	}
+}
+
+type ExtraAnnotationsMutator struct {
+	extraAnnos map[string]string
+}
+
+func (m *ExtraAnnotationsMutator) Mutate(res resource.Resourcer, operationType common.ClientOperationType) (resource.Resourcer, error) {
+	if !res.PartOfRelease() {
+		return res, nil
+	}
+
+	switch operationType {
+	case common.ClientOperationTypeCreate, common.ClientOperationTypeUpdate, common.ClientOperationTypeSmartApply:
+	default:
+		return res, nil
+	}
+
+	for k, v := range m.extraAnnos {
+		if err := unstructured.SetNestedField(res.Unstructured().UnstructuredContent(), v, "metadata", "annotations", k); err != nil {
+			return nil, fmt.Errorf("error adding extra annotation: %w", err)
+		}
+	}
+
+	return res, nil
+}