diff --git a/cmd/werf/bundle/apply/apply.go b/cmd/werf/bundle/apply/apply.go index db22155f41..19373d19ce 100644 --- a/cmd/werf/bundle/apply/apply.go +++ b/cmd/werf/bundle/apply/apply.go @@ -152,8 +152,14 @@ func runApply(ctx context.Context) error { return err } + namespace := common.GetNamespace(&commonCmdData) + releaseName, err := common.GetRequiredRelease(&commonCmdData) + if err != nil { + return err + } + actionConfig := new(action.Configuration) - if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), *commonCmdData.Namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ + if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ StatusProgressPeriod: time.Duration(*commonCmdData.StatusProgressPeriodSeconds) * time.Second, HooksStatusProgressPeriod: time.Duration(*commonCmdData.HooksStatusProgressPeriodSeconds) * time.Second, KubeConfigOptions: kube.KubeConfigOptions{ @@ -164,7 +170,7 @@ func runApply(ctx context.Context) error { }, ReleasesHistoryMax: *commonCmdData.ReleasesHistoryMax, RegistryClient: helmRegistryClient, - }); err != nil { + }, nil); err != nil { return err } @@ -175,12 +181,6 @@ func runApply(ctx context.Context) error { return fmt.Errorf("unable to pull bundle: %w", err) } - namespace := common.GetNamespace(&commonCmdData) - releaseName, err := common.GetRequiredRelease(&commonCmdData) - if err != nil { - return err - } - var lockManager *lock_manager.LockManager if m, err := lock_manager.NewLockManager(namespace); err != nil { return fmt.Errorf("unable to create lock manager: %w", err) diff --git a/cmd/werf/bundle/render/render.go b/cmd/werf/bundle/render/render.go index d2b4e2d2f1..58f04a6a83 100644 --- a/cmd/werf/bundle/render/render.go +++ b/cmd/werf/bundle/render/render.go @@ -150,8 +150,11 @@ func runRender(ctx context.Context) error { return fmt.Errorf("unable to create helm registry client: %w", err) } + namespace := common.GetNamespace(&commonCmdData) + releaseName := common.GetOptionalRelease(&commonCmdData) + actionConfig := new(action.Configuration) - if err := helm.InitActionConfig(ctx, nil, *commonCmdData.Namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{RegistryClient: helmRegistryClient}); err != nil { + if err := helm.InitActionConfig(ctx, nil, releaseName, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{RegistryClient: helmRegistryClient}, nil); err != nil { return err } @@ -181,9 +184,6 @@ func runRender(ctx context.Context) error { } } - namespace := common.GetNamespace(&commonCmdData) - releaseName := common.GetOptionalRelease(&commonCmdData) - if *commonCmdData.Environment != "" { userExtraAnnotations["project.werf.io/env"] = *commonCmdData.Environment } diff --git a/cmd/werf/common/helm.go b/cmd/werf/common/helm.go index 4a06650f26..96569746fd 100644 --- a/cmd/werf/common/helm.go +++ b/cmd/werf/common/helm.go @@ -7,6 +7,7 @@ import ( helm_v3 "helm.sh/helm/v3/cmd/helm" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/registry" + "helm.sh/helm/v3/pkg/werf/mutator" "github.com/werf/kubedog/pkg/kube" "github.com/werf/logboek" @@ -54,10 +55,10 @@ func NewBundlesRegistryClient(ctx context.Context, commonCmdData *CmdData) (*bun ) } -func NewActionConfig(ctx context.Context, kubeInitializer helm.KubeInitializer, namespace string, commonCmdData *CmdData, registryClient *registry.Client) (*action.Configuration, error) { +func NewActionConfig(ctx context.Context, kubeInitializer helm.KubeInitializer, releaseName, namespace string, commonCmdData *CmdData, registryClient *registry.Client, extraMutators []mutator.RuntimeResourceMutator) (*action.Configuration, error) { actionConfig := new(action.Configuration) - if err := helm.InitActionConfig(ctx, kubeInitializer, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ + if err := helm.InitActionConfig(ctx, kubeInitializer, releaseName, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ StatusProgressPeriod: time.Duration(*commonCmdData.StatusProgressPeriodSeconds) * time.Second, HooksStatusProgressPeriod: time.Duration(*commonCmdData.HooksStatusProgressPeriodSeconds) * time.Second, KubeConfigOptions: kube.KubeConfigOptions{ @@ -68,7 +69,7 @@ func NewActionConfig(ctx context.Context, kubeInitializer helm.KubeInitializer, }, ReleasesHistoryMax: *commonCmdData.ReleasesHistoryMax, RegistryClient: registryClient, - }); err != nil { + }, extraMutators); err != nil { return nil, err } diff --git a/cmd/werf/converge/converge.go b/cmd/werf/converge/converge.go index eadc7d9530..9cd9a5560e 100644 --- a/cmd/werf/converge/converge.go +++ b/cmd/werf/converge/converge.go @@ -16,6 +16,7 @@ import ( "helm.sh/helm/v3/pkg/cli/values" "helm.sh/helm/v3/pkg/postrender" "helm.sh/helm/v3/pkg/registry" + "helm.sh/helm/v3/pkg/werf/mutator" "github.com/werf/kubedog/pkg/kube" "github.com/werf/logboek" @@ -469,51 +470,95 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken FileValues: common.GetSetFile(&commonCmdData), } - actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, &commonCmdData, helmRegistryClient) + var extraRuntimeResourceMutators []mutator.RuntimeResourceMutator + if util.GetBoolEnvironmentDefaultFalse("WERF_EXPERIMENTAL_DEPLOY_ENGINE") { + extraRuntimeResourceMutators = []mutator.RuntimeResourceMutator{ + helm.NewExtraAnnotationsMutator(userExtraAnnotations), + helm.NewExtraLabelsMutator(userExtraLabels), + helm.NewServiceAnnotationsMutator(*commonCmdData.Environment, werfConfig.Meta.Project), + } + } + + actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, &commonCmdData, helmRegistryClient, extraRuntimeResourceMutators) if err != nil { return err } maintenanceHelper := createMaintenanceHelper(ctx, actionConfig, kubeConfigOptions) - if err := migrateHelm2ToHelm3(ctx, releaseName, namespace, maintenanceHelper, wc.ChainPostRenderer, valueOpts, filepath.Join(giterminismManager.ProjectDir(), chartDir), helmRegistryClient); err != nil { + if err := migrateHelm2ToHelm3(ctx, releaseName, namespace, maintenanceHelper, wc.ChainPostRenderer, valueOpts, filepath.Join(giterminismManager.ProjectDir(), chartDir), helmRegistryClient, extraRuntimeResourceMutators); err != nil { return err } - actionConfig, err = common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, &commonCmdData, helmRegistryClient) + actionConfig, err = common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, &commonCmdData, helmRegistryClient, extraRuntimeResourceMutators) if err != nil { return err } - var deployReportPath *string - if common.GetSaveDeployReport(&commonCmdData) { - if path, err := common.GetDeployReportPath(&commonCmdData); err != nil { - return fmt.Errorf("unable to get deploy report path: %w", err) - } else { - deployReportPath = &path + if util.GetBoolEnvironmentDefaultFalse("WERF_EXPERIMENTAL_DEPLOY_ENGINE") { + // FIXME(ilya-lesikov): implement rollback + autoRollback := common.NewBool(cmdData.AutoRollback) + if *autoRollback { + return fmt.Errorf("--auto-rollback and --atomic not yet supported with new deploy engine") } - } - helmUpgradeCmd, _ := helm_v3.NewUpgradeCmd(actionConfig, logboek.OutStream(), helm_v3.UpgradeCmdOptions{ - StagesSplitter: helm.NewStagesSplitter(), - StagesExternalDepsGenerator: helm.NewStagesExternalDepsGenerator(&actionConfig.RESTClientGetter, &namespace), - ChainPostRenderer: wc.ChainPostRenderer, - ValueOpts: valueOpts, - CreateNamespace: common.NewBool(true), - Install: common.NewBool(true), - Wait: common.NewBool(true), - Atomic: common.NewBool(cmdData.AutoRollback), - Timeout: common.NewDuration(time.Duration(cmdData.Timeout) * time.Second), - IgnorePending: common.NewBool(true), - CleanupOnFail: common.NewBool(true), - DeployReportPath: deployReportPath, - }) + var deployReportPath string + if common.GetSaveDeployReport(&commonCmdData) { + deployReportPath, err = common.GetDeployReportPath(&commonCmdData) + if err != nil { + return fmt.Errorf("unable to get deploy report path: %w", err) + } + } + + helmDeployCmd := action.NewDeploy(releaseName, chartDir, actionConfig, helm_v3.Settings, action.DeployOptions{ + OutStream: logboek.OutStream(), + ErrStream: logboek.ErrStream(), + ValueOptions: valueOpts, + ReleaseNamespace: namespace, + RollbackOnFailure: *autoRollback, + TrackTimeout: *common.NewDuration(time.Duration(cmdData.Timeout) * time.Second), + KeepHistoryLimit: *commonCmdData.ReleasesHistoryMax, + DeployReportPath: deployReportPath, + }) - return command_helpers.LockReleaseWrapper(ctx, releaseName, lockManager, func() error { - if err := helmUpgradeCmd.RunE(helmUpgradeCmd, []string{releaseName, filepath.Join(giterminismManager.ProjectDir(), chartDir)}); err != nil { - return fmt.Errorf("helm upgrade have failed: %w", err) + return command_helpers.LockReleaseWrapper(ctx, releaseName, lockManager, func() error { + if err := helmDeployCmd.Run(ctx); err != nil { + return fmt.Errorf("helm deploy failed: %w", err) + } + + return nil + }) + } else { + var deployReportPath *string + if common.GetSaveDeployReport(&commonCmdData) { + if path, err := common.GetDeployReportPath(&commonCmdData); err != nil { + return fmt.Errorf("unable to get deploy report path: %w", err) + } else { + deployReportPath = &path + } } - return nil - }) + + helmUpgradeCmd, _ := helm_v3.NewUpgradeCmd(actionConfig, logboek.OutStream(), helm_v3.UpgradeCmdOptions{ + StagesSplitter: helm.NewStagesSplitter(), + StagesExternalDepsGenerator: helm.NewStagesExternalDepsGenerator(&actionConfig.RESTClientGetter, &namespace), + ChainPostRenderer: wc.ChainPostRenderer, + ValueOpts: valueOpts, + CreateNamespace: common.NewBool(true), + Install: common.NewBool(true), + Wait: common.NewBool(true), + Atomic: common.NewBool(cmdData.AutoRollback), + Timeout: common.NewDuration(time.Duration(cmdData.Timeout) * time.Second), + IgnorePending: common.NewBool(true), + CleanupOnFail: common.NewBool(true), + DeployReportPath: deployReportPath, + }) + + return command_helpers.LockReleaseWrapper(ctx, releaseName, lockManager, func() error { + if err := helmUpgradeCmd.RunE(helmUpgradeCmd, []string{releaseName, filepath.Join(giterminismManager.ProjectDir(), chartDir)}); err != nil { + return fmt.Errorf("helm upgrade have failed: %w", err) + } + return nil + }) + } } func createMaintenanceHelper(ctx context.Context, actionConfig *action.Configuration, kubeConfigOptions kube.KubeConfigOptions) *maintenance_helper.MaintenanceHelper { @@ -545,7 +590,7 @@ func createMaintenanceHelper(ctx context.Context, actionConfig *action.Configura return maintenance_helper.NewMaintenanceHelper(actionConfig, maintenanceOpts) } -func migrateHelm2ToHelm3(ctx context.Context, releaseName, namespace string, maintenanceHelper *maintenance_helper.MaintenanceHelper, chainPostRenderer func(postrender.PostRenderer) postrender.PostRenderer, valueOpts *values.Options, fullChartDir string, helmRegistryClient *registry.Client) error { +func migrateHelm2ToHelm3(ctx context.Context, releaseName, namespace string, maintenanceHelper *maintenance_helper.MaintenanceHelper, chainPostRenderer func(postrender.PostRenderer) postrender.PostRenderer, valueOpts *values.Options, fullChartDir string, helmRegistryClient *registry.Client, extraMutators []mutator.RuntimeResourceMutator) error { if helm2Exists, err := checkHelm2AvailableAndReleaseExists(ctx, releaseName, namespace, maintenanceHelper); err != nil { return fmt.Errorf("error checking availability of helm 2 and existence of helm 2 release %q: %w", releaseName, err) } else if !helm2Exists { @@ -572,7 +617,7 @@ func migrateHelm2ToHelm3(ctx context.Context, releaseName, namespace string, mai logboek.Context(ctx).Default().LogOptionalLn() if err := logboek.Context(ctx).LogProcess("Rendering helm 3 templates for the current project state").DoError(func() error { - actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, &commonCmdData, helmRegistryClient) + actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, &commonCmdData, helmRegistryClient, extraMutators) if err != nil { return err } diff --git a/cmd/werf/dismiss/dismiss.go b/cmd/werf/dismiss/dismiss.go index 10aec037dc..cb5256e0a1 100644 --- a/cmd/werf/dismiss/dismiss.go +++ b/cmd/werf/dismiss/dismiss.go @@ -209,7 +209,7 @@ func runDismiss(ctx context.Context) error { } actionConfig := new(action.Configuration) - if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ + if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), release, namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ StatusProgressPeriod: time.Duration(*commonCmdData.StatusProgressPeriodSeconds) * time.Second, HooksStatusProgressPeriod: time.Duration(*commonCmdData.HooksStatusProgressPeriodSeconds) * time.Second, KubeConfigOptions: kube.KubeConfigOptions{ @@ -219,7 +219,7 @@ func runDismiss(ctx context.Context) error { }, ReleasesHistoryMax: *commonCmdData.ReleasesHistoryMax, RegistryClient: helmRegistryClient, - }); err != nil { + }, nil); err != nil { return err } diff --git a/cmd/werf/helm/helm.go b/cmd/werf/helm/helm.go index bed0b44eef..9416564467 100644 --- a/cmd/werf/helm/helm.go +++ b/cmd/werf/helm/helm.go @@ -200,7 +200,7 @@ func NewCmd(ctx context.Context) (*cobra.Command, error) { common.SetupOndemandKubeInitializer(*_commonCmdData.KubeContext, *_commonCmdData.KubeConfig, *_commonCmdData.KubeConfigBase64, *_commonCmdData.KubeConfigPathMergeList) - helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ + helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), "", namespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ StatusProgressPeriod: time.Duration(*_commonCmdData.StatusProgressPeriodSeconds) * time.Second, HooksStatusProgressPeriod: time.Duration(*_commonCmdData.HooksStatusProgressPeriodSeconds) * time.Second, KubeConfigOptions: kube.KubeConfigOptions{ @@ -210,7 +210,7 @@ func NewCmd(ctx context.Context) (*cobra.Command, error) { ConfigDataBase64: *_commonCmdData.KubeConfigBase64, }, ReleasesHistoryMax: *_commonCmdData.ReleasesHistoryMax, - }) + }, nil) if oldRunE != nil { return oldRunE(cmd, args) diff --git a/cmd/werf/helm/migrate2to3.go b/cmd/werf/helm/migrate2to3.go index cad18a2604..86799b4359 100644 --- a/cmd/werf/helm/migrate2to3.go +++ b/cmd/werf/helm/migrate2to3.go @@ -140,10 +140,10 @@ func runMigrate2To3(ctx context.Context) error { } actionConfig := new(action.Configuration) - if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), targetNamespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ + if err := helm.InitActionConfig(ctx, common.GetOndemandKubeInitializer(), targetReleaseName, targetNamespace, helm_v3.Settings, actionConfig, helm.InitActionConfigOptions{ KubeConfigOptions: kubeConfigOptions, RegistryClient: helmRegistryClient, - }); err != nil { + }, nil); err != nil { return err } diff --git a/cmd/werf/render/render.go b/cmd/werf/render/render.go index 44fde39da0..bca36f40c0 100644 --- a/cmd/werf/render/render.go +++ b/cmd/werf/render/render.go @@ -423,7 +423,7 @@ func runRender(ctx context.Context, imagesToProcess build.ImagesToProcess) error wc.SetServiceValues(vals) } - actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), namespace, &commonCmdData, helmRegistryClient) + actionConfig, err := common.NewActionConfig(ctx, common.GetOndemandKubeInitializer(), releaseName, namespace, &commonCmdData, helmRegistryClient, nil) if err != nil { return err } diff --git a/go.mod b/go.mod index 3643747373..7bd59a8051 100644 --- a/go.mod +++ b/go.mod @@ -347,6 +347,6 @@ replace ( github.com/helm/helm-2to3 => github.com/werf/3p-helm-2to3 v0.0.0-20230313155428-cf9dd655c0e3 // switch back to upstream when merged: https://github.com/helm/helm-2to3/pull/224 github.com/jaguilar/vt100 => github.com/tonistiigi/vt100 v0.0.0-20190402012908-ad4c4a574305 // upstream not maintained github.com/maorfr/helm-plugin-utils => github.com/werf/3p-helm-plugin-utils v0.6.1-0.20230313152239-057595ce9d57 // switch back to upstream when merged: https://github.com/maorfr/helm-plugin-utils/pull/17 - helm.sh/helm/v3 => github.com/werf/3p-helm/v3 v3.0.0-20230309133321-09d8a1e63ebd + helm.sh/helm/v3 => github.com/werf/3p-helm/v3 v3.0.0-20230518112226-7a0b661fec7e k8s.io/helm => github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f ) diff --git a/go.sum b/go.sum index 8b82ef38cc..faeaa92973 100644 --- a/go.sum +++ b/go.sum @@ -1738,8 +1738,8 @@ github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln github.com/weppos/publicsuffix-go v0.20.1-0.20221209102050-40d9c30084b3 h1:ypyhoprZWFzU0ydOBv3I5SS7/jLFJ+ujPAU+BD/EVFM= github.com/werf/3p-helm-2to3 v0.0.0-20230313155428-cf9dd655c0e3 h1:pdKLiuiUoOFswc0HV6GX1+JHgP14HqE035xl2DQJi8A= github.com/werf/3p-helm-2to3 v0.0.0-20230313155428-cf9dd655c0e3/go.mod h1:lTSQBLomjl67m4HQMbg3BHKWes3pOmjNBg8yygXYugk= -github.com/werf/3p-helm/v3 v3.0.0-20230309133321-09d8a1e63ebd h1:wkuBgBCZsQ5XwjGIJugEwUvVarcDdn49GAJxG6EMokg= -github.com/werf/3p-helm/v3 v3.0.0-20230309133321-09d8a1e63ebd/go.mod h1:6n2FO7dTVoTlSkrU+oceTIXZrd8GbPiEshqXHdVIckc= +github.com/werf/3p-helm/v3 v3.0.0-20230518112226-7a0b661fec7e h1:dyyHwYMYuY0UAzFGGPGM1MBYvoD2EcSiw11VTtxoLlY= +github.com/werf/3p-helm/v3 v3.0.0-20230518112226-7a0b661fec7e/go.mod h1:vH6ckglZT6mhJefVLbUM9759Zt/wQnqqRe0BY4jmmHM= github.com/werf/copy-recurse v0.2.7 h1:3FTOarbJ9uhFLi75oeUCioK9zxZwuV7o28kuUBPDZPM= github.com/werf/copy-recurse v0.2.7/go.mod h1:6Ypb+qN+hRBJgoCgEkX1vpbqcQ+8q69BQ3hi8s8Y6Qc= github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f h1:81YscYTF9mmTf0ULOsCmm42YWQp+qWDzWi1HjWniZrg= diff --git a/pkg/deploy/helm/extra_annotations_mutator.go b/pkg/deploy/helm/extra_annotations_mutator.go new file mode 100644 index 0000000000..50be63a83b --- /dev/null +++ b/pkg/deploy/helm/extra_annotations_mutator.go @@ -0,0 +1,42 @@ +package helm + +import ( + "fmt" + + "helm.sh/helm/v3/pkg/werf/common" + "helm.sh/helm/v3/pkg/werf/mutator" + "helm.sh/helm/v3/pkg/werf/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var _ mutator.RuntimeResourceMutator = (*ExtraAnnotationsMutator)(nil) + +func NewExtraAnnotationsMutator(extraAnnos map[string]string) *ExtraAnnotationsMutator { + return &ExtraAnnotationsMutator{ + extraAnnos: extraAnnos, + } +} + +type ExtraAnnotationsMutator struct { + extraAnnos map[string]string +} + +func (m *ExtraAnnotationsMutator) Mutate(res resource.Resourcer, operationType common.ClientOperationType) (resource.Resourcer, error) { + if !res.PartOfRelease() { + return res, nil + } + + switch operationType { + case common.ClientOperationTypeCreate, common.ClientOperationTypeUpdate, common.ClientOperationTypeSmartApply: + default: + return res, nil + } + + for k, v := range m.extraAnnos { + if err := unstructured.SetNestedField(res.Unstructured().UnstructuredContent(), v, "metadata", "annotations", k); err != nil { + return nil, fmt.Errorf("error adding extra annotation: %w", err) + } + } + + return res, nil +} diff --git a/pkg/deploy/helm/extra_labels_mutator.go b/pkg/deploy/helm/extra_labels_mutator.go new file mode 100644 index 0000000000..f4d018fadb --- /dev/null +++ b/pkg/deploy/helm/extra_labels_mutator.go @@ -0,0 +1,42 @@ +package helm + +import ( + "fmt" + + "helm.sh/helm/v3/pkg/werf/common" + "helm.sh/helm/v3/pkg/werf/mutator" + "helm.sh/helm/v3/pkg/werf/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var _ mutator.RuntimeResourceMutator = (*ExtraLabelsMutator)(nil) + +func NewExtraLabelsMutator(extraLabels map[string]string) *ExtraLabelsMutator { + return &ExtraLabelsMutator{ + extraLabels: extraLabels, + } +} + +type ExtraLabelsMutator struct { + extraLabels map[string]string +} + +func (m *ExtraLabelsMutator) Mutate(res resource.Resourcer, operationType common.ClientOperationType) (resource.Resourcer, error) { + if !res.PartOfRelease() { + return res, nil + } + + switch operationType { + case common.ClientOperationTypeCreate, common.ClientOperationTypeUpdate, common.ClientOperationTypeSmartApply: + default: + return res, nil + } + + for k, v := range m.extraLabels { + if err := unstructured.SetNestedField(res.Unstructured().UnstructuredContent(), v, "metadata", "labels", k); err != nil { + return nil, fmt.Errorf("error adding extra labels: %w", err) + } + } + + return res, nil +} diff --git a/pkg/deploy/helm/init.go b/pkg/deploy/helm/init.go index 3e1d6d8d6c..22c4acfd32 100644 --- a/pkg/deploy/helm/init.go +++ b/pkg/deploy/helm/init.go @@ -16,10 +16,16 @@ import ( "helm.sh/helm/v3/pkg/registry" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/storage/driver" + "helm.sh/helm/v3/pkg/werf/client" + "helm.sh/helm/v3/pkg/werf/kubeclient" + "helm.sh/helm/v3/pkg/werf/mutator" + "helm.sh/helm/v3/pkg/werf/resourcetracker" + "helm.sh/helm/v3/pkg/werf/resourcewaiter" "sigs.k8s.io/yaml" "github.com/werf/kubedog/pkg/kube" "github.com/werf/logboek" + "github.com/werf/werf/pkg/util" ) type InitActionConfigOptions struct { @@ -30,7 +36,7 @@ type InitActionConfigOptions struct { RegistryClient *registry.Client } -func InitActionConfig(ctx context.Context, kubeInitializer KubeInitializer, namespace string, envSettings *cli.EnvSettings, actionConfig *action.Configuration, opts InitActionConfigOptions) error { +func InitActionConfig(ctx context.Context, kubeInitializer KubeInitializer, releaseName, namespace string, envSettings *cli.EnvSettings, actionConfig *action.Configuration, opts InitActionConfigOptions, extraMutators []mutator.RuntimeResourceMutator) error { configGetter, err := kube.NewKubeConfigGetter(kube.KubeConfigGetterOptions{ KubeConfigOptions: opts.KubeConfigOptions, Namespace: namespace, @@ -72,6 +78,29 @@ func InitActionConfig(ctx context.Context, kubeInitializer KubeInitializer, name actionConfig.RegistryClient = opts.RegistryClient } + if util.GetBoolEnvironmentDefaultFalse("WERF_EXPERIMENTAL_DEPLOY_ENGINE") { + deferredKubeClient := kubeclient.NewDeferredKubeClient(configGetter) + + waiter := resourcewaiter.NewResourceWaiter(deferredKubeClient.Dynamic(), deferredKubeClient.Mapper()) + + tracker := resourcetracker.NewResourceTracker(opts.StatusProgressPeriod, opts.HooksStatusProgressPeriod) + + cli, err := client.NewClient(deferredKubeClient.Static(), deferredKubeClient.Dynamic(), deferredKubeClient.Discovery(), deferredKubeClient.Mapper(), waiter) + if err != nil { + return fmt.Errorf("error creating client: %w", err) + } + cli.AddTargetResourceMutators(extraMutators...) + cli.AddTargetResourceMutators( + mutator.NewReplicasOnCreationMutator(), + mutator.NewReleaseMetadataMutator(releaseName, namespace), + ) + + actionConfig.DeferredKubeClient = deferredKubeClient + actionConfig.Waiter = waiter + actionConfig.Tracker = tracker + actionConfig.Client = cli + } + return nil } diff --git a/pkg/deploy/helm/service_annotations_mutator.go b/pkg/deploy/helm/service_annotations_mutator.go new file mode 100644 index 0000000000..1fa74402ff --- /dev/null +++ b/pkg/deploy/helm/service_annotations_mutator.go @@ -0,0 +1,52 @@ +package helm + +import ( + "fmt" + + "helm.sh/helm/v3/pkg/werf/common" + "helm.sh/helm/v3/pkg/werf/mutator" + "helm.sh/helm/v3/pkg/werf/resource" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + + "github.com/werf/werf/pkg/werf" +) + +var _ mutator.RuntimeResourceMutator = (*ServiceAnnotationsMutator)(nil) + +func NewServiceAnnotationsMutator(werfEnv, werfProject string) *ServiceAnnotationsMutator { + return &ServiceAnnotationsMutator{ + werfEnv: werfEnv, + werfProject: werfProject, + } +} + +type ServiceAnnotationsMutator struct { + werfEnv string + werfProject string +} + +func (m *ServiceAnnotationsMutator) Mutate(res resource.Resourcer, operationType common.ClientOperationType) (resource.Resourcer, error) { + if !res.PartOfRelease() { + return res, nil + } + + switch operationType { + case common.ClientOperationTypeCreate, common.ClientOperationTypeUpdate, common.ClientOperationTypeSmartApply: + default: + return res, nil + } + + if err := unstructured.SetNestedField(res.Unstructured().UnstructuredContent(), werf.Version, "metadata", "annotations", "werf.io/version"); err != nil { + return nil, fmt.Errorf("error adding werf version annotation: %w", err) + } + + if err := unstructured.SetNestedField(res.Unstructured().UnstructuredContent(), m.werfProject, "metadata", "annotations", "project.werf.io/name"); err != nil { + return nil, fmt.Errorf("error adding werf project name annotation: %w", err) + } + + if err := unstructured.SetNestedField(res.Unstructured().UnstructuredContent(), m.werfEnv, "metadata", "annotations", "project.werf.io/env"); err != nil { + return nil, fmt.Errorf("error adding werf project env annotation: %w", err) + } + + return res, nil +} diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/Chart.lock b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/Chart.lock new file mode 100644 index 0000000000..fd81127590 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/Chart.lock @@ -0,0 +1,10 @@ +dependencies: + - name: hello + repository: oci://ghcr.io/werf + version: 0.1.0 + - name: local-chart + repository: "" + version: 0.1.0 +digest: sha256:fe76d5af7ce7fc3948deb95766ad10eee77d4866c385638934981d118afb316b +generated: "2022-09-19T16:13:01.337342528+03:00" + diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/Chart.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/Chart.yaml new file mode 100644 index 0000000000..5bd84ca317 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/Chart.yaml @@ -0,0 +1,12 @@ +apiVersion: v2 +name: werf-test-e2e-converge-complex +version: 0.1.0 +dependencies: + - name: hello + version: 0.1.0 + repository: oci://ghcr.io/werf + - name: local-chart + version: 0.1.0 + export-values: + - parent: werf + child: werf diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/charts/local-chart/Chart.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/charts/local-chart/Chart.yaml new file mode 100644 index 0000000000..3dab793ca1 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/charts/local-chart/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: local-chart +version: 0.1.0 diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/charts/local-chart/templates/configmap.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/charts/local-chart/templates/configmap.yaml new file mode 100644 index 0000000000..b862251e76 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/charts/local-chart/templates/configmap.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: local-chart-config +data: + werfEnv: {{ .Values.werf.env }} + option: {{ .Values.option }} diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/secret-values.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/secret-values.yaml new file mode 100644 index 0000000000..1dd0f0010a --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/secret-values.yaml @@ -0,0 +1,2 @@ +app1: + secretOption: 1000a493a30a9787a7785a6226baefef32a8d54724f6504ef07202557d5cb4d61a7f11fef08d52f3d9e52003be6e492587ef diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/secret/app1-secret-config.txt b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/secret/app1-secret-config.txt new file mode 100644 index 0000000000..4f93fcd250 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/secret/app1-secret-config.txt @@ -0,0 +1 @@ +100052fb0fa1cc8ef1cb123be089cdb853cc153772691b8fb743d612a7b64d65614d4d8745250752564941227eb8d7161523 diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/_helpers.tpl b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/_helpers.tpl new file mode 100644 index 0000000000..e303f13877 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/_helpers.tpl @@ -0,0 +1,3 @@ +{{- define "app1Name" }} +{{- print "app1" }} +{{- end }} diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/configmap-app1.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/configmap-app1.yaml new file mode 100644 index 0000000000..9f8e391787 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/configmap-app1.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app1-config + annotations: + werf.io/weight: "-10" +data: + werfNamespace: {{ .Values.werf.namespace }} + werfEnv: {{ .Values.werf.env }} + option: {{ .Values.app1.option }} + secretOption: {{ .Values.app1.secretOption }} + secretConfig: {{ werf_secret_file "app1-secret-config.txt" }} diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/deployment-app1.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/deployment-app1.yaml new file mode 100644 index 0000000000..592fe12947 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/deployment-app1.yaml @@ -0,0 +1,26 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "app1Name" . }} + annotations: + sa.external-dependency.werf.io/resource: serviceaccount/default +spec: + selector: + matchLabels: + app: app1 + template: + metadata: + labels: + app: app1 + spec: + imagePullSecrets: + - name: registry + containers: + - name: app + image: {{ .Values.werf.image.app1 | quote }} + command: + - sh + - -ec + - | + echo app1 started + tail -f /dev/null diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/deployment-app2.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/deployment-app2.yaml new file mode 100644 index 0000000000..ab1259b019 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/deployment-app2.yaml @@ -0,0 +1,25 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app2 + annotations: + werf.io/track-termination-mode: NonBlocking + werf.io/fail-mode: IgnoreAndContinueDeployProcess +spec: + selector: + matchLabels: + app: app2 + template: + metadata: + labels: + app: app2 + spec: + containers: + - name: app + image: ubuntu:22.04 + command: + - sh + - -ec + - | + echo app2 started + false diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/hook.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/hook.yaml new file mode 100644 index 0000000000..92b081203d --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/templates/hook.yaml @@ -0,0 +1,15 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: hook + annotations: + helm.sh/hook: pre-upgrade,pre-install +spec: + backoffLimit: 0 + template: + spec: + containers: + - name: hook + image: ubuntu:22.04 + command: ["echo", "hook started"] + restartPolicy: Never diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/values.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/values.yaml new file mode 100644 index 0000000000..47e00c3557 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.helm/values.yaml @@ -0,0 +1,5 @@ +app1: + option: wouldBeOverridden + +local-chart: + option: optionValue diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/.werf_secret_key b/test/e2e/converge-new-engine/_fixtures/complex/state0/.werf_secret_key new file mode 100644 index 0000000000..efd00f5e20 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/.werf_secret_key @@ -0,0 +1 @@ +bc3458408a5687e60b9417adb84e4ad0 diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state0/werf.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state0/werf.yaml new file mode 100644 index 0000000000..36df7841e1 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state0/werf.yaml @@ -0,0 +1,10 @@ +project: werf-test-e2e-converge-complex +configVersion: 1 + +--- +image: app1 +from: ubuntu:22.04 + +--- +image: app2 +from: ubuntu:22.04 diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/Chart.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/Chart.yaml new file mode 100644 index 0000000000..25ea8ee1be --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: werf-test-e2e-converge-complex +version: 0.1.0 diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/templates/configmap-app1.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/templates/configmap-app1.yaml new file mode 100644 index 0000000000..3d246d4d24 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/templates/configmap-app1.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: app1-config +data: + option2: {{ .Values.app1.option }} diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/templates/hook.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/templates/hook.yaml new file mode 100644 index 0000000000..92b081203d --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state1/.helm/templates/hook.yaml @@ -0,0 +1,15 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: hook + annotations: + helm.sh/hook: pre-upgrade,pre-install +spec: + backoffLimit: 0 + template: + spec: + containers: + - name: hook + image: ubuntu:22.04 + command: ["echo", "hook started"] + restartPolicy: Never diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state1/werf.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state1/werf.yaml new file mode 100644 index 0000000000..5ee3a85c48 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state1/werf.yaml @@ -0,0 +1,2 @@ +project: werf-test-e2e-converge-complex +configVersion: 1 diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state2/.helm/Chart.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state2/.helm/Chart.yaml new file mode 100644 index 0000000000..25ea8ee1be --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state2/.helm/Chart.yaml @@ -0,0 +1,3 @@ +apiVersion: v2 +name: werf-test-e2e-converge-complex +version: 0.1.0 diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state2/.helm/templates/deployment-app2.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state2/.helm/templates/deployment-app2.yaml new file mode 100644 index 0000000000..98b4973e14 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state2/.helm/templates/deployment-app2.yaml @@ -0,0 +1,24 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: app2 + annotations: + werf.io/failures-allowed-per-replica: "0" +spec: + selector: + matchLabels: + app: app2 + template: + metadata: + labels: + app: app2 + spec: + containers: + - name: app + image: ubuntu:22.04 + command: + - sh + - -ec + - | + echo app2 started + false diff --git a/test/e2e/converge-new-engine/_fixtures/complex/state2/werf.yaml b/test/e2e/converge-new-engine/_fixtures/complex/state2/werf.yaml new file mode 100644 index 0000000000..5ee3a85c48 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/complex/state2/werf.yaml @@ -0,0 +1,2 @@ +project: werf-test-e2e-converge-complex +configVersion: 1 diff --git a/test/e2e/converge-new-engine/_fixtures/simple/state0/.helm/templates/configmap.yaml b/test/e2e/converge-new-engine/_fixtures/simple/state0/.helm/templates/configmap.yaml new file mode 100644 index 0000000000..82a3d481cb --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/simple/state0/.helm/templates/configmap.yaml @@ -0,0 +1,6 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: test1 +data: + key1: value1 diff --git a/test/e2e/converge-new-engine/_fixtures/simple/state0/werf.yaml b/test/e2e/converge-new-engine/_fixtures/simple/state0/werf.yaml new file mode 100644 index 0000000000..01ea971430 --- /dev/null +++ b/test/e2e/converge-new-engine/_fixtures/simple/state0/werf.yaml @@ -0,0 +1,2 @@ +project: werf-test-e2e-converge-simple +configVersion: 1 diff --git a/test/e2e/converge-new-engine/common_test.go b/test/e2e/converge-new-engine/common_test.go new file mode 100644 index 0000000000..31e48a4350 --- /dev/null +++ b/test/e2e/converge-new-engine/common_test.go @@ -0,0 +1,22 @@ +package e2e_converge_test + +import ( + "os" + "strings" + + "github.com/werf/werf/pkg/util" +) + +func setupEnv() { + SuiteData.Stubs.SetEnv("WERF_REPO", strings.Join([]string{os.Getenv("WERF_TEST_K8S_DOCKER_REGISTRY"), SuiteData.ProjectName}, "/")) + SuiteData.Stubs.SetEnv("WERF_ENV", "test") + SuiteData.Stubs.SetEnv("WERF_EXPERIMENTAL_DEPLOY_ENGINE", "1") + + if util.GetBoolEnvironmentDefaultFalse("WERF_TEST_K8S_DOCKER_REGISTRY_INSECURE") { + SuiteData.Stubs.SetEnv("WERF_INSECURE_REGISTRY", "1") + SuiteData.Stubs.SetEnv("WERF_SKIP_TLS_VERIFY_REGISTRY", "1") + } else { + SuiteData.Stubs.UnsetEnv("WERF_INSECURE_REGISTRY") + SuiteData.Stubs.UnsetEnv("WERF_SKIP_TLS_VERIFY_REGISTRY") + } +} diff --git a/test/e2e/converge-new-engine/complex_test.go b/test/e2e/converge-new-engine/complex_test.go new file mode 100644 index 0000000000..8c00f0d0c2 --- /dev/null +++ b/test/e2e/converge-new-engine/complex_test.go @@ -0,0 +1,222 @@ +package e2e_converge_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "helm.sh/helm/v3/pkg/release" + appsv1 "k8s.io/api/apps/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + + "github.com/werf/kubedog/pkg/kube" + "github.com/werf/werf/test/pkg/utils" + "github.com/werf/werf/test/pkg/werf" +) + +var _ = Describe("Complex converge", Label("e2e", "converge", "complex"), func() { + var repoDirname string + + AfterEach(func() { + utils.RunSucceedCommand(SuiteData.GetTestRepoPath(repoDirname), SuiteData.WerfBinPath, "dismiss", "--with-namespace") + }) + + It("should complete and deploy expected resources", + func() { + By("initializing") + repoDirname = "repo0" + setupEnv() + + By("state0: starting") + { + fixtureRelPath := "complex/state0" + deployReportName := ".werf-deploy-report.json" + + By("state0: preparing test repo") + SuiteData.InitTestRepo(repoDirname, fixtureRelPath) + werfProject := werf.NewProject(SuiteData.WerfBinPath, SuiteData.GetTestRepoPath(repoDirname)) + + By("state0: prepare namespace") + werfProject.CreateNamespace() + werfProject.CreateRegistryPullSecretFromDockerConfig() + + By("state0: execute converge") + _, deployReport := werfProject.ConvergeWithReport(SuiteData.GetDeployReportPath(deployReportName), &werf.ConvergeWithReportOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{ + "--set=app1.option=optionValue", + }, + }, + }) + + By("state0: check deploy report") + Expect(deployReport.Release).To(Equal(werfProject.Release())) + Expect(deployReport.Namespace).To(Equal(werfProject.Namespace())) + Expect(deployReport.Revision).To(Equal(1)) + Expect(deployReport.Status).To(Equal(release.StatusDeployed)) + + By("state0: check deployed resources in cluster") + cm, err := kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get(context.Background(), "app1-config", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(Equal(map[string]string{ + "werfNamespace": werfProject.Namespace(), + "werfEnv": "test", + "option": "optionValue", + "secretOption": "secretOptionValue", + "secretConfig": "secretConfigContent", + })) + checkServiceLabelsAndAnnos(cm.Labels, cm.Annotations, werfProject) + + cm, err = kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get(context.Background(), "local-chart-config", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(Equal(map[string]string{ + "werfEnv": "test", + "option": "optionValue", + })) + checkServiceLabelsAndAnnos(cm.Labels, cm.Annotations, werfProject) + + cm, err = kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get(context.Background(), "hello", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(Equal(map[string]string{"hello": "world"})) + checkServiceLabelsAndAnnos(cm.Labels, cm.Annotations, werfProject) + + deployment, err := kube.Client.AppsV1().Deployments(werfProject.Namespace()).Get(context.Background(), "app1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(deploymentAvailable(deployment)).To(BeTrue()) + checkServiceLabelsAndAnnos(deployment.Labels, deployment.Annotations, werfProject) + + deployment, err = kube.Client.AppsV1().Deployments(werfProject.Namespace()).Get(context.Background(), "app2", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(deploymentAvailable(deployment)).To(BeFalse()) + checkServiceLabelsAndAnnos(deployment.Labels, deployment.Annotations, werfProject) + } + + By("state1: starting") + { + fixtureRelPath := "complex/state1" + deployReportName := ".werf-deploy-report.json" + + By("state1: preparing test repo") + SuiteData.UpdateTestRepo(repoDirname, fixtureRelPath) + werfProject := werf.NewProject(SuiteData.WerfBinPath, SuiteData.GetTestRepoPath(repoDirname)) + + By("state1: patch configmap in cluster, emulating manual changes by a user") + _, err := kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Patch( + context.Background(), + "app1-config", + types.StrategicMergePatchType, + []byte(`{"data":{"option3": "setInClusterValue"}}`), + metav1.PatchOptions{}, + ) + Expect(err).NotTo(HaveOccurred()) + + By("state1: execute converge") + _, deployReport := werfProject.ConvergeWithReport(SuiteData.GetDeployReportPath(deployReportName), &werf.ConvergeWithReportOptions{ + CommonOptions: werf.CommonOptions{ + ExtraArgs: []string{ + "--set=app1.option=optionValue", + }, + }, + }) + + By("state1: check deploy report") + Expect(deployReport.Revision).To(Equal(2)) + Expect(deployReport.Status).To(Equal(release.StatusDeployed)) + + By("state1: check deployed configmap in cluster") + cm, err := kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get(context.Background(), "app1-config", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(Equal(map[string]string{ + "option2": "optionValue", + "option3": "setInClusterValue", + })) + checkServiceLabelsAndAnnos(cm.Labels, cm.Annotations, werfProject) + + By("state1: check removed resources in cluster") + resourceShouldNotExist(kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get( + context.Background(), "local-chart-config", metav1.GetOptions{}, + )) + resourceShouldNotExist(kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get( + context.Background(), "hello", metav1.GetOptions{}, + )) + resourceShouldNotExist(kube.Client.AppsV1().Deployments(werfProject.Namespace()).Get( + context.Background(), "app1", metav1.GetOptions{}, + )) + resourceShouldNotExist(kube.Client.AppsV1().Deployments(werfProject.Namespace()).Get( + context.Background(), "app2", metav1.GetOptions{}, + )) + } + + By("state2: starting") + { + fixtureRelPath := "complex/state2" + deployReportName := ".werf-deploy-report.json" + + By("state2: preparing test repo") + SuiteData.UpdateTestRepo(repoDirname, fixtureRelPath) + werfProject := werf.NewProject(SuiteData.WerfBinPath, SuiteData.GetTestRepoPath(repoDirname)) + + By("state2: execute converge") + _, deployReport := werfProject.ConvergeWithReport(SuiteData.GetDeployReportPath(deployReportName), &werf.ConvergeWithReportOptions{ + CommonOptions: werf.CommonOptions{ + ShouldFail: true, + }, + }) + + By("state2: check deploy report") + Expect(deployReport.Revision).To(Equal(3)) + Expect(deployReport.Status).To(Equal(release.StatusFailed)) + + By("state2: check deployed deployment in cluster") + deployment, err := kube.Client.AppsV1().Deployments(werfProject.Namespace()).Get(context.Background(), "app2", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(deploymentAvailable(deployment)).To(BeFalse()) + checkServiceLabelsAndAnnos(deployment.Labels, deployment.Annotations, werfProject) + + By("state2: check removed resources in cluster") + resourceShouldNotExist(kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get( + context.Background(), "app1-config", metav1.GetOptions{}, + )) + resourceShouldNotExist(kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get( + context.Background(), "local-chart-config", metav1.GetOptions{}, + )) + resourceShouldNotExist(kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get( + context.Background(), "hello", metav1.GetOptions{}, + )) + resourceShouldNotExist(kube.Client.AppsV1().Deployments(werfProject.Namespace()).Get( + context.Background(), "app1", metav1.GetOptions{}, + )) + resourceShouldNotExist(kube.Client.AppsV1().Deployments(werfProject.Namespace()).Get( + context.Background(), "app2", metav1.GetOptions{}, + )) + } + }, + ) +}) + +func checkServiceLabelsAndAnnos(labels, annotations map[string]string, werfProject *werf.Project) { + Expect(labels).To(HaveKeyWithValue("app.kubernetes.io/managed-by", "Helm")) + + Expect(annotations).To(HaveKeyWithValue("meta.helm.sh/release-name", werfProject.Release())) + Expect(annotations).To(HaveKeyWithValue("meta.helm.sh/release-namespace", werfProject.Namespace())) + Expect(annotations).To(HaveKeyWithValue("werf.io/version", "dev")) + Expect(annotations).To(HaveKeyWithValue("project.werf.io/env", "test")) +} + +func deploymentAvailable(deployment *appsv1.Deployment) bool { + for _, cond := range deployment.Status.Conditions { + if cond.Type == "Available" { + return cond.Status == "True" + } + } + + return false +} + +func resourceShouldNotExist(_ interface{}, err error) { + if !apierrors.IsNotFound(err) { + Expect(err).NotTo(HaveOccurred()) + } +} diff --git a/test/e2e/converge-new-engine/simple_test.go b/test/e2e/converge-new-engine/simple_test.go new file mode 100644 index 0000000000..f393a412ef --- /dev/null +++ b/test/e2e/converge-new-engine/simple_test.go @@ -0,0 +1,54 @@ +package e2e_converge_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "helm.sh/helm/v3/pkg/release" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/werf/kubedog/pkg/kube" + "github.com/werf/werf/test/pkg/utils" + "github.com/werf/werf/test/pkg/werf" +) + +var _ = Describe("Simple converge", Label("e2e", "converge", "simple"), func() { + var repoDirname string + + AfterEach(func() { + utils.RunSucceedCommand(SuiteData.GetTestRepoPath(repoDirname), SuiteData.WerfBinPath, "dismiss", "--with-namespace") + }) + + It("should succeed and deploy expected resources", + func() { + By("initializing") + repoDirname = "repo0" + setupEnv() + + By("state0: starting") + { + fixtureRelPath := "simple/state0" + deployReportName := ".werf-deploy-report.json" + + By("state0: preparing test repo") + SuiteData.InitTestRepo(repoDirname, fixtureRelPath) + werfProject := werf.NewProject(SuiteData.WerfBinPath, SuiteData.GetTestRepoPath(repoDirname)) + + By("state0: execute converge") + _, deployReport := werfProject.ConvergeWithReport(SuiteData.GetDeployReportPath(deployReportName), &werf.ConvergeWithReportOptions{}) + + By("state0: check deploy report") + Expect(deployReport.Release).To(Equal(werfProject.Release())) + Expect(deployReport.Namespace).To(Equal(werfProject.Namespace())) + Expect(deployReport.Revision).To(Equal(1)) + Expect(deployReport.Status).To(Equal(release.StatusDeployed)) + + By("state0: check deployed resources in cluster") + cm, err := kube.Client.CoreV1().ConfigMaps(werfProject.Namespace()).Get(context.Background(), "test1", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + Expect(cm.Data).To(Equal(map[string]string{"key1": "value1"})) + } + }, + ) +}) diff --git a/test/e2e/converge-new-engine/suite_test.go b/test/e2e/converge-new-engine/suite_test.go new file mode 100644 index 0000000000..3dbac6cf0b --- /dev/null +++ b/test/e2e/converge-new-engine/suite_test.go @@ -0,0 +1,41 @@ +package e2e_converge_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/kubedog/pkg/kube" + "github.com/werf/werf/test/pkg/suite_init" + "github.com/werf/werf/test/pkg/utils" +) + +func TestSuite(t *testing.T) { + suite_init.MakeTestSuiteEntrypointFunc("E2E converge suite", suite_init.TestSuiteEntrypointFuncOptions{ + RequiredSuiteTools: []string{"docker", "git"}, + RequiredSuiteEnvs: []string{ + "WERF_TEST_K8S_DOCKER_REGISTRY", + }, + })(t) +} + +var SuiteData suite_init.SuiteData + +var ( + _ = SuiteData.SetupStubs(suite_init.NewStubsData()) + _ = SuiteData.SetupSynchronizedSuiteCallbacks(suite_init.NewSynchronizedSuiteCallbacksData()) + _ = SuiteData.SetupWerfBinary(suite_init.NewWerfBinaryData(SuiteData.SynchronizedSuiteCallbacksData)) + _ = SuiteData.SetupProjectName(suite_init.NewProjectNameData(SuiteData.StubsData)) + _ = SuiteData.SetupTmp(suite_init.NewTmpDirData()) + + _ = SuiteData.SetupK8sDockerRegistry(suite_init.NewK8sDockerRegistryData(SuiteData.ProjectNameData, SuiteData.StubsData)) + + _ = BeforeEach(func() { + Expect(kube.Init(kube.InitOptions{})).To(Succeed()) + }) + + _ = AfterEach(func() { + utils.RunSucceedCommand("", SuiteData.WerfBinPath, "host", "purge", "--force", "--project-name", SuiteData.ProjectName) + }) +) diff --git a/test/pkg/suite_init/suite_data.go b/test/pkg/suite_init/suite_data.go index a03141a14d..7e086bf302 100644 --- a/test/pkg/suite_init/suite_data.go +++ b/test/pkg/suite_init/suite_data.go @@ -66,6 +66,12 @@ func (data *SuiteData) GetBuildReportPath(filename string) string { return filepath.Join(buildReportsDir, filename) } +func (data *SuiteData) GetDeployReportPath(filename string) string { + deployReportsDir := filepath.Join(data.TestDirPath, "deploy-reports") + Expect(os.MkdirAll(deployReportsDir, os.ModePerm)).To(Succeed()) + return filepath.Join(deployReportsDir, filename) +} + func (data *SuiteData) InitTestRepo(dirname, fixtureRelPath string) { testRepoPath := data.GetTestRepoPath(dirname) utils.CopyIn(utils.FixturePath(fixtureRelPath), testRepoPath) diff --git a/test/pkg/werf/project.go b/test/pkg/werf/project.go index b537dd6b8d..b1d7668030 100644 --- a/test/pkg/werf/project.go +++ b/test/pkg/werf/project.go @@ -10,6 +10,7 @@ import ( "sync" . "github.com/onsi/gomega" + "helm.sh/helm/v3/pkg/release" "github.com/werf/werf/pkg/build" iutils "github.com/werf/werf/test/pkg/utils" @@ -48,6 +49,10 @@ type ConvergeOptions struct { CommonOptions } +type ConvergeWithReportOptions struct { + CommonOptions +} + type KubeRunOptions struct { CommonOptions Command []string @@ -102,6 +107,23 @@ func (p *Project) Converge(opts *ConvergeOptions) (combinedOut string) { return string(outb) } +func (p *Project) ConvergeWithReport(deployReportPath string, opts *ConvergeWithReportOptions) (combinedOut string, report release.DeployReport) { + if opts == nil { + opts = &ConvergeWithReportOptions{} + } + + args := append([]string{"converge", "--save-deploy-report", "--deploy-report-path", deployReportPath}, opts.ExtraArgs...) + out := p.runCommand(runCommandOptions{Args: args, ShouldFail: opts.ShouldFail}) + + deployReportRaw, err := os.ReadFile(deployReportPath) + Expect(err).NotTo(HaveOccurred()) + + var deployReport release.DeployReport + Expect(json.Unmarshal(deployReportRaw, &deployReport)).To(Succeed()) + + return out, deployReport +} + func (p *Project) KubeRun(opts *KubeRunOptions) string { if opts == nil { opts = &KubeRunOptions{}