diff --git a/cmd/werf/build/main.go b/cmd/werf/build/main.go index a16e7dd4ca..c581ac27c3 100644 --- a/cmd/werf/build/main.go +++ b/cmd/werf/build/main.go @@ -63,7 +63,7 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image common.LogVersion() return common.LogRunningTime(func() error { - return runMain(ctx, args) + return runMain(ctx, common.GetImagesToProcess(args, false)) }) }, })) @@ -123,7 +123,7 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image return cmd } -func runMain(ctx context.Context, args []string) error { +func runMain(ctx context.Context, imagesToProcess build.ImagesToProcess) error { global_warnings.PostponeMultiwerfNotUpToDateWarning() if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil { @@ -187,25 +187,24 @@ func runMain(ctx context.Context, args []string) error { if *commonCmdData.Follow { logboek.LogOptionalLn() return common.FollowGitHead(ctx, &commonCmdData, func(ctx context.Context, headCommitGiterminismManager giterminism_manager.Interface) error { - return run(ctx, containerBackend, headCommitGiterminismManager, args) + return run(ctx, containerBackend, headCommitGiterminismManager, imagesToProcess) }) } else { - return run(ctx, containerBackend, giterminismManager, args) + return run(ctx, containerBackend, giterminismManager, imagesToProcess) } } -func run(ctx context.Context, containerBackend container_backend.ContainerBackend, giterminismManager giterminism_manager.Interface, imagesToProcess []string) error { +func run(ctx context.Context, containerBackend container_backend.ContainerBackend, giterminismManager giterminism_manager.Interface, imagesToProcess build.ImagesToProcess) error { _, werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, true)) if err != nil { return fmt.Errorf("unable to load werf config: %w", err) } - - projectName := werfConfig.Meta.Project - - if err := werfConfig.CheckThatImagesExist(imagesToProcess); err != nil { + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { return err } + projectName := werfConfig.Meta.Project + projectTmpDir, err := tmp_manager.CreateProjectDir(ctx) if err != nil { return fmt.Errorf("getting project tmp dir failed: %w", err) @@ -245,14 +244,14 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken return err } - conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions) + conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, imagesToProcess, buildOptions) if err != nil { return err } logboek.LogOptionalLn() - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, imagesToProcess, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) defer conveyorWithRetry.Terminate() if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { diff --git a/cmd/werf/bundle/export/export.go b/cmd/werf/bundle/export/export.go index fba6431d8a..3a90895807 100644 --- a/cmd/werf/bundle/export/export.go +++ b/cmd/werf/bundle/export/export.go @@ -40,7 +40,7 @@ var commonCmdData common.CmdData func NewCmd(ctx context.Context) *cobra.Command { ctx = common.NewContextWithCmdData(ctx, &commonCmdData) cmd := common.SetCommandContext(ctx, &cobra.Command{ - Use: "export", + Use: "export [IMAGE_NAME...]", Short: "Export bundle", Hidden: true, // Deprecated command Long: common.GetLongCommandDescription(`Export bundle into the provided directory (or into directory named as a resulting chart in the current working directory). werf bundle contains built images defined in the werf.yaml, helm chart, service values which contain built images tags, any custom values and set values params provided during publish invocation, werf service templates and values.`), @@ -76,11 +76,13 @@ func NewCmd(ctx context.Context) *cobra.Command { common.LogVersion() return common.LogRunningTime(func() error { - return runExport(ctx) + return runExport(ctx, common.GetImagesToProcess(args, *commonCmdData.WithoutImages)) }) }, }) + commonCmdData.SetupWithoutImages(cmd) + common.SetupDir(&commonCmdData, cmd) common.SetupGitWorkTree(&commonCmdData, cmd) common.SetupGiterminismOptions(&commonCmdData, cmd) @@ -142,7 +144,7 @@ func NewCmd(ctx context.Context) *cobra.Command { return cmd } -func runExport(ctx context.Context) error { +func runExport(ctx context.Context, imagesToProcess build.ImagesToProcess) error { if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil { return fmt.Errorf("initialization error: %w", err) } @@ -195,6 +197,9 @@ func runExport(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to load werf config: %w", err) } + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { + return err + } projectName := werfConfig.Meta.Project @@ -239,7 +244,7 @@ func runExport(ctx context.Context) error { var imagesInfoGetters []*image.InfoGetter var imagesRepo string - if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 { + if !imagesToProcess.WithoutImages && (len(werfConfig.StapelImages)+len(werfConfig.ImagesFromDockerfile) > 0) { stagesStorage, err := common.GetStagesStorage(ctx, containerBackend, &commonCmdData) if err != nil { return err @@ -273,12 +278,12 @@ func runExport(ctx context.Context) error { imagesRepo = storageManager.GetServiceValuesRepo() - conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions) + conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, imagesToProcess, buildOptions) if err != nil { return err } - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, nil, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) defer conveyorWithRetry.Terminate() if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { diff --git a/cmd/werf/bundle/publish/publish.go b/cmd/werf/bundle/publish/publish.go index deef65a69b..a5628ef817 100644 --- a/cmd/werf/bundle/publish/publish.go +++ b/cmd/werf/bundle/publish/publish.go @@ -43,7 +43,7 @@ var commonCmdData common.CmdData func NewCmd(ctx context.Context) *cobra.Command { ctx = common.NewContextWithCmdData(ctx, &commonCmdData) cmd := common.SetCommandContext(ctx, &cobra.Command{ - Use: "publish", + Use: "publish [IMAGE_NAME...]", Short: "Publish bundle", Long: common.GetLongCommandDescription(`Publish bundle into the container registry. werf bundle contains built images defined in the werf.yaml, helm chart, service values which contain built images tags, any custom values and set values params provided during publish invocation, werf addon templates (like werf_image). @@ -65,10 +65,12 @@ Published into container registry bundle can be rolled out by the "werf bundle" common.LogVersion() - return common.LogRunningTime(func() error { return runPublish(ctx) }) + return common.LogRunningTime(func() error { return runPublish(ctx, common.GetImagesToProcess(args, *commonCmdData.WithoutImages)) }) }, }) + commonCmdData.SetupWithoutImages(cmd) + common.SetupDir(&commonCmdData, cmd) common.SetupGitWorkTree(&commonCmdData, cmd) common.SetupGiterminismOptions(&commonCmdData, cmd) @@ -139,7 +141,7 @@ Published into container registry bundle can be rolled out by the "werf bundle" return cmd } -func runPublish(ctx context.Context) error { +func runPublish(ctx context.Context, imagesToProcess build.ImagesToProcess) error { global_warnings.PostponeMultiwerfNotUpToDateWarning() if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil { @@ -194,6 +196,9 @@ func runPublish(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to load werf config: %w", err) } + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { + return err + } projectName := werfConfig.Meta.Project @@ -252,7 +257,7 @@ func runPublish(ctx context.Context) error { var imagesInfoGetters []*image.InfoGetter var imagesRepo string - if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 { + if !imagesToProcess.WithoutImages && (len(werfConfig.StapelImages)+len(werfConfig.ImagesFromDockerfile) > 0) { synchronization, err := common.GetSynchronization(ctx, &commonCmdData, projectName, stagesStorage) if err != nil { return err @@ -278,12 +283,12 @@ func runPublish(ctx context.Context) error { imagesRepo = storageManager.GetServiceValuesRepo() - conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions) + conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, imagesToProcess, buildOptions) if err != nil { return err } - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, nil, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) defer conveyorWithRetry.Terminate() if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { diff --git a/cmd/werf/common/cmd_data.go b/cmd/werf/common/cmd_data.go index 642203e6d9..4fb7c52803 100644 --- a/cmd/werf/common/cmd_data.go +++ b/cmd/werf/common/cmd_data.go @@ -1,5 +1,11 @@ package common +import ( + "github.com/spf13/cobra" + + "github.com/werf/werf/pkg/util" +) + type CmdData struct { GitWorkTree *string ProjectName *string @@ -32,8 +38,9 @@ type CmdData struct { SecretValues *[]string IgnoreSecretKey *bool - Repo *RepoData - FinalRepo *RepoData + WithoutImages *bool + Repo *RepoData + FinalRepo *RepoData SecondaryStagesStorage *[]string CacheStagesStorage *[]string @@ -92,3 +99,8 @@ type CmdData struct { Platform *string } + +func (cmdData *CmdData) SetupWithoutImages(cmd *cobra.Command) { + cmdData.WithoutImages = new(bool) + cmd.Flags().BoolVarP(cmdData.WithoutImages, "without-images", "", util.GetBoolEnvironmentDefaultFalse("WERF_WITHOUT_IMAGES"), "Disable building of images defined in the werf.yaml (if any) and usage of such images in the .helm/templates ($WERF_WITHOUT_IMAGES or false by default — e.g. enable all images defined in the werf.yaml by default)") +} diff --git a/cmd/werf/common/conveyor_options.go b/cmd/werf/common/conveyor_options.go index b0ac7070a0..8040318b67 100644 --- a/cmd/werf/common/conveyor_options.go +++ b/cmd/werf/common/conveyor_options.go @@ -16,16 +16,17 @@ import ( "github.com/werf/werf/pkg/storage" ) -func GetConveyorOptions(commonCmdData *CmdData) build.ConveyorOptions { +func GetConveyorOptions(commonCmdData *CmdData, imagesToProcess build.ImagesToProcess) build.ConveyorOptions { return build.ConveyorOptions{ LocalGitRepoVirtualMergeOptions: stage.VirtualMergeOptions{ VirtualMerge: *commonCmdData.VirtualMerge, }, + ImagesToProcess: imagesToProcess, } } -func GetConveyorOptionsWithParallel(commonCmdData *CmdData, buildStagesOptions build.BuildOptions) (build.ConveyorOptions, error) { - conveyorOptions := GetConveyorOptions(commonCmdData) +func GetConveyorOptionsWithParallel(commonCmdData *CmdData, imagesToProcess build.ImagesToProcess, buildStagesOptions build.BuildOptions) (build.ConveyorOptions, error) { + conveyorOptions := GetConveyorOptions(commonCmdData, imagesToProcess) conveyorOptions.Parallel = !(buildStagesOptions.ImageBuildOptions.IntrospectAfterError || buildStagesOptions.ImageBuildOptions.IntrospectBeforeError || len(buildStagesOptions.Targets) != 0) && *commonCmdData.Parallel parallelTasksLimit, err := GetParallelTasksLimit(commonCmdData) diff --git a/cmd/werf/common/process_images.go b/cmd/werf/common/process_images.go new file mode 100644 index 0000000000..9451311b1e --- /dev/null +++ b/cmd/werf/common/process_images.go @@ -0,0 +1,12 @@ +package common + +import "github.com/werf/werf/pkg/build" + +func GetImagesToProcess(onlyImages []string, withoutImages bool) build.ImagesToProcess { + if withoutImages { + return build.NewImagesToProcess(nil, true) + } else if len(onlyImages) > 0 { + return build.NewImagesToProcess(onlyImages, false) + } + return build.NewImagesToProcess(nil, false) +} diff --git a/cmd/werf/compose/main.go b/cmd/werf/compose/main.go index f25baa4b85..dd9dfb4b48 100644 --- a/cmd/werf/compose/main.go +++ b/cmd/werf/compose/main.go @@ -198,6 +198,8 @@ services: }, }) + commonCmdData.SetupWithoutImages(cmd) + common.SetupDir(&commonCmdData, cmd) common.SetupGitWorkTree(&commonCmdData, cmd) common.SetupConfigTemplatesDir(&commonCmdData, cmd) @@ -372,78 +374,82 @@ func runMain(ctx context.Context, dockerComposeCmdName string, cmdData composeCm } func run(ctx context.Context, containerBackend container_backend.ContainerBackend, giterminismManager giterminism_manager.Interface, commonCmdData common.CmdData, cmdData composeCmdData, dockerComposeCmdName string) error { + imagesToProcess := common.GetImagesToProcess(cmdData.WerfImagesToProcess, *commonCmdData.WithoutImages) + _, werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, true)) if err != nil { return fmt.Errorf("unable to load werf config: %w", err) } - - if err := werfConfig.CheckThatImagesExist(cmdData.WerfImagesToProcess); err != nil { + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { return err } projectName := werfConfig.Meta.Project - projectTmpDir, err := tmp_manager.CreateProjectDir(ctx) - if err != nil { - return fmt.Errorf("getting project tmp dir failed: %w", err) - } - defer tmp_manager.ReleaseProjectDir(projectTmpDir) + var envArray []string - stagesStorage, err := common.GetStagesStorage(ctx, containerBackend, &commonCmdData) - if err != nil { - return err - } - finalStagesStorage, err := common.GetOptionalFinalStagesStorage(ctx, containerBackend, &commonCmdData) - if err != nil { - return err - } - synchronization, err := common.GetSynchronization(ctx, &commonCmdData, projectName, stagesStorage) - if err != nil { - return err - } - storageLockManager, err := common.GetStorageLockManager(ctx, synchronization) - if err != nil { - return err - } - secondaryStagesStorageList, err := common.GetSecondaryStagesStorageList(ctx, stagesStorage, containerBackend, &commonCmdData) - if err != nil { - return err - } - cacheStagesStorageList, err := common.GetCacheStagesStorageList(ctx, containerBackend, &commonCmdData) - if err != nil { - return err - } + if !imagesToProcess.WithoutImages && (len(werfConfig.StapelImages)+len(werfConfig.ImagesFromDockerfile) > 0) { + projectTmpDir, err := tmp_manager.CreateProjectDir(ctx) + if err != nil { + return fmt.Errorf("getting project tmp dir failed: %w", err) + } + defer tmp_manager.ReleaseProjectDir(projectTmpDir) + + stagesStorage, err := common.GetStagesStorage(ctx, containerBackend, &commonCmdData) + if err != nil { + return err + } + finalStagesStorage, err := common.GetOptionalFinalStagesStorage(ctx, containerBackend, &commonCmdData) + if err != nil { + return err + } + synchronization, err := common.GetSynchronization(ctx, &commonCmdData, projectName, stagesStorage) + if err != nil { + return err + } + storageLockManager, err := common.GetStorageLockManager(ctx, synchronization) + if err != nil { + return err + } + secondaryStagesStorageList, err := common.GetSecondaryStagesStorageList(ctx, stagesStorage, containerBackend, &commonCmdData) + if err != nil { + return err + } + cacheStagesStorageList, err := common.GetCacheStagesStorageList(ctx, containerBackend, &commonCmdData) + if err != nil { + return err + } - storageManager := manager.NewStorageManager(projectName, stagesStorage, finalStagesStorage, secondaryStagesStorageList, cacheStagesStorageList, storageLockManager) + storageManager := manager.NewStorageManager(projectName, stagesStorage, finalStagesStorage, secondaryStagesStorageList, cacheStagesStorageList, storageLockManager) - logboek.Context(ctx).Info().LogOptionalLn() + logboek.Context(ctx).Info().LogOptionalLn() - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, cmdData.WerfImagesToProcess, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData)) - defer conveyorWithRetry.Terminate() + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData, imagesToProcess)) + defer conveyorWithRetry.Terminate() - var envArray []string - if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { - if *commonCmdData.SkipBuild { - if err := c.ShouldBeBuilt(ctx, build.ShouldBeBuiltOptions{}); err != nil { - return err - } - } else { - if err := c.Build(ctx, build.BuildOptions{SkipImageMetadataPublication: *commonCmdData.Dev}); err != nil { - return err + if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { + if *commonCmdData.SkipBuild { + if err := c.ShouldBeBuilt(ctx, build.ShouldBeBuiltOptions{}); err != nil { + return err + } + } else { + if err := c.Build(ctx, build.BuildOptions{SkipImageMetadataPublication: *commonCmdData.Dev}); err != nil { + return err + } } - } - for _, imageName := range c.GetExportedImagesNames() { - if err := c.FetchLastImageStage(ctx, imageName); err != nil { - return err + for _, imageName := range c.GetExportedImagesNames() { + if err := c.FetchLastImageStage(ctx, imageName); err != nil { + return err + } } - } - envArray = c.GetImagesEnvArray() + envArray = c.GetImagesEnvArray() - return nil - }); err != nil { - return err + return nil + }); err != nil { + return err + } } var dockerComposeArgs []string diff --git a/cmd/werf/config/graph/graph.go b/cmd/werf/config/graph/graph.go index b8f0d59207..812ddc7732 100644 --- a/cmd/werf/config/graph/graph.go +++ b/cmd/werf/config/graph/graph.go @@ -86,12 +86,17 @@ func NewCmd(ctx context.Context) *cobra.Command { return err } + imagesToProcess := common.GetImagesToProcess(args, false) + _, werfConfig, err := config.GetWerfConfig(ctx, customWerfConfigRelPath, customWerfConfigTemplatesDirRelPath, giterminismManager, configOpts) if err != nil { return err } + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { + return err + } - graphList, err := werfConfig.GetImageGraphList(args) + graphList, err := werfConfig.GetImageGraphList(imagesToProcess.OnlyImages, imagesToProcess.WithoutImages) if err != nil { return err } diff --git a/cmd/werf/converge/converge.go b/cmd/werf/converge/converge.go index 7c84bf3327..d85d7c483d 100644 --- a/cmd/werf/converge/converge.go +++ b/cmd/werf/converge/converge.go @@ -54,7 +54,7 @@ var commonCmdData common.CmdData func NewCmd(ctx context.Context) *cobra.Command { ctx = common.NewContextWithCmdData(ctx, &commonCmdData) cmd := common.SetCommandContext(ctx, &cobra.Command{ - Use: "converge", + Use: "converge [IMAGE_NAME...]", Short: "Build and push images, then deploy application into Kubernetes", Long: common.GetLongCommandDescription(`Build and push images, then deploy application into Kubernetes. @@ -82,11 +82,13 @@ werf converge --repo registry.mydomain.com/web --env production`, common.LogVersion() return common.LogRunningTime(func() error { - return runMain(ctx) + return runMain(ctx, common.GetImagesToProcess(args, *commonCmdData.WithoutImages)) }) }, }) + commonCmdData.SetupWithoutImages(cmd) + common.SetupDir(&commonCmdData, cmd) common.SetupGitWorkTree(&commonCmdData, cmd) common.SetupConfigTemplatesDir(&commonCmdData, cmd) @@ -169,7 +171,7 @@ werf converge --repo registry.mydomain.com/web --env production`, return cmd } -func runMain(ctx context.Context) error { +func runMain(ctx context.Context, imagesToProcess build.ImagesToProcess) error { global_warnings.PostponeMultiwerfNotUpToDateWarning() if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil { @@ -238,18 +240,21 @@ func runMain(ctx context.Context) error { if *commonCmdData.Follow { logboek.LogOptionalLn() return common.FollowGitHead(ctx, &commonCmdData, func(ctx context.Context, headCommitGiterminismManager giterminism_manager.Interface) error { - return run(ctx, containerBackend, headCommitGiterminismManager) + return run(ctx, containerBackend, headCommitGiterminismManager, imagesToProcess) }) } else { - return run(ctx, containerBackend, giterminismManager) + return run(ctx, containerBackend, giterminismManager, imagesToProcess) } } -func run(ctx context.Context, containerBackend container_backend.ContainerBackend, giterminismManager giterminism_manager.Interface) error { +func run(ctx context.Context, containerBackend container_backend.ContainerBackend, giterminismManager giterminism_manager.Interface, imagesToProcess build.ImagesToProcess) error { werfConfigPath, werfConfig, err := common.GetRequiredWerfConfig(ctx, &commonCmdData, giterminismManager, common.GetWerfConfigOptions(&commonCmdData, true)) if err != nil { return fmt.Errorf("unable to load werf config: %w", err) } + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { + return err + } chartDir, err := common.GetHelmChartDir(werfConfigPath, werfConfig, giterminismManager) if err != nil { @@ -271,7 +276,8 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken var imagesInfoGetters []*image.InfoGetter var imagesRepo string - if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 { + + if !imagesToProcess.WithoutImages && (len(werfConfig.StapelImages)+len(werfConfig.ImagesFromDockerfile) > 0) { stagesStorage, err := common.GetStagesStorage(ctx, containerBackend, &commonCmdData) if err != nil { return err @@ -306,12 +312,12 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken imagesRepo = storageManager.GetServiceValuesRepo() - conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions) + conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, imagesToProcess, buildOptions) if err != nil { return err } - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, nil, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) defer conveyorWithRetry.Terminate() if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { diff --git a/cmd/werf/export/export.go b/cmd/werf/export/export.go index 05d33f0ca6..4dd68b2ebf 100644 --- a/cmd/werf/export/export.go +++ b/cmd/werf/export/export.go @@ -62,7 +62,7 @@ All meta-information related to werf is removed from the exported images, and th return fmt.Errorf("required at least one tag template: use the --tag option to specify templates") } - return run(ctx, args, tagTemplateList) + return run(ctx, common.GetImagesToProcess(args, false), tagTemplateList) }, }) @@ -110,7 +110,11 @@ It is necessary to use image name shortcut %image% or %image_slug% if multiple i return cmd } -func run(ctx context.Context, imagesToProcess, tagTemplateList []string) error { +func run(ctx context.Context, imagesToProcess build.ImagesToProcess, tagTemplateList []string) error { + if imagesToProcess.WithoutImages { + return nil + } + if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil { return fmt.Errorf("initialization error: %w", err) } @@ -167,8 +171,7 @@ func run(ctx context.Context, imagesToProcess, tagTemplateList []string) error { if err != nil { return fmt.Errorf("unable to load werf config: %w", err) } - - if err := werfConfig.CheckThatImagesExist(imagesToProcess); err != nil { + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { return err } @@ -209,17 +212,23 @@ func run(ctx context.Context, imagesToProcess, tagTemplateList []string) error { logboek.Context(ctx).Info().LogOptionalLn() - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, imagesToProcess, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData)) + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData, imagesToProcess)) defer conveyorWithRetry.Terminate() return conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { - if len(imagesToProcess) == 0 { - for _, img := range werfConfig.GetAllImages() { - imagesToProcess = append(imagesToProcess, img.GetName()) + var imageNameList []string + + if !imagesToProcess.WithoutImages { + if imagesToProcess.OnlyImages == nil { + for _, img := range werfConfig.GetAllImages() { + imageNameList = append(imageNameList, img.GetName()) + } + } else { + imageNameList = append(imageNameList, imagesToProcess.OnlyImages...) } } - tagFuncList, err := getTagFuncList(imagesToProcess, tagTemplateList) + tagFuncList, err := getTagFuncList(imageNameList, tagTemplateList) if err != nil { return err } diff --git a/cmd/werf/helm/get_autogenerated_values.go b/cmd/werf/helm/get_autogenerated_values.go index 5ad9445564..c70ad0a4e0 100644 --- a/cmd/werf/helm/get_autogenerated_values.go +++ b/cmd/werf/helm/get_autogenerated_values.go @@ -29,7 +29,7 @@ var getAutogeneratedValuedCmdData common.CmdData func NewGetAutogeneratedValuesCmd(ctx context.Context) *cobra.Command { ctx = common.NewContextWithCmdData(ctx, &getAutogeneratedValuedCmdData) cmd := common.SetCommandContext(ctx, &cobra.Command{ - Use: "get-autogenerated-values", + Use: "get-autogenerated-values [IMAGE_NAME...]", Short: "Get service values yaml generated by werf for helm chart during deploy", Long: common.GetLongCommandDescription(`Get service values generated by werf for helm chart during deploy. @@ -46,10 +46,12 @@ These values includes project name, docker images ids and other`), return err } - return runGetServiceValues(ctx) + return runGetServiceValues(ctx, common.GetImagesToProcess(args, *getAutogeneratedValuedCmdData.WithoutImages)) }, }) + getAutogeneratedValuedCmdData.SetupWithoutImages(cmd) + common.SetupDir(&getAutogeneratedValuedCmdData, cmd) common.SetupGitWorkTree(&getAutogeneratedValuedCmdData, cmd) common.SetupConfigTemplatesDir(&getAutogeneratedValuedCmdData, cmd) @@ -90,7 +92,7 @@ These values includes project name, docker images ids and other`), return cmd } -func runGetServiceValues(ctx context.Context) error { +func runGetServiceValues(ctx context.Context, imagesToProcess build.ImagesToProcess) error { logboek.SetAcceptedLevel(level.Error) if err := werf.Init(*getAutogeneratedValuedCmdData.TmpDir, *getAutogeneratedValuedCmdData.HomeDir); err != nil { @@ -137,6 +139,9 @@ func runGetServiceValues(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to load werf config: %w", err) } + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { + return err + } projectName := werfConfig.Meta.Project environment := *getAutogeneratedValuedCmdData.Environment @@ -158,10 +163,11 @@ func runGetServiceValues(ctx context.Context) error { var imagesRepository string var imagesInfoGetters []*image.InfoGetter + if *getAutogeneratedValuedCmdData.StubTags { imagesInfoGetters = common.StubImageInfoGetters(werfConfig) imagesRepository = common.StubRepoAddress - } else { + } else if !imagesToProcess.WithoutImages && (len(werfConfig.StapelImages)+len(werfConfig.ImagesFromDockerfile) > 0) { projectTmpDir, err := tmp_manager.CreateProjectDir(ctx) if err != nil { return fmt.Errorf("getting project tmp dir failed: %w", err) @@ -203,7 +209,7 @@ func runGetServiceValues(ctx context.Context) error { storageManager := manager.NewStorageManager(projectName, stagesStorage, finalStagesStorage, secondaryStagesStorageList, cacheStagesStorageList, storageLockManager) - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, []string{}, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&getAutogeneratedValuedCmdData)) + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&getAutogeneratedValuedCmdData, imagesToProcess)) defer conveyorWithRetry.Terminate() if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { diff --git a/cmd/werf/kube_run/kube_run.go b/cmd/werf/kube_run/kube_run.go index 7e889223df..8d09d25ca7 100644 --- a/cmd/werf/kube_run/kube_run.go +++ b/cmd/werf/kube_run/kube_run.go @@ -393,7 +393,9 @@ func run(ctx context.Context, pod, secret, namespace string, werfConfig *config. logboek.Context(ctx).Info().LogOptionalLn() - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, []string{imageName}, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData)) + imagesToProcess := build.NewImagesToProcess([]string{imageName}, false) + + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData, imagesToProcess)) defer conveyorWithRetry.Terminate() var image string diff --git a/cmd/werf/render/render.go b/cmd/werf/render/render.go index 94f087aadd..7de3a4101b 100644 --- a/cmd/werf/render/render.go +++ b/cmd/werf/render/render.go @@ -50,7 +50,7 @@ var commonCmdData common.CmdData func NewCmd(ctx context.Context) *cobra.Command { ctx = common.NewContextWithCmdData(ctx, &commonCmdData) cmd := common.SetCommandContext(ctx, &cobra.Command{ - Use: "render", + Use: "render [IMAGE_NAME...]", Short: "Render Kubernetes templates", Long: common.GetLongCommandDescription(`Render Kubernetes templates. This command will calculate digests and build (if needed) all images defined in the werf.yaml.`), DisableFlagsInUseLine: true, @@ -73,10 +73,12 @@ func NewCmd(ctx context.Context) *cobra.Command { common.LogVersion() - return common.LogRunningTime(func() error { return runRender(ctx) }) + return common.LogRunningTime(func() error { return runRender(ctx, common.GetImagesToProcess(args, *commonCmdData.WithoutImages)) }) }, }) + commonCmdData.SetupWithoutImages(cmd) + common.SetupDir(&commonCmdData, cmd) common.SetupGitWorkTree(&commonCmdData, cmd) common.SetupConfigTemplatesDir(&commonCmdData, cmd) @@ -153,7 +155,7 @@ func getShowOnly() []string { return append(util.PredefinedValuesByEnvNamePrefix("WERF_SHOW_ONLY"), cmdData.ShowOnly...) } -func runRender(ctx context.Context) error { +func runRender(ctx context.Context, imagesToProcess build.ImagesToProcess) error { if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil { return fmt.Errorf("initialization error: %w", err) } @@ -196,6 +198,9 @@ func runRender(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to load werf config: %w", err) } + if err := werfConfig.CheckThatImagesExist(imagesToProcess.OnlyImages); err != nil { + return err + } projectName := werfConfig.Meta.Project @@ -254,7 +259,7 @@ func runRender(ctx context.Context) error { var isStub bool var stubImagesNames []string - if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 { + if !imagesToProcess.WithoutImages && (len(werfConfig.StapelImages)+len(werfConfig.ImagesFromDockerfile) > 0) { addr, err := commonCmdData.Repo.GetAddress() if err != nil { return err @@ -298,12 +303,12 @@ func runRender(ctx context.Context) error { imagesRepo = storageManager.GetServiceValuesRepo() - conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, buildOptions) + conveyorOptions, err := common.GetConveyorOptionsWithParallel(&commonCmdData, imagesToProcess, buildOptions) if err != nil { return err } - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, nil, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, conveyorOptions) defer conveyorWithRetry.Terminate() if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { diff --git a/cmd/werf/run/run.go b/cmd/werf/run/run.go index fac37f56c6..7e416f9ff9 100644 --- a/cmd/werf/run/run.go +++ b/cmd/werf/run/run.go @@ -379,7 +379,9 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken logboek.Context(ctx).Info().LogOptionalLn() - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, []string{imageName}, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData)) + imagesToProcess := build.NewImagesToProcess([]string{imageName}, false) + + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData, imagesToProcess)) defer conveyorWithRetry.Terminate() var dockerImageName string diff --git a/cmd/werf/stage/image/main.go b/cmd/werf/stage/image/main.go index 7a758280e0..44db24b857 100644 --- a/cmd/werf/stage/image/main.go +++ b/cmd/werf/stage/image/main.go @@ -192,7 +192,9 @@ func run(ctx context.Context, imageName string) error { storageManager := manager.NewStorageManager(projectName, stagesStorage, finalStagesStorage, secondaryStagesStorageList, cacheStagesStorageList, storageLockManager) - conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, []string{imageName}, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData)) + imagesToProcess := build.NewImagesToProcess([]string{imageName}, false) + + conveyorWithRetry := build.NewConveyorWithRetryWrapper(werfConfig, giterminismManager, giterminismManager.ProjectDir(), projectTmpDir, ssh_agent.SSHAuthSock, containerBackend, storageManager, storageLockManager, common.GetConveyorOptions(&commonCmdData, imagesToProcess)) defer conveyorWithRetry.Terminate() if err := conveyorWithRetry.WithRetryBlock(ctx, func(c *build.Conveyor) error { diff --git a/pkg/build/conveyor.go b/pkg/build/conveyor.go index 38d62b7284..f83c71b648 100644 --- a/pkg/build/conveyor.go +++ b/pkg/build/conveyor.go @@ -37,8 +37,7 @@ import ( ) type Conveyor struct { - werfConfig *config.WerfConfig - imageNamesToProcess []string + werfConfig *config.WerfConfig projectDir string containerWerfDir string @@ -77,12 +76,13 @@ type ConveyorOptions struct { Parallel bool ParallelTasksLimit int64 LocalGitRepoVirtualMergeOptions stage.VirtualMergeOptions + + ImagesToProcess } -func NewConveyor(werfConfig *config.WerfConfig, giterminismManager giterminism_manager.Interface, imageNamesToProcess []string, projectDir, baseTmpDir, sshAuthSock string, containerBackend container_backend.ContainerBackend, storageManager manager.StorageManagerInterface, storageLockManager storage.LockManager, opts ConveyorOptions) *Conveyor { +func NewConveyor(werfConfig *config.WerfConfig, giterminismManager giterminism_manager.Interface, projectDir, baseTmpDir, sshAuthSock string, containerBackend container_backend.ContainerBackend, storageManager manager.StorageManagerInterface, storageLockManager storage.LockManager, opts ConveyorOptions) *Conveyor { return &Conveyor{ - werfConfig: werfConfig, - imageNamesToProcess: imageNamesToProcess, + werfConfig: werfConfig, projectDir: projectDir, containerWerfDir: "/.werf", @@ -452,7 +452,7 @@ func (c *Conveyor) determineStages(ctx context.Context) error { } func (c *Conveyor) doDetermineStages(ctx context.Context) error { - imageConfigSets, err := c.werfConfig.GroupImagesByIndependentSets(c.imageNamesToProcess) + imageConfigSets, err := c.werfConfig.GroupImagesByIndependentSets(c.ConveyorOptions.OnlyImages, c.ConveyorOptions.WithoutImages) if err != nil { return err } diff --git a/pkg/build/conveyor_with_retry.go b/pkg/build/conveyor_with_retry.go index e442f9cf82..f4fd3ae0e1 100644 --- a/pkg/build/conveyor_with_retry.go +++ b/pkg/build/conveyor_with_retry.go @@ -11,31 +11,29 @@ import ( ) type ConveyorWithRetryWrapper struct { - WerfConfig *config.WerfConfig - GiterminismManager giterminism_manager.Interface - ImageNamesToProcess []string - ProjectDir string - BaseTmpDir string - SshAuthSock string - ContainerBackend container_backend.ContainerBackend - StorageManager *manager.StorageManager - StorageLockManager storage.LockManager + WerfConfig *config.WerfConfig + GiterminismManager giterminism_manager.Interface + ProjectDir string + BaseTmpDir string + SshAuthSock string + ContainerBackend container_backend.ContainerBackend + StorageManager *manager.StorageManager + StorageLockManager storage.LockManager ConveyorOptions ConveyorOptions } -func NewConveyorWithRetryWrapper(werfConfig *config.WerfConfig, giterminismManager giterminism_manager.Interface, imageNamesToProcess []string, projectDir, baseTmpDir, sshAuthSock string, containerBackend container_backend.ContainerBackend, storageManager *manager.StorageManager, storageLockManager storage.LockManager, opts ConveyorOptions) *ConveyorWithRetryWrapper { +func NewConveyorWithRetryWrapper(werfConfig *config.WerfConfig, giterminismManager giterminism_manager.Interface, projectDir, baseTmpDir, sshAuthSock string, containerBackend container_backend.ContainerBackend, storageManager *manager.StorageManager, storageLockManager storage.LockManager, opts ConveyorOptions) *ConveyorWithRetryWrapper { return &ConveyorWithRetryWrapper{ - WerfConfig: werfConfig, - GiterminismManager: giterminismManager, - ImageNamesToProcess: imageNamesToProcess, - ProjectDir: projectDir, - BaseTmpDir: baseTmpDir, - SshAuthSock: sshAuthSock, - ContainerBackend: containerBackend, - StorageManager: storageManager, - StorageLockManager: storageLockManager, - ConveyorOptions: opts, + WerfConfig: werfConfig, + GiterminismManager: giterminismManager, + ProjectDir: projectDir, + BaseTmpDir: baseTmpDir, + SshAuthSock: sshAuthSock, + ContainerBackend: containerBackend, + StorageManager: storageManager, + StorageLockManager: storageLockManager, + ConveyorOptions: opts, } } @@ -48,7 +46,6 @@ func (wrapper *ConveyorWithRetryWrapper) WithRetryBlock(ctx context.Context, f f newConveyor := NewConveyor( wrapper.WerfConfig, wrapper.GiterminismManager, - wrapper.ImageNamesToProcess, wrapper.ProjectDir, wrapper.BaseTmpDir, wrapper.SshAuthSock, diff --git a/pkg/build/images_to_process.go b/pkg/build/images_to_process.go new file mode 100644 index 0000000000..772fa3a12a --- /dev/null +++ b/pkg/build/images_to_process.go @@ -0,0 +1,10 @@ +package build + +type ImagesToProcess struct { + OnlyImages []string + WithoutImages bool +} + +func NewImagesToProcess(onlyImages []string, withoutImages bool) ImagesToProcess { + return ImagesToProcess{OnlyImages: onlyImages, WithoutImages: withoutImages} +} diff --git a/pkg/config/werf.go b/pkg/config/werf.go index a98fe22ef1..fbae3b1c4e 100644 --- a/pkg/config/werf.go +++ b/pkg/config/werf.go @@ -38,18 +38,20 @@ func (c *WerfConfig) CheckThatImagesExist(names []string) error { return nil } -func (c *WerfConfig) GetSpecificImages(names []string) ([]ImageInterface, error) { +func (c *WerfConfig) GetSpecificImages(onlyImages []string, withoutImages bool) ([]ImageInterface, error) { var imageConfigsToProcess []ImageInterface - if len(names) == 0 { + if withoutImages { + return nil, nil + } else if onlyImages == nil { return c.GetAllImages(), nil } - if err := c.CheckThatImagesExist(names); err != nil { + if err := c.CheckThatImagesExist(onlyImages); err != nil { return nil, err } - for _, name := range names { + for _, name := range onlyImages { var imageToProcess ImageInterface imageToProcess = c.GetImage(name) if imageToProcess == nil { @@ -352,8 +354,8 @@ func (c *WerfConfig) validateInfiniteLoopBetweenRelatedImages() error { return nil } -func (c *WerfConfig) GroupImagesByIndependentSets(names []string) (sets [][]ImageInterface, err error) { - images, err := c.GetSpecificImages(names) +func (c *WerfConfig) GroupImagesByIndependentSets(onlyImages []string, withoutImages bool) (sets [][]ImageInterface, err error) { + images, err := c.GetSpecificImages(onlyImages, withoutImages) if err != nil { return nil, err } @@ -550,8 +552,8 @@ type imageGraph struct { } `yaml:"dependsOn,omitempty"` } -func (c *WerfConfig) GetImageGraphList(names []string) ([]imageGraph, error) { - images, err := c.GetSpecificImages(names) +func (c *WerfConfig) GetImageGraphList(onlyImages []string, withoutImages bool) ([]imageGraph, error) { + images, err := c.GetSpecificImages(onlyImages, withoutImages) if err != nil { return nil, err }