From 16dbd2e8e9a2f3e15bb85f598a009b479a636188 Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Mon, 25 Apr 2022 17:15:36 +0300 Subject: [PATCH] feat(bundle): implement 'bundle copy' command ``` werf bundle copy --repo REPO [--tag latest] --to NEWREPO [--to-tag NEWTAG] ``` Signed-off-by: Timofey Kirillov --- cmd/werf/build/main.go | 7 +- cmd/werf/bundle/apply/apply.go | 6 +- cmd/werf/bundle/copy/copy.go | 116 ++++++ cmd/werf/bundle/download/download.go | 8 +- cmd/werf/bundle/export/export.go | 11 +- cmd/werf/bundle/publish/publish.go | 8 +- cmd/werf/bundle/render/render.go | 16 +- cmd/werf/cleanup/cleanup.go | 8 +- cmd/werf/common/common.go | 184 ++-------- cmd/werf/common/conveyor_options.go | 2 +- cmd/werf/common/repo_data.go | 352 +++++++++++++++++++ cmd/werf/common/storage_repo_data.go | 256 -------------- cmd/werf/compose/main.go | 7 +- cmd/werf/converge/converge.go | 10 +- cmd/werf/dismiss/dismiss.go | 4 +- cmd/werf/export/export.go | 7 +- cmd/werf/helm/get_autogenerated_values.go | 8 +- cmd/werf/host/purge/purge.go | 5 +- cmd/werf/kube_run/kube_run.go | 10 +- cmd/werf/main.go | 2 + cmd/werf/managed_images/add/add.go | 11 +- cmd/werf/managed_images/ls/ls.go | 10 +- cmd/werf/managed_images/rm/rm.go | 10 +- cmd/werf/purge/purge.go | 9 +- cmd/werf/render/render.go | 13 +- cmd/werf/run/run.go | 7 +- cmd/werf/stage/image/main.go | 7 +- pkg/deploy/bundles/copy.go | 128 +++++++ pkg/deploy/helm/chart_extender/werf_chart.go | 4 +- pkg/storage/repo_stages_storage.go | 14 +- pkg/storage/stages_storage.go | 12 - test/pkg/utils/storage.go | 20 +- 32 files changed, 721 insertions(+), 551 deletions(-) create mode 100644 cmd/werf/bundle/copy/copy.go create mode 100644 cmd/werf/common/repo_data.go delete mode 100644 cmd/werf/common/storage_repo_data.go create mode 100644 pkg/deploy/bundles/copy.go diff --git a/cmd/werf/build/main.go b/cmd/werf/build/main.go index e5431979c7..8bf58a4974 100644 --- a/cmd/werf/build/main.go +++ b/cmd/werf/build/main.go @@ -81,8 +81,8 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{OptionalRepo: true}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo, to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -214,8 +214,7 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken } defer tmp_manager.ReleaseProjectDir(projectTmpDir) - stagesStorageAddress := common.GetOptionalStagesStorageAddress(&commonCmdData) - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/bundle/apply/apply.go b/cmd/werf/bundle/apply/apply.go index 57d3583cfc..29ad49c447 100644 --- a/cmd/werf/bundle/apply/apply.go +++ b/cmd/werf/bundle/apply/apply.go @@ -61,8 +61,8 @@ func NewCmd() *cobra.Command { common.SetupTmpDir(&commonCmdData, cmd, common.SetupTmpDirOptions{}) common.SetupHomeDir(&commonCmdData, cmd, common.SetupHomeDirOptions{}) - common.SetupStagesStorageOptions(&commonCmdData, cmd) // FIXME - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo, to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -128,7 +128,7 @@ func runApply() error { return err } - repoAddress, err := common.GetStagesStorageAddress(&commonCmdData) + repoAddress, err := commonCmdData.Repo.GetAddress() if err != nil { return err } diff --git a/cmd/werf/bundle/copy/copy.go b/cmd/werf/bundle/copy/copy.go new file mode 100644 index 0000000000..d9b9cf98c7 --- /dev/null +++ b/cmd/werf/bundle/copy/copy.go @@ -0,0 +1,116 @@ +package copy + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + helm_v3 "helm.sh/helm/v3/cmd/helm" + + "github.com/werf/werf/cmd/werf/common" + "github.com/werf/werf/pkg/deploy/bundles" + "github.com/werf/werf/pkg/werf" + "github.com/werf/werf/pkg/werf/global_warnings" +) + +var cmdData struct { + Repo *common.RepoData + Tag string + To *common.RepoData + ToTag string +} + +var commonCmdData common.CmdData + +func NewCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "copy", + Short: "Copy published bundle into another location", + Long: common.GetLongCommandDescription(`Take latest bundle from the specified container registry using specified version tag and copy it either into a different tag within the same container registry or into another container registry.`), + DisableFlagsInUseLine: true, + Annotations: map[string]string{ + common.CmdEnvAnno: common.EnvsDescription(), + }, + RunE: func(cmd *cobra.Command, args []string) error { + defer global_warnings.PrintGlobalWarnings(common.GetContext()) + + if err := common.ProcessLogOptions(&commonCmdData); err != nil { + common.PrintHelp(cmd) + return err + } + + common.LogVersion() + + return common.LogRunningTime(runCopy) + }, + } + + common.SetupTmpDir(&commonCmdData, cmd, common.SetupTmpDirOptions{}) + common.SetupHomeDir(&commonCmdData, cmd, common.SetupHomeDirOptions{}) + + common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repos") + common.SetupInsecureRegistry(&commonCmdData, cmd) + common.SetupInsecureHelmDependencies(&commonCmdData, cmd) + common.SetupSkipTlsVerifyRegistry(&commonCmdData, cmd) + + cmdData.Repo = common.NewRepoData("repo", common.RepoDataOptions{OnlyAddress: true}) + cmdData.Repo.SetupCmd(cmd) + + cmdData.To = common.NewRepoData("to", common.RepoDataOptions{OnlyAddress: true}) + cmdData.To.SetupCmd(cmd) + + common.SetupLogOptions(&commonCmdData, cmd) + common.SetupLogProjectDir(&commonCmdData, cmd) + common.SetupPlatform(&commonCmdData, cmd) + + defaultTag := os.Getenv("WERF_TAG") + if defaultTag == "" { + defaultTag = "latest" + } + cmd.Flags().StringVarP(&cmdData.Tag, "tag", "", defaultTag, "Provide from tag version of the bundle to copy ($WERF_TAG or latest by default)") + cmd.Flags().StringVarP(&cmdData.ToTag, "to-tag", "", "", "Provide to tag version of the bundle to copy ($WERF_TO_TAG or same as --tag by default)") + + return cmd +} + +func runCopy() error { + ctx := common.GetContext() + + if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil { + return fmt.Errorf("initialization error: %w", err) + } + + if err := common.DockerRegistryInit(ctx, &commonCmdData); err != nil { + return err + } + + helm_v3.Settings.Debug = *commonCmdData.LogDebug + + bundlesRegistryClient, err := common.NewBundlesRegistryClient(ctx, &commonCmdData) + if err != nil { + return err + } + + if *cmdData.Repo.Address == "" { + return fmt.Errorf("--repo=ADDRESS param required") + } + if *cmdData.To.Address == "" { + return fmt.Errorf("--to=ADDRESS param required") + } + + fromRegistry, err := cmdData.Repo.CreateDockerRegistry(*commonCmdData.InsecureRegistry, *commonCmdData.SkipTlsVerifyRegistry) + if err != nil { + return fmt.Errorf("error creating container registry accessor for repo %s: %w", *cmdData.Repo.Address, err) + } + + fromTag := cmdData.Tag + toTag := cmdData.ToTag + if toTag == "" { + toTag = fromTag + } + + fromRef := fmt.Sprintf("%s:%s", *cmdData.Repo.Address, fromTag) + toRef := fmt.Sprintf("%s:%s", *cmdData.To.Address, toTag) + + return bundles.Copy(ctx, fromRef, toRef, bundlesRegistryClient, fromRegistry) +} diff --git a/cmd/werf/bundle/download/download.go b/cmd/werf/bundle/download/download.go index f1821f9bc2..99385c5292 100644 --- a/cmd/werf/bundle/download/download.go +++ b/cmd/werf/bundle/download/download.go @@ -5,7 +5,7 @@ import ( "os" "github.com/spf13/cobra" - "helm.sh/helm/v3/cmd/helm" + helm_v3 "helm.sh/helm/v3/cmd/helm" "github.com/werf/werf/cmd/werf/common" "github.com/werf/werf/pkg/deploy/bundles" @@ -50,8 +50,8 @@ func NewCmd() *cobra.Command { common.SetupInsecureHelmDependencies(&commonCmdData, cmd) common.SetupSkipTlsVerifyRegistry(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) // FIXME - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupLogOptions(&commonCmdData, cmd) common.SetupLogProjectDir(&commonCmdData, cmd) @@ -77,7 +77,7 @@ func runDownload() error { return err } - repoAddress, err := common.GetStagesStorageAddress(&commonCmdData) + repoAddress, err := commonCmdData.Repo.GetAddress() if err != nil { return err } diff --git a/cmd/werf/bundle/export/export.go b/cmd/werf/bundle/export/export.go index 74bc2642d2..4d352ae7f1 100644 --- a/cmd/werf/bundle/export/export.go +++ b/cmd/werf/bundle/export/export.go @@ -82,8 +82,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo and to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -223,16 +223,11 @@ func runExport(ctx context.Context) error { logboek.LogOptionalLn() - repoAddress, err := common.GetStagesStorageAddress(&commonCmdData) - if err != nil { - return err - } - var imagesInfoGetters []*image.InfoGetter var imagesRepository string if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 { - stagesStorage, err := common.GetStagesStorage(repoAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/bundle/publish/publish.go b/cmd/werf/bundle/publish/publish.go index 056caed589..efb4ccb978 100644 --- a/cmd/werf/bundle/publish/publish.go +++ b/cmd/werf/bundle/publish/publish.go @@ -88,8 +88,8 @@ Published into container registry bundle can be rolled out by the "werf bundle" common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo and to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -239,7 +239,7 @@ func runPublish(ctx context.Context) error { logboek.LogOptionalLn() - repoAddress, err := common.GetStagesStorageAddress(&commonCmdData) + repoAddress, err := commonCmdData.Repo.GetAddress() if err != nil { return err } @@ -253,7 +253,7 @@ func runPublish(ctx context.Context) error { var imagesRepository string if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 { - stagesStorage, err := common.GetStagesStorage(repoAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/bundle/render/render.go b/cmd/werf/bundle/render/render.go index a0ffeb2bf9..34d1f4ec88 100644 --- a/cmd/werf/bundle/render/render.go +++ b/cmd/werf/bundle/render/render.go @@ -9,7 +9,7 @@ import ( uuid "github.com/satori/go.uuid" "github.com/spf13/cobra" - "helm.sh/helm/v3/cmd/helm" + helm_v3 "helm.sh/helm/v3/cmd/helm" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" @@ -64,8 +64,8 @@ func NewCmd() *cobra.Command { common.SetupTmpDir(&commonCmdData, cmd, common.SetupTmpDirOptions{}) common.SetupHomeDir(&commonCmdData, cmd, common.SetupHomeDirOptions{}) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo, to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -114,17 +114,17 @@ func runRender(ctx context.Context) error { var isLocal bool switch { case cmdData.BundleDir != "": - if *commonCmdData.StagesStorage != "" { + if *commonCmdData.Repo.Address != "" { return fmt.Errorf("only one of --bundle-dir or --repo should be specified, but both provided") } - if *commonCmdData.FinalStagesStorage != "" { + if *commonCmdData.FinalRepo.Address != "" { return fmt.Errorf("only one of --bundle-dir or --final-repo should be specified, but both provided") } isLocal = true - case *commonCmdData.StagesStorage == storage.LocalStorageAddress: + case *commonCmdData.Repo.Address == storage.LocalStorageAddress: return fmt.Errorf("--repo %s is not allowed, specify remote storage address", storage.LocalStorageAddress) - case *commonCmdData.StagesStorage != "": + case *commonCmdData.Repo.Address != "": isLocal = false default: return fmt.Errorf("either --bundle-dir or --repo required") @@ -165,7 +165,7 @@ func runRender(ctx context.Context) error { return err } - repoAddress, err := common.GetStagesStorageAddress(&commonCmdData) + repoAddress, err := commonCmdData.Repo.GetAddress() if err != nil { return err } diff --git a/cmd/werf/cleanup/cleanup.go b/cmd/werf/cleanup/cleanup.go index c0bda5d68b..ddc3a7fec2 100644 --- a/cmd/werf/cleanup/cleanup.go +++ b/cmd/werf/cleanup/cleanup.go @@ -65,8 +65,8 @@ It is safe to run this command periodically (daily is enough) by automated clean common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultCleanupParallelTasksLimit) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and delete images from the specified repo") @@ -189,14 +189,14 @@ func runCleanup(ctx context.Context) error { projectName := werfConfig.Meta.Project - stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData) + _, err = commonCmdData.Repo.GetAddress() if err != nil { logboek.Context(ctx).Default().LogLnDetails(`The "werf cleanup" command is only used to cleaning the container registry. In case you need to clean the runner or the localhost, use the commands of the "werf host" group. It is worth noting that auto-cleaning is enabled by default, and manual use is usually not required (if not, we would appreciate feedback and creating an issue https://github.com/werf/werf/issues/new).`) return err } - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/common/common.go b/cmd/werf/common/common.go index 68aea229f1..0e9b10214e 100644 --- a/cmd/werf/common/common.go +++ b/cmd/werf/common/common.go @@ -64,11 +64,8 @@ type CmdData struct { SecretValues *[]string IgnoreSecretKey *bool - CommonRepoData *RepoData - StagesStorage *string - - CommonFinalRepoData *RepoData - FinalStagesStorage *string + Repo *RepoData + FinalRepo *RepoData SecondaryStagesStorage *[]string CacheStagesStorage *[]string @@ -390,34 +387,6 @@ func GetFirstExistingEnvVarAsString(envNames ...string) string { return "" } -func SetupCommonRepoData(cmdData *CmdData, cmd *cobra.Command) { - cmdData.CommonRepoData = &RepoData{IsCommon: true} - - SetupImplementationForRepoData(cmdData.CommonRepoData, cmd, "repo-implementation", []string{"WERF_REPO_IMPLEMENTATION"}) // legacy - SetupContainerRegistryForRepoData(cmdData.CommonRepoData, cmd, "repo-container-registry", []string{"WERF_REPO_CONTAINER_REGISTRY"}) - SetupDockerHubUsernameForRepoData(cmdData.CommonRepoData, cmd, "repo-docker-hub-username", []string{"WERF_REPO_DOCKER_HUB_USERNAME"}) - SetupDockerHubPasswordForRepoData(cmdData.CommonRepoData, cmd, "repo-docker-hub-password", []string{"WERF_REPO_DOCKER_HUB_PASSWORD"}) - SetupDockerHubTokenForRepoData(cmdData.CommonRepoData, cmd, "repo-docker-hub-token", []string{"WERF_REPO_DOCKER_HUB_TOKEN"}) - SetupGithubTokenForRepoData(cmdData.CommonRepoData, cmd, "repo-github-token", []string{"WERF_REPO_GITHUB_TOKEN"}) - SetupHarborUsernameForRepoData(cmdData.CommonRepoData, cmd, "repo-harbor-username", []string{"WERF_REPO_HARBOR_USERNAME"}) - SetupHarborPasswordForRepoData(cmdData.CommonRepoData, cmd, "repo-harbor-password", []string{"WERF_REPO_HARBOR_PASSWORD"}) - SetupQuayTokenForRepoData(cmdData.CommonRepoData, cmd, "repo-quay-token", []string{"WERF_REPO_QUAY_TOKEN"}) -} - -func SetupCommonFinalRepoData(cmdData *CmdData, cmd *cobra.Command) { - cmdData.CommonFinalRepoData = &RepoData{} - - cmdData.CommonFinalRepoData.Implementation = new(string) // legacy - SetupContainerRegistryForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-container-registry", []string{"WERF_FINAL_REPO_CONTAINER_REGISTRY"}) - SetupDockerHubUsernameForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-docker-hub-username", []string{"WERF_FINAL_REPO_DOCKER_HUB_USERNAME"}) - SetupDockerHubPasswordForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-docker-hub-password", []string{"WERF_FINAL_REPO_DOCKER_HUB_PASSWORD"}) - SetupDockerHubTokenForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-docker-hub-token", []string{"WERF_FINAL_REPO_DOCKER_HUB_TOKEN"}) - SetupGithubTokenForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-github-token", []string{"WERF_FINAL_REPO_GITHUB_TOKEN"}) - SetupHarborUsernameForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-harbor-username", []string{"WERF_FINAL_REPO_HARBOR_USERNAME"}) - SetupHarborPasswordForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-harbor-password", []string{"WERF_FINAL_REPO_HARBOR_PASSWORD"}) - SetupQuayTokenForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-quay-token", []string{"WERF_FINAL_REPO_QUAY_TOKEN"}) -} - func SetupSecondaryStagesStorageOptions(cmdData *CmdData, cmd *cobra.Command) { cmdData.SecondaryStagesStorage = new([]string) cmd.Flags().StringArrayVarP(cmdData.SecondaryStagesStorage, "secondary-repo", "", []string{}, `Specify one or multiple secondary read-only repos with images that will be used as a cache. @@ -450,26 +419,20 @@ func SetupCacheStagesStorageOptions(cmdData *CmdData, cmd *cobra.Command) { Also, can be specified with $WERF_CACHE_REPO_* (e.g. $WERF_CACHE_REPO_1=..., $WERF_CACHE_REPO_2=...)`) } -func SetupStagesStorageOptions(cmdData *CmdData, cmd *cobra.Command) { +func SetupRepoOptions(cmdData *CmdData, cmd *cobra.Command, opts RepoDataOptions) { SetupInsecureRegistry(cmdData, cmd) SetupSkipTlsVerifyRegistry(cmdData, cmd) - SetupCommonRepoData(cmdData, cmd) - setupStagesStorage(cmdData, cmd) -} - -func SetupFinalStagesStorageOptions(cmdData *CmdData, cmd *cobra.Command) { - SetupCommonFinalRepoData(cmdData, cmd) - setupFinalStagesStorage(cmdData, cmd) + SetupRepo(cmdData, cmd, opts) } -func setupStagesStorage(cmdData *CmdData, cmd *cobra.Command) { - cmdData.StagesStorage = new(string) - cmd.Flags().StringVarP(cmdData.StagesStorage, "repo", "", os.Getenv("WERF_REPO"), fmt.Sprintf("Docker Repo to store stages (default $WERF_REPO)")) +func SetupRepo(cmdData *CmdData, cmd *cobra.Command, opts RepoDataOptions) { + cmdData.Repo = NewRepoData("repo", opts) + cmdData.Repo.SetupCmd(cmd) } -func setupFinalStagesStorage(cmdData *CmdData, cmd *cobra.Command) { - cmdData.FinalStagesStorage = new(string) - cmd.Flags().StringVarP(cmdData.FinalStagesStorage, "final-repo", "", os.Getenv("WERF_FINAL_REPO"), fmt.Sprintf("Docker Repo to store only those stages which are going to be used by the Kubernetes cluster, in other word final images (default $WERF_FINAL_REPO)")) +func SetupFinalRepo(cmdData *CmdData, cmd *cobra.Command) { + cmdData.FinalRepo = NewRepoData("final-repo", RepoDataOptions{}) + cmdData.FinalRepo.SetupCmd(cmd) } func SetupStatusProgressPeriod(cmdData *CmdData, cmd *cobra.Command) { @@ -974,104 +937,43 @@ func GetParallelTasksLimit(cmdData *CmdData) (int64, error) { } } -func GetStagesStorageAddress(cmdData *CmdData) (string, error) { - if *cmdData.StagesStorage == "" || *cmdData.StagesStorage == storage.LocalStorageAddress { - return "", fmt.Errorf("--repo=ADDRESS param required") - } - - return *cmdData.StagesStorage, nil -} - -func GetOptionalStagesStorageAddress(cmdData *CmdData) string { - if *cmdData.StagesStorage == "" { - return storage.LocalStorageAddress - } - - return *cmdData.StagesStorage +func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) storage.StagesStorage { + return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)) } -func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) (storage.StagesStorage, error) { - return storage.NewStagesStorage( - storage.LocalStorageAddress, - containerBackend, - storage.StagesStorageOptions{}, - ) -} - -func GetStagesStorage(stagesStorageAddress string, containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.StagesStorage, error) { +func GetStagesStorage(containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.StagesStorage, error) { if _, match := containerBackend.(*container_backend.BuildahBackend); match { - if stagesStorageAddress == "" || stagesStorageAddress == storage.LocalStorageAddress { - return nil, fmt.Errorf(`"--repo" should be specified and not equal ":local" for Buildah container backend`) + addr, err := cmdData.Repo.GetAddress() + if err != nil { + return nil, err } - } - if err := ValidateRepoContainerRegistry(cmdData.CommonRepoData.GetContainerRegistry()); err != nil { - return nil, err + if addr == storage.LocalStorageAddress { + return nil, fmt.Errorf(`"--repo" should be specified and not equal ":local" for Buildah container backend`) + } } - - return storage.NewStagesStorage( - stagesStorageAddress, - containerBackend, - storage.StagesStorageOptions{ - RepoStagesStorageOptions: storage.RepoStagesStorageOptions{ - ContainerRegistry: cmdData.CommonRepoData.GetContainerRegistry(), - DockerRegistryOptions: docker_registry.DockerRegistryOptions{ - InsecureRegistry: *cmdData.InsecureRegistry, - SkipTlsVerifyRegistry: *cmdData.SkipTlsVerifyRegistry, - DockerHubUsername: *cmdData.CommonRepoData.DockerHubUsername, - DockerHubPassword: *cmdData.CommonRepoData.DockerHubPassword, - DockerHubToken: *cmdData.CommonRepoData.DockerHubToken, - GitHubToken: *cmdData.CommonRepoData.GitHubToken, - HarborUsername: *cmdData.CommonRepoData.HarborUsername, - HarborPassword: *cmdData.CommonRepoData.HarborPassword, - QuayToken: *cmdData.CommonRepoData.QuayToken, - }, - }, - }, - ) + return cmdData.Repo.CreateStagesStorage(containerBackend, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) } func GetOptionalFinalStagesStorage(containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.StagesStorage, error) { - finalRepoAddress := *cmdData.FinalStagesStorage - if finalRepoAddress == "" { + if *cmdData.FinalRepo.Address == "" { return nil, nil } - - if err := ValidateRepoContainerRegistry(cmdData.CommonFinalRepoData.GetContainerRegistry()); err != nil { - return nil, err - } - - return storage.NewStagesStorage( - finalRepoAddress, - containerBackend, - storage.StagesStorageOptions{ - RepoStagesStorageOptions: storage.RepoStagesStorageOptions{ - ContainerRegistry: cmdData.CommonFinalRepoData.GetContainerRegistry(), - DockerRegistryOptions: docker_registry.DockerRegistryOptions{ - InsecureRegistry: *cmdData.InsecureRegistry, - SkipTlsVerifyRegistry: *cmdData.SkipTlsVerifyRegistry, - DockerHubUsername: *cmdData.CommonFinalRepoData.DockerHubUsername, - DockerHubPassword: *cmdData.CommonFinalRepoData.DockerHubPassword, - DockerHubToken: *cmdData.CommonFinalRepoData.DockerHubToken, - GitHubToken: *cmdData.CommonFinalRepoData.GitHubToken, - HarborUsername: *cmdData.CommonFinalRepoData.HarborUsername, - HarborPassword: *cmdData.CommonFinalRepoData.HarborPassword, - QuayToken: *cmdData.CommonFinalRepoData.QuayToken, - }, - }, - }, - ) + return cmdData.FinalRepo.CreateStagesStorage(containerBackend, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) } func GetCacheStagesStorageList(containerBackend container_backend.ContainerBackend, cmdData *CmdData) ([]storage.StagesStorage, error) { var res []storage.StagesStorage for _, address := range GetCacheStagesStorage(cmdData) { - repoStagesStorage, err := storage.NewStagesStorage(address, containerBackend, storage.StagesStorageOptions{}) + repoData := NewRepoData("cache-repo", RepoDataOptions{}) + repoData.Address = &address + + storage, err := repoData.CreateStagesStorage(containerBackend, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) if err != nil { - return nil, fmt.Errorf("unable to create cache stages storage at %s: %w", address, err) + return nil, fmt.Errorf("unable to create cache stages storage in %s: %w", address, err) } - res = append(res, repoStagesStorage) + res = append(res, storage) } return res, nil @@ -1080,22 +982,21 @@ func GetCacheStagesStorageList(containerBackend container_backend.ContainerBacke func GetSecondaryStagesStorageList(stagesStorage storage.StagesStorage, containerBackend container_backend.ContainerBackend, cmdData *CmdData) ([]storage.StagesStorage, error) { var res []storage.StagesStorage - if _, matched := containerBackend.(*container_backend.DockerServerBackend); matched { + if dockerBackend, matched := containerBackend.(*container_backend.DockerServerBackend); matched { if stagesStorage.Address() != storage.LocalStorageAddress { - localStagesStorage, err := storage.NewStagesStorage(storage.LocalStorageAddress, containerBackend, storage.StagesStorageOptions{}) - if err != nil { - return nil, fmt.Errorf("unable to create local secondary stages storage: %w", err) - } - res = append(res, localStagesStorage) + res = append(res, storage.NewDockerServerStagesStorage(dockerBackend)) } } for _, address := range GetSecondaryStagesStorage(cmdData) { - repoStagesStorage, err := storage.NewStagesStorage(address, containerBackend, storage.StagesStorageOptions{}) + repoData := NewRepoData("secondary-repo", RepoDataOptions{}) + repoData.Address = &address + + storage, err := repoData.CreateStagesStorage(containerBackend, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) if err != nil { - return nil, fmt.Errorf("unable to create secondary stages storage at %s: %w", address, err) + return nil, fmt.Errorf("unable to create secondary stages storage in %s: %w", address, err) } - res = append(res, repoStagesStorage) + res = append(res, storage) } return res, nil @@ -1468,19 +1369,6 @@ func DockerRegistryInit(ctx context.Context, cmdData *CmdData) error { return docker_registry.Init(ctx, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) } -func ValidateRepoContainerRegistry(containerRegistry string) error { - supportedValues := docker_registry.ImplementationList() - supportedValues = append(supportedValues, "auto", "") - - for _, supportedContainerRegistry := range supportedValues { - if supportedContainerRegistry == containerRegistry { - return nil - } - } - - return fmt.Errorf("specified container registry %q is not supported", containerRegistry) -} - func ValidateMinimumNArgs(minArgs int, args []string, cmd *cobra.Command) error { if len(args) < minArgs { PrintHelp(cmd) diff --git a/cmd/werf/common/conveyor_options.go b/cmd/werf/common/conveyor_options.go index 19c771e567..ff5889d4b3 100644 --- a/cmd/werf/common/conveyor_options.go +++ b/cmd/werf/common/conveyor_options.go @@ -84,7 +84,7 @@ func getCustomTagFuncList(commonCmdData *CmdData, giterminismManager giterminism return nil, nil } - if *commonCmdData.StagesStorage == "" || *commonCmdData.StagesStorage == storage.LocalStorageAddress { + if *commonCmdData.Repo.Address == "" || *commonCmdData.Repo.Address == storage.LocalStorageAddress { return nil, fmt.Errorf("custom tags can only be used with remote storage: --repo=ADDRESS param required") } diff --git a/cmd/werf/common/repo_data.go b/cmd/werf/common/repo_data.go new file mode 100644 index 0000000000..1d794a9867 --- /dev/null +++ b/cmd/werf/common/repo_data.go @@ -0,0 +1,352 @@ +package common + +import ( + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/container_backend" + "github.com/werf/werf/pkg/docker_registry" + "github.com/werf/werf/pkg/storage" +) + +func (repoData *RepoData) CreateDockerRegistry(insecureRegistry, skipTlsVerifyRegistry bool) (docker_registry.Interface, error) { + addr, err := repoData.GetAddress() + if err != nil { + return nil, err + } + + cr := repoData.GetContainerRegistry() + if err := ValidateRepoContainerRegistry(cr); err != nil { + return nil, err + } + + regOpts := repoData.GetDockerRegistryOptions(insecureRegistry, skipTlsVerifyRegistry) + dockerRegistry, err := docker_registry.NewDockerRegistry(addr, cr, regOpts) + if err != nil { + return nil, fmt.Errorf("error creating container registry accessor for repo %q: %w", addr, err) + } + + return dockerRegistry, nil +} + +func (repoData *RepoData) CreateStagesStorage(containerBackend container_backend.ContainerBackend, insecureRegistry, skipTlsVerifyRegistry bool) (storage.StagesStorage, error) { + addr, err := repoData.GetAddress() + if err != nil { + return nil, err + } + + if addr == storage.LocalStorageAddress { + return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)), nil + } else { + dockerRegistry, err := repoData.CreateDockerRegistry(insecureRegistry, skipTlsVerifyRegistry) + if err != nil { + return nil, err + } + return storage.NewRepoStagesStorage(addr, containerBackend, dockerRegistry), nil + } +} + +type RepoData struct { + Name string + + Address *string + Implementation *string // legacy + ContainerRegistry *string + DockerHubUsername *string + DockerHubPassword *string + DockerHubToken *string + GitHubToken *string + HarborUsername *string + HarborPassword *string + QuayToken *string + + RepoDataOptions +} + +type RepoDataOptions struct { + OnlyAddress bool + OptionalRepo bool +} + +func NewRepoData(name string, opts RepoDataOptions) *RepoData { + return &RepoData{Name: name, RepoDataOptions: opts} +} + +func (d *RepoData) GetAddress() (string, error) { + addr := *d.Address + if addr == "" { + addr = storage.LocalStorageAddress + } + if !d.OptionalRepo && addr == storage.LocalStorageAddress { + return "", fmt.Errorf("--%s=ADDRESS param required", d.Name) + } + return addr, nil +} + +func (d *RepoData) GetContainerRegistry() string { + if d.OnlyAddress { + return "" + } + + switch { + case *d.ContainerRegistry != "": + return *d.ContainerRegistry + case *d.Implementation != "": + repoNameUpper := strings.ToUpper(strings.ReplaceAll(d.Name, "-", "_")) + logboek.Context(GetContext()).Warn().LogLn("DEPRECATION WARNING: The option --%s-implementation ($WERF_%s_IMPLEMENTATION) is renamed to --%s-container-registry ($WERF_%s_CONTAINER_REGISTRY) and will be removed in v1.3!", d.Name, repoNameUpper, d.Name, repoNameUpper) + return *d.Implementation + default: + return "" + } +} + +func (d *RepoData) GetDockerRegistryOptions(insecureRegistry, skipTlsVerifyRegistry bool) docker_registry.DockerRegistryOptions { + opts := docker_registry.DockerRegistryOptions{ + InsecureRegistry: insecureRegistry, + SkipTlsVerifyRegistry: skipTlsVerifyRegistry, + } + + if !d.OnlyAddress { + opts.DockerHubUsername = *d.DockerHubUsername + opts.DockerHubPassword = *d.DockerHubPassword + opts.DockerHubToken = *d.DockerHubToken + opts.GitHubToken = *d.GitHubToken + opts.HarborUsername = *d.HarborUsername + opts.HarborPassword = *d.HarborPassword + opts.QuayToken = *d.QuayToken + } + + return opts +} + +func (repoData *RepoData) SetupCmd(cmd *cobra.Command) { + repoNameUpper := strings.ToUpper(strings.ReplaceAll(repoData.Name, "-", "_")) + makeEnvVar := func(variableSuffix string) string { + return fmt.Sprintf("WERF_%s_%s", repoNameUpper, variableSuffix) + } + makeOpt := func(optSuffix string) string { + return fmt.Sprintf("%s-%s", repoData.Name, optSuffix) + } + + repoData.SetupAddressForRepoData(cmd, repoData.Name, []string{fmt.Sprintf("WERF_%s", repoNameUpper)}) + if repoData.OnlyAddress { + return + } + + repoData.SetupImplementationForRepoData(cmd, makeOpt("implementation"), []string{makeEnvVar("IMPLEMENTATION")}) // legacy + repoData.SetupContainerRegistryForRepoData(cmd, makeOpt("container-registry"), []string{makeEnvVar("CONTAINER_REGISTRY")}) + repoData.SetupDockerHubUsernameForRepoData(cmd, makeOpt("docker-hub-username"), []string{makeEnvVar("DOCKER_HUB_USERNAME")}) + repoData.SetupDockerHubPasswordForRepoData(cmd, makeOpt("docker-hub-password"), []string{makeEnvVar("DOCKER_HUB_PASSWORD")}) + repoData.SetupDockerHubTokenForRepoData(cmd, makeOpt("docker-hub-token"), []string{makeEnvVar("DOCKER_HUB_TOKEN")}) + repoData.SetupGithubTokenForRepoData(cmd, makeOpt("github-token"), []string{makeEnvVar("GITHUB_TOKEN")}) + repoData.SetupHarborUsernameForRepoData(cmd, makeOpt("harbor-username"), []string{makeEnvVar("HARBOR_USERNAME")}) + repoData.SetupHarborPasswordForRepoData(cmd, makeOpt("harbor-password"), []string{makeEnvVar("HARBOR_PASSWORD")}) + repoData.SetupQuayTokenForRepoData(cmd, makeOpt("quay-token"), []string{makeEnvVar("QUAY_TOKEN")}) +} + +func MergeRepoData(repoDataArr ...*RepoData) *RepoData { + res := &RepoData{} + + for _, repoData := range repoDataArr { + if res.GetContainerRegistry() == "" { + value := repoData.GetContainerRegistry() + res.ContainerRegistry = &value + } + if res.DockerHubUsername == nil || *res.DockerHubUsername == "" { + res.DockerHubUsername = repoData.DockerHubUsername + } + if res.DockerHubPassword == nil || *res.DockerHubPassword == "" { + res.DockerHubPassword = repoData.DockerHubPassword + } + if res.DockerHubToken == nil || *res.DockerHubToken == "" { + res.DockerHubToken = repoData.DockerHubToken + } + if res.GitHubToken == nil || *res.GitHubToken == "" { + res.GitHubToken = repoData.GitHubToken + } + if res.HarborUsername == nil || *res.HarborUsername == "" { + res.HarborUsername = repoData.HarborUsername + } + if res.HarborPassword == nil || *res.HarborPassword == "" { + res.HarborPassword = repoData.HarborPassword + } + if res.QuayToken == nil || *res.QuayToken == "" { + res.QuayToken = repoData.QuayToken + } + } + + return res +} + +func (repoData *RepoData) SetupAddressForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("Container registry storage address (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.Address = new(string) + cmd.Flags().StringVarP( + repoData.Address, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +// legacy +func (repoData *RepoData) SetupImplementationForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + repoData.Implementation = new(string) + cmd.Flags().StringVarP( + repoData.Implementation, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + "", + ) + cmd.Flag(paramName).Hidden = true +} + +func (repoData *RepoData) SetupContainerRegistryForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usageTitle := fmt.Sprintf("Choose %s container registry implementation", repoData.Name) + + repoData.ContainerRegistry = new(string) + cmd.Flags().StringVarP( + repoData.ContainerRegistry, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + fmt.Sprintf(`%s. +The following container registries are supported: %s. +Default %s or auto mode (detect container registry by repo address).`, + usageTitle, + strings.Join(docker_registry.ImplementationList(), ", "), + strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", "), + ), + ) +} + +func (repoData *RepoData) SetupDockerHubUsernameForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Docker Hub username (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.DockerHubUsername = new(string) + cmd.Flags().StringVarP( + repoData.DockerHubUsername, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupDockerHubPasswordForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Docker Hub password (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.DockerHubPassword = new(string) + cmd.Flags().StringVarP( + repoData.DockerHubPassword, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupDockerHubTokenForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Docker Hub token (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.DockerHubToken = new(string) + cmd.Flags().StringVarP( + repoData.DockerHubToken, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupGithubTokenForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s GitHub token (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.GitHubToken = new(string) + cmd.Flags().StringVarP( + repoData.GitHubToken, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupHarborUsernameForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Harbor username (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.HarborUsername = new(string) + cmd.Flags().StringVarP( + repoData.HarborUsername, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupHarborPasswordForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Harbor password (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.HarborPassword = new(string) + cmd.Flags().StringVarP( + repoData.HarborPassword, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupQuayTokenForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s quay.io token (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.QuayToken = new(string) + cmd.Flags().StringVarP( + repoData.QuayToken, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func getDefaultValueByParamEnvNames(paramEnvNames []string) string { + var defaultValue string + for _, paramEnvName := range paramEnvNames { + value := os.Getenv(paramEnvName) + if value != "" { + defaultValue = value + break + } + } + return defaultValue +} + +func getParamEnvNamesForUsageDescription(paramEnvNames []string) []string { + paramEnvNamesWithDollar := []string{} + for _, paramEnvName := range paramEnvNames { + paramEnvNamesWithDollar = append(paramEnvNamesWithDollar, "$"+paramEnvName) + } + return paramEnvNamesWithDollar +} + +func ValidateRepoContainerRegistry(containerRegistry string) error { + supportedValues := docker_registry.ImplementationList() + supportedValues = append(supportedValues, "auto", "") + + for _, supportedContainerRegistry := range supportedValues { + if supportedContainerRegistry == containerRegistry { + return nil + } + } + + return fmt.Errorf("specified container registry %q is not supported", containerRegistry) +} diff --git a/cmd/werf/common/storage_repo_data.go b/cmd/werf/common/storage_repo_data.go deleted file mode 100644 index 2a3cbee776..0000000000 --- a/cmd/werf/common/storage_repo_data.go +++ /dev/null @@ -1,256 +0,0 @@ -package common - -import ( - "fmt" - "os" - "strings" - - "github.com/spf13/cobra" - - "github.com/werf/logboek" - "github.com/werf/werf/pkg/docker_registry" -) - -type RepoData struct { - IsCommon bool - DesignationStorageName string - - Implementation *string // legacy - ContainerRegistry *string - DockerHubUsername *string - DockerHubPassword *string - DockerHubToken *string - GitHubToken *string - HarborUsername *string - HarborPassword *string - QuayToken *string -} - -func (d *RepoData) GetContainerRegistry() string { - switch { - case *d.ContainerRegistry != "": - return *d.ContainerRegistry - case *d.Implementation != "": - logboek.Context(GetContext()).Warn().LogLn("DEPRECATION WARNING: The option --repo-implementation ($WERF_REPO_IMPLEMENTATION) is renamed to --repo-container-registry ($WERF_REPO_CONTAINER_REGISTRY) and will be removed in v1.3!") - return *d.Implementation - default: - return "" - } -} - -func MergeRepoData(repoDataArr ...*RepoData) *RepoData { - res := &RepoData{} - - for _, repoData := range repoDataArr { - if res.GetContainerRegistry() == "" { - value := repoData.GetContainerRegistry() - res.ContainerRegistry = &value - } - if res.DockerHubUsername == nil || *res.DockerHubUsername == "" { - res.DockerHubUsername = repoData.DockerHubUsername - } - if res.DockerHubPassword == nil || *res.DockerHubPassword == "" { - res.DockerHubPassword = repoData.DockerHubPassword - } - if res.DockerHubToken == nil || *res.DockerHubToken == "" { - res.DockerHubToken = repoData.DockerHubToken - } - if res.GitHubToken == nil || *res.GitHubToken == "" { - res.GitHubToken = repoData.GitHubToken - } - if res.HarborUsername == nil || *res.HarborUsername == "" { - res.HarborUsername = repoData.HarborUsername - } - if res.HarborPassword == nil || *res.HarborPassword == "" { - res.HarborPassword = repoData.HarborPassword - } - if res.QuayToken == nil || *res.QuayToken == "" { - res.QuayToken = repoData.QuayToken - } - } - - return res -} - -// legacy -func SetupImplementationForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - repoData.Implementation = new(string) - cmd.Flags().StringVarP( - repoData.Implementation, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - "", - ) - cmd.Flag(paramName).Hidden = true -} - -func SetupContainerRegistryForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usageTitle string - if repoData.IsCommon { - usageTitle = "Choose repo container registry" - } else { - usageTitle = fmt.Sprintf("Choose repo container registry for %s", repoData.DesignationStorageName) - } - - repoData.ContainerRegistry = new(string) - cmd.Flags().StringVarP( - repoData.ContainerRegistry, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - fmt.Sprintf(`%s. -The following container registries are supported: %s. -Default %s or auto mode (detect container registry by repo address).`, - usageTitle, - strings.Join(docker_registry.ImplementationList(), ", "), - strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", "), - ), - ) -} - -func SetupDockerHubUsernameForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Docker Hub username (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Docker Hub username for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.DockerHubUsername = new(string) - cmd.Flags().StringVarP( - repoData.DockerHubUsername, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupDockerHubPasswordForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Docker Hub password (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Docker Hub password for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.DockerHubPassword = new(string) - cmd.Flags().StringVarP( - repoData.DockerHubPassword, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupDockerHubTokenForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Docker Hub token (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Docker Hub token for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.DockerHubToken = new(string) - cmd.Flags().StringVarP( - repoData.DockerHubToken, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupGithubTokenForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("GitHub token (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("GitHub token for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.GitHubToken = new(string) - cmd.Flags().StringVarP( - repoData.GitHubToken, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupHarborUsernameForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Harbor username (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Harbor username for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.HarborUsername = new(string) - cmd.Flags().StringVarP( - repoData.HarborUsername, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupHarborPasswordForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Harbor password (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Harbor password for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.HarborPassword = new(string) - cmd.Flags().StringVarP( - repoData.HarborPassword, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupQuayTokenForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("quay.io token (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("quay.io token for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.QuayToken = new(string) - cmd.Flags().StringVarP( - repoData.QuayToken, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func getDefaultValueByParamEnvNames(paramEnvNames []string) string { - var defaultValue string - for _, paramEnvName := range paramEnvNames { - value := os.Getenv(paramEnvName) - if value != "" { - defaultValue = value - break - } - } - return defaultValue -} - -func getParamEnvNamesForUsageDescription(paramEnvNames []string) []string { - paramEnvNamesWithDollar := []string{} - for _, paramEnvName := range paramEnvNames { - paramEnvNamesWithDollar = append(paramEnvNamesWithDollar, "$"+paramEnvName) - } - return paramEnvNamesWithDollar -} diff --git a/cmd/werf/compose/main.go b/cmd/werf/compose/main.go index 5ff5434f32..e0fdf677c9 100644 --- a/cmd/werf/compose/main.go +++ b/cmd/werf/compose/main.go @@ -210,8 +210,8 @@ services: common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{OptionalRepo: true}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSkipBuild(&commonCmdData, cmd) @@ -390,8 +390,7 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken } defer tmp_manager.ReleaseProjectDir(projectTmpDir) - stagesStorageAddress := common.GetOptionalStagesStorageAddress(&commonCmdData) - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/converge/converge.go b/cmd/werf/converge/converge.go index 6866beb4b0..d643808d8c 100644 --- a/cmd/werf/converge/converge.go +++ b/cmd/werf/converge/converge.go @@ -102,8 +102,8 @@ werf converge --repo registry.mydomain.com/web --env production`, common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo, to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -271,11 +271,7 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken var imagesInfoGetters []*image.InfoGetter var imagesRepository string if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 { - stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData) - if err != nil { - return err - } - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/dismiss/dismiss.go b/cmd/werf/dismiss/dismiss.go index 5ea6b54946..2d44872610 100644 --- a/cmd/werf/dismiss/dismiss.go +++ b/cmd/werf/dismiss/dismiss.go @@ -83,8 +83,8 @@ Read more info about Helm Release name, Kubernetes Namespace and how to change i common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSynchronization(&commonCmdData, cmd) common.SetupRelease(&commonCmdData, cmd) diff --git a/cmd/werf/export/export.go b/cmd/werf/export/export.go index 8e31bdd5fc..c169a70481 100644 --- a/cmd/werf/export/export.go +++ b/cmd/werf/export/export.go @@ -78,8 +78,8 @@ All meta-information related to werf is removed from the exported images, and th common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{OptionalRepo: true}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSkipBuild(&commonCmdData, cmd) @@ -178,8 +178,7 @@ func run(ctx context.Context, imagesToProcess, tagTemplateList []string) error { } defer tmp_manager.ReleaseProjectDir(projectTmpDir) - stagesStorageAddress := common.GetOptionalStagesStorageAddress(&commonCmdData) - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/helm/get_autogenerated_values.go b/cmd/werf/helm/get_autogenerated_values.go index 92eaf94b14..0954735e87 100644 --- a/cmd/werf/helm/get_autogenerated_values.go +++ b/cmd/werf/helm/get_autogenerated_values.go @@ -63,8 +63,8 @@ These values includes project name, docker images ids and other`), common.SetupSecondaryStagesStorageOptions(&getAutogeneratedValuedCmdData, cmd) common.SetupCacheStagesStorageOptions(&getAutogeneratedValuedCmdData, cmd) - common.SetupStagesStorageOptions(&getAutogeneratedValuedCmdData, cmd) - common.SetupFinalStagesStorageOptions(&getAutogeneratedValuedCmdData, cmd) + common.SetupRepoOptions(&getAutogeneratedValuedCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&getAutogeneratedValuedCmdData, cmd) common.SetupSynchronization(&getAutogeneratedValuedCmdData, cmd) common.SetupKubeConfig(&getAutogeneratedValuedCmdData, cmd) @@ -167,11 +167,11 @@ func runGetServiceValues(ctx context.Context) error { } defer tmp_manager.ReleaseProjectDir(projectTmpDir) - stagesStorageAddress, err := common.GetStagesStorageAddress(&getAutogeneratedValuedCmdData) + _, err = getAutogeneratedValuedCmdData.Repo.GetAddress() if err != nil { return fmt.Errorf("%w (use --stub-tags option to get service values without real tags)", err) } - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &getAutogeneratedValuedCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &getAutogeneratedValuedCmdData) if err != nil { return err } diff --git a/cmd/werf/host/purge/purge.go b/cmd/werf/host/purge/purge.go index 54de1cdfa3..c50c52dac2 100644 --- a/cmd/werf/host/purge/purge.go +++ b/cmd/werf/host/purge/purge.go @@ -106,10 +106,7 @@ func runReset(ctx context.Context) error { return err } } else { - stagesStorage, err := common.GetLocalStagesStorage(containerBackend) - if err != nil { - return err - } + stagesStorage := common.GetLocalStagesStorage(containerBackend) synchronization, err := common.GetSynchronization(ctx, &commonCmdData, projectName, stagesStorage) if err != nil { return err diff --git a/cmd/werf/kube_run/kube_run.go b/cmd/werf/kube_run/kube_run.go index 13f09dea29..05a51f8ebe 100644 --- a/cmd/werf/kube_run/kube_run.go +++ b/cmd/werf/kube_run/kube_run.go @@ -142,8 +142,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSkipBuild(&commonCmdData, cmd) @@ -332,11 +332,7 @@ func run(ctx context.Context, pod, secret, namespace string, werfConfig *config. return fmt.Errorf("image %q is not defined in werf.yaml", logging.ImageLogName(imageName, false)) } - stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData) - if err != nil { - return err - } - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/main.go b/cmd/werf/main.go index d4cb1d23a6..639a8a8ce0 100644 --- a/cmd/werf/main.go +++ b/cmd/werf/main.go @@ -12,6 +12,7 @@ import ( "github.com/werf/logboek" "github.com/werf/werf/cmd/werf/build" bundle_apply "github.com/werf/werf/cmd/werf/bundle/apply" + bundle_copy "github.com/werf/werf/cmd/werf/bundle/copy" bundle_download "github.com/werf/werf/cmd/werf/bundle/download" bundle_export "github.com/werf/werf/cmd/werf/bundle/export" bundle_publish "github.com/werf/werf/cmd/werf/bundle/publish" @@ -186,6 +187,7 @@ func bundleCmd() *cobra.Command { bundle_export.NewCmd(), bundle_download.NewCmd(), bundle_render.NewCmd(), + bundle_copy.NewCmd(), ) return cmd diff --git a/cmd/werf/managed_images/add/add.go b/cmd/werf/managed_images/add/add.go index 0c7c47af5d..df24b68a6d 100644 --- a/cmd/werf/managed_images/add/add.go +++ b/cmd/werf/managed_images/add/add.go @@ -55,8 +55,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read and write images to the specified repo") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -139,12 +139,7 @@ func run(ctx context.Context, imageName string) error { return fmt.Errorf("run command in the project directory with werf.yaml") } - stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData) - if err != nil { - return err - } - - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/managed_images/ls/ls.go b/cmd/werf/managed_images/ls/ls.go index 5da3b26e2f..dc3677c2d1 100644 --- a/cmd/werf/managed_images/ls/ls.go +++ b/cmd/werf/managed_images/ls/ls.go @@ -51,8 +51,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read images from the specified repo") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -135,11 +135,7 @@ func run(ctx context.Context) error { return fmt.Errorf("run command in the project directory with werf.yaml") } - stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData) - if err != nil { - return err - } - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/managed_images/rm/rm.go b/cmd/werf/managed_images/rm/rm.go index f6347d9e4f..d4d619389f 100644 --- a/cmd/werf/managed_images/rm/rm.go +++ b/cmd/werf/managed_images/rm/rm.go @@ -55,8 +55,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read and write images to the specified repo") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -139,11 +139,7 @@ func run(ctx context.Context, imageNames []string) error { return fmt.Errorf("run command in the project directory with werf.yaml") } - stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData) - if err != nil { - return err - } - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/purge/purge.go b/cmd/werf/purge/purge.go index 616be72296..d6af95c512 100644 --- a/cmd/werf/purge/purge.go +++ b/cmd/werf/purge/purge.go @@ -57,8 +57,8 @@ WARNING: Images that are being used in the Kubernetes cluster will also be delet common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultCleanupParallelTasksLimit) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to delete images from the specified repo") @@ -140,14 +140,13 @@ func runPurge(ctx context.Context) error { logboek.LogOptionalLn() - stagesStorageAddress, err := common.GetStagesStorageAddress(&commonCmdData) + _, err = commonCmdData.Repo.GetAddress() if err != nil { logboek.Context(ctx).Default().LogLnDetails(`The "werf purge" command is only used to cleaning the container registry. In case you need to clean the runner or the localhost, use the commands of the "werf host" group. It is worth noting that auto-cleaning is enabled by default, and manual use is usually not required (if not, we would appreciate feedback and creating an issue https://github.com/werf/werf/issues/new).`) - return err } - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/render/render.go b/cmd/werf/render/render.go index 5113234daf..495b948b37 100644 --- a/cmd/werf/render/render.go +++ b/cmd/werf/render/render.go @@ -89,8 +89,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{OptionalRepo: true}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo and to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -239,14 +239,17 @@ func runRender(ctx context.Context) error { var stubImagesNames []string if len(werfConfig.StapelImages) != 0 || len(werfConfig.ImagesFromDockerfile) != 0 { - stagesStorageAddress := common.GetOptionalStagesStorageAddress(&commonCmdData) + addr, err := commonCmdData.Repo.GetAddress() + if err != nil { + return err + } - if stagesStorageAddress != storage.LocalStorageAddress { + if addr != storage.LocalStorageAddress { if err := common.DockerRegistryInit(ctx, &commonCmdData); err != nil { return err } - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/run/run.go b/cmd/werf/run/run.go index 269c999687..b720288243 100644 --- a/cmd/werf/run/run.go +++ b/cmd/werf/run/run.go @@ -124,8 +124,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{OptionalRepo: true}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSkipBuild(&commonCmdData, cmd) @@ -347,8 +347,7 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken return fmt.Errorf("image %q is not defined in werf.yaml", logging.ImageLogName(imageName, false)) } - stagesStorageAddress := common.GetOptionalStagesStorageAddress(&commonCmdData) - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/cmd/werf/stage/image/main.go b/cmd/werf/stage/image/main.go index 28f800c262..a82d281a76 100644 --- a/cmd/werf/stage/image/main.go +++ b/cmd/werf/stage/image/main.go @@ -64,8 +64,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd, common.RepoDataOptions{OptionalRepo: true}) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read and pull images from the specified stages storage") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -163,8 +163,7 @@ func run(ctx context.Context, imageName string) error { return fmt.Errorf("image %q is not defined in werf.yaml", logging.ImageLogName(imageName, false)) } - stagesStorageAddress := common.GetOptionalStagesStorageAddress(&commonCmdData) - stagesStorage, err := common.GetStagesStorage(stagesStorageAddress, containerBackend, &commonCmdData) + stagesStorage, err := common.GetStagesStorage(containerBackend, &commonCmdData) if err != nil { return err } diff --git a/pkg/deploy/bundles/copy.go b/pkg/deploy/bundles/copy.go new file mode 100644 index 0000000000..14009be43b --- /dev/null +++ b/pkg/deploy/bundles/copy.go @@ -0,0 +1,128 @@ +package bundles + +import ( + "context" + "fmt" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" + "sigs.k8s.io/yaml" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/deploy/bundles/registry" + "github.com/werf/werf/pkg/docker_registry" + "github.com/werf/werf/pkg/util" +) + +func Copy(ctx context.Context, fromAddr, toAddr string, bundlesRegistryClient *registry.Client, fromRegistry docker_registry.Interface) error { + fromRef, err := registry.ParseReference(fromAddr) + if err != nil { + return fmt.Errorf("unable to parse source address %q: %w", fromAddr, err) + } + toRef, err := registry.ParseReference(toAddr) + if err != nil { + return fmt.Errorf("unable to parse destination address %q: %w", toAddr, err) + } + + if err := logboek.Context(ctx).LogProcess("Pulling bundle %s", fromRef.FullName()).DoError(func() error { + if err := bundlesRegistryClient.PullChartToCache(fromRef); err != nil { + return fmt.Errorf("unable to pull bundle %s: %w", fromRef.FullName(), err) + } + return nil + }); err != nil { + return err + } + + var ch *chart.Chart + if err := logboek.Context(ctx).LogProcess("Loading bundle %s", fromRef.FullName()).DoError(func() error { + ch, err = bundlesRegistryClient.LoadChart(fromRef) + if err != nil { + return fmt.Errorf("unable to load pulled bundle %s: %w", fromRef.FullName(), err) + } + return nil + }); err != nil { + return err + } + + { + d, err := yaml.Marshal(ch.Values) + logboek.Context(ctx).Debug().LogF("Values before change (%v):\n%s\n---\n", err, d) + } + + if werfVals, ok := ch.Values["werf"].(map[string]interface{}); ok { + if imageVals, ok := werfVals["image"].(map[string]interface{}); ok { + newImageVals := make(map[string]interface{}) + + for imageName, v := range imageVals { + if image, ok := v.(string); ok { + ref, err := registry.ParseReference(image) + if err != nil { + return fmt.Errorf("unable to parse bundle image %s: %w", image, err) + } + + ref.Repo = toRef.Repo + + // TODO: copy images in parallel + if image != ref.FullName() { + if err := logboek.Context(ctx).LogProcess("Copy image").DoError(func() error { + logboek.Context(ctx).Default().LogFDetails("Source: %s\n", image) + logboek.Context(ctx).Default().LogFDetails("Destination: %s\n", ref.FullName()) + + if err := fromRegistry.MutateAndPushImage(ctx, image, ref.FullName(), func(cfg v1.Config) (v1.Config, error) { return cfg, nil }); err != nil { + return fmt.Errorf("error copying image %s into %s: %w", image, ref.FullName(), err) + } + return nil + }); err != nil { + return err + } + } + + newImageVals[imageName] = ref.FullName() + } else { + return fmt.Errorf("unexpected value .Values.werf.image.%s=%v", imageName, v) + } + } + + werfVals["image"] = newImageVals + } + + werfVals["repo"] = toRef.Repo + } + + valuesRaw, err := yaml.Marshal(ch.Values) + if err != nil { + return fmt.Errorf("unable to marshal changed bundle values: %w", err) + } + logboek.Context(ctx).Debug().LogF("Values after change (%v):\n%s\n---\n", err, valuesRaw) + + for _, f := range ch.Raw { + if f.Name == chartutil.ValuesfileName { + f.Data = valuesRaw + break + } + } + + ch.Metadata.Name = util.Reverse(strings.SplitN(util.Reverse(toRef.Repo), "/", 2)[0]) + + if err := logboek.Context(ctx).LogProcess("Saving bundle %s", toRef.FullName()).DoError(func() error { + if err := bundlesRegistryClient.SaveChart(ch, toRef); err != nil { + return fmt.Errorf("unable to save bundle %s to the local chart helm cache: %w", toRef.FullName(), err) + } + return nil + }); err != nil { + return err + } + + if err := logboek.Context(ctx).LogProcess("Pushing bundle %s", toRef.FullName()).DoError(func() error { + if err := bundlesRegistryClient.PushChart(toRef); err != nil { + return fmt.Errorf("unable to push bundle %s: %w", toRef.FullName(), err) + } + return nil + }); err != nil { + return err + } + + return nil +} diff --git a/pkg/deploy/helm/chart_extender/werf_chart.go b/pkg/deploy/helm/chart_extender/werf_chart.go index c55761630b..4fbcfe5477 100644 --- a/pkg/deploy/helm/chart_extender/werf_chart.go +++ b/pkg/deploy/helm/chart_extender/werf_chart.go @@ -293,7 +293,7 @@ func (wc *WerfChart) CreateNewBundle(ctx context.Context, destDir, chartVersion return nil, fmt.Errorf("unable to construct bundle input values: %w", err) } - valsData, err = json.MarshalIndent(vals, "", " ") + valsData, err = yaml.Marshal(vals) if err != nil { return nil, fmt.Errorf("unable to prepare values: %w", err) } @@ -313,7 +313,7 @@ func (wc *WerfChart) CreateNewBundle(ctx context.Context, destDir, chartVersion logboek.Context(ctx).Debug().LogF("Saving bundle values:\n%s\n---\n", valsData) valuesFile := filepath.Join(destDir, "values.yaml") - if err := ioutil.WriteFile(valuesFile, append(valsData, []byte("\n")...), os.ModePerm); err != nil { + if err := ioutil.WriteFile(valuesFile, valsData, os.ModePerm); err != nil { return nil, fmt.Errorf("unable to write %q: %w", valuesFile, err) } diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index 7e0bf1a15d..48ebda9638 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -64,22 +64,12 @@ type RepoStagesStorage struct { ContainerBackend container_backend.ContainerBackend } -type RepoStagesStorageOptions struct { - docker_registry.DockerRegistryOptions - ContainerRegistry string -} - -func NewRepoStagesStorage(repoAddress string, containerBackend container_backend.ContainerBackend, options RepoStagesStorageOptions) (*RepoStagesStorage, error) { - dockerRegistry, err := docker_registry.NewDockerRegistry(repoAddress, options.ContainerRegistry, options.DockerRegistryOptions) - if err != nil { - return nil, fmt.Errorf("error creating container registry accessor for repo %q: %w", repoAddress, err) - } - +func NewRepoStagesStorage(repoAddress string, containerBackend container_backend.ContainerBackend, dockerRegistry docker_registry.Interface) *RepoStagesStorage { return &RepoStagesStorage{ RepoAddress: repoAddress, DockerRegistry: dockerRegistry, ContainerBackend: containerBackend, - }, nil + } } func (storage *RepoStagesStorage) ConstructStageImageName(_, digest string, uniqueID int64) string { diff --git a/pkg/storage/stages_storage.go b/pkg/storage/stages_storage.go index 894deb28e0..bf8f1e7c79 100644 --- a/pkg/storage/stages_storage.go +++ b/pkg/storage/stages_storage.go @@ -89,15 +89,3 @@ func (rec *ClientIDRecord) String() string { type ImageMetadata struct { ContentDigest string } - -type StagesStorageOptions struct { - RepoStagesStorageOptions -} - -func NewStagesStorage(stagesStorageAddress string, containerBackend container_backend.ContainerBackend, options StagesStorageOptions) (StagesStorage, error) { - if stagesStorageAddress == LocalStorageAddress { - return NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)), nil - } else { // Docker registry based stages storage - return NewRepoStagesStorage(stagesStorageAddress, containerBackend, options.RepoStagesStorageOptions) - } -} diff --git a/test/pkg/utils/storage.go b/test/pkg/utils/storage.go index c4814f9473..07468efd2f 100644 --- a/test/pkg/utils/storage.go +++ b/test/pkg/utils/storage.go @@ -11,19 +11,13 @@ import ( ) func NewStagesStorage(stagesStorageAddress string, implementationName string, dockerRegistryOptions docker_registry.DockerRegistryOptions) storage.StagesStorage { - s, err := storage.NewStagesStorage( - stagesStorageAddress, - &container_backend.DockerServerBackend{}, - storage.StagesStorageOptions{ - RepoStagesStorageOptions: storage.RepoStagesStorageOptions{ - DockerRegistryOptions: dockerRegistryOptions, - ContainerRegistry: implementationName, - }, - }, - ) - Ω(err).ShouldNot(HaveOccurred()) - - return s + if stagesStorageAddress == storage.LocalStorageAddress { + return storage.NewDockerServerStagesStorage(&container_backend.DockerServerBackend{}) + } else { + dockerRegistry, err := docker_registry.NewDockerRegistry(stagesStorageAddress, implementationName, dockerRegistryOptions) + Expect(err).ShouldNot(HaveOccurred()) + return storage.NewRepoStagesStorage(stagesStorageAddress, &container_backend.DockerServerBackend{}, dockerRegistry) + } } func StagesCount(ctx context.Context, stagesStorage storage.StagesStorage) int {