diff --git a/cmd/werf/build/main.go b/cmd/werf/build/main.go index e5431979c7..610c042687 100644 --- a/cmd/werf/build/main.go +++ b/cmd/werf/build/main.go @@ -81,8 +81,8 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo, to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/bundle/apply/apply.go b/cmd/werf/bundle/apply/apply.go index 57d3583cfc..fb14e7e86a 100644 --- a/cmd/werf/bundle/apply/apply.go +++ b/cmd/werf/bundle/apply/apply.go @@ -61,8 +61,8 @@ func NewCmd() *cobra.Command { common.SetupTmpDir(&commonCmdData, cmd, common.SetupTmpDirOptions{}) common.SetupHomeDir(&commonCmdData, cmd, common.SetupHomeDirOptions{}) - common.SetupStagesStorageOptions(&commonCmdData, cmd) // FIXME - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) // FIXME + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo, to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/bundle/copy/copy.go b/cmd/werf/bundle/copy/copy.go new file mode 100644 index 0000000000..d9b9cf98c7 --- /dev/null +++ b/cmd/werf/bundle/copy/copy.go @@ -0,0 +1,116 @@ +package copy + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + helm_v3 "helm.sh/helm/v3/cmd/helm" + + "github.com/werf/werf/cmd/werf/common" + "github.com/werf/werf/pkg/deploy/bundles" + "github.com/werf/werf/pkg/werf" + "github.com/werf/werf/pkg/werf/global_warnings" +) + +var cmdData struct { + Repo *common.RepoData + Tag string + To *common.RepoData + ToTag string +} + +var commonCmdData common.CmdData + +func NewCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "copy", + Short: "Copy published bundle into another location", + Long: common.GetLongCommandDescription(`Take latest bundle from the specified container registry using specified version tag and copy it either into a different tag within the same container registry or into another container registry.`), + DisableFlagsInUseLine: true, + Annotations: map[string]string{ + common.CmdEnvAnno: common.EnvsDescription(), + }, + RunE: func(cmd *cobra.Command, args []string) error { + defer global_warnings.PrintGlobalWarnings(common.GetContext()) + + if err := common.ProcessLogOptions(&commonCmdData); err != nil { + common.PrintHelp(cmd) + return err + } + + common.LogVersion() + + return common.LogRunningTime(runCopy) + }, + } + + common.SetupTmpDir(&commonCmdData, cmd, common.SetupTmpDirOptions{}) + common.SetupHomeDir(&commonCmdData, cmd, common.SetupHomeDirOptions{}) + + common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repos") + common.SetupInsecureRegistry(&commonCmdData, cmd) + common.SetupInsecureHelmDependencies(&commonCmdData, cmd) + common.SetupSkipTlsVerifyRegistry(&commonCmdData, cmd) + + cmdData.Repo = common.NewRepoData("repo", common.RepoDataOptions{OnlyAddress: true}) + cmdData.Repo.SetupCmd(cmd) + + cmdData.To = common.NewRepoData("to", common.RepoDataOptions{OnlyAddress: true}) + cmdData.To.SetupCmd(cmd) + + common.SetupLogOptions(&commonCmdData, cmd) + common.SetupLogProjectDir(&commonCmdData, cmd) + common.SetupPlatform(&commonCmdData, cmd) + + defaultTag := os.Getenv("WERF_TAG") + if defaultTag == "" { + defaultTag = "latest" + } + cmd.Flags().StringVarP(&cmdData.Tag, "tag", "", defaultTag, "Provide from tag version of the bundle to copy ($WERF_TAG or latest by default)") + cmd.Flags().StringVarP(&cmdData.ToTag, "to-tag", "", "", "Provide to tag version of the bundle to copy ($WERF_TO_TAG or same as --tag by default)") + + return cmd +} + +func runCopy() error { + ctx := common.GetContext() + + if err := werf.Init(*commonCmdData.TmpDir, *commonCmdData.HomeDir); err != nil { + return fmt.Errorf("initialization error: %w", err) + } + + if err := common.DockerRegistryInit(ctx, &commonCmdData); err != nil { + return err + } + + helm_v3.Settings.Debug = *commonCmdData.LogDebug + + bundlesRegistryClient, err := common.NewBundlesRegistryClient(ctx, &commonCmdData) + if err != nil { + return err + } + + if *cmdData.Repo.Address == "" { + return fmt.Errorf("--repo=ADDRESS param required") + } + if *cmdData.To.Address == "" { + return fmt.Errorf("--to=ADDRESS param required") + } + + fromRegistry, err := cmdData.Repo.CreateDockerRegistry(*commonCmdData.InsecureRegistry, *commonCmdData.SkipTlsVerifyRegistry) + if err != nil { + return fmt.Errorf("error creating container registry accessor for repo %s: %w", *cmdData.Repo.Address, err) + } + + fromTag := cmdData.Tag + toTag := cmdData.ToTag + if toTag == "" { + toTag = fromTag + } + + fromRef := fmt.Sprintf("%s:%s", *cmdData.Repo.Address, fromTag) + toRef := fmt.Sprintf("%s:%s", *cmdData.To.Address, toTag) + + return bundles.Copy(ctx, fromRef, toRef, bundlesRegistryClient, fromRegistry) +} diff --git a/cmd/werf/bundle/download/download.go b/cmd/werf/bundle/download/download.go index f1821f9bc2..533e120a44 100644 --- a/cmd/werf/bundle/download/download.go +++ b/cmd/werf/bundle/download/download.go @@ -5,7 +5,7 @@ import ( "os" "github.com/spf13/cobra" - "helm.sh/helm/v3/cmd/helm" + helm_v3 "helm.sh/helm/v3/cmd/helm" "github.com/werf/werf/cmd/werf/common" "github.com/werf/werf/pkg/deploy/bundles" @@ -50,8 +50,8 @@ func NewCmd() *cobra.Command { common.SetupInsecureHelmDependencies(&commonCmdData, cmd) common.SetupSkipTlsVerifyRegistry(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) // FIXME - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) // FIXME + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupLogOptions(&commonCmdData, cmd) common.SetupLogProjectDir(&commonCmdData, cmd) diff --git a/cmd/werf/bundle/export/export.go b/cmd/werf/bundle/export/export.go index 74bc2642d2..ad41d306d2 100644 --- a/cmd/werf/bundle/export/export.go +++ b/cmd/werf/bundle/export/export.go @@ -82,8 +82,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo and to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/bundle/publish/publish.go b/cmd/werf/bundle/publish/publish.go index 056caed589..9d323d0532 100644 --- a/cmd/werf/bundle/publish/publish.go +++ b/cmd/werf/bundle/publish/publish.go @@ -88,8 +88,8 @@ Published into container registry bundle can be rolled out by the "werf bundle" common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo and to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/bundle/render/render.go b/cmd/werf/bundle/render/render.go index a0ffeb2bf9..57bc6ade3d 100644 --- a/cmd/werf/bundle/render/render.go +++ b/cmd/werf/bundle/render/render.go @@ -9,7 +9,7 @@ import ( uuid "github.com/satori/go.uuid" "github.com/spf13/cobra" - "helm.sh/helm/v3/cmd/helm" + helm_v3 "helm.sh/helm/v3/cmd/helm" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" @@ -64,8 +64,8 @@ func NewCmd() *cobra.Command { common.SetupTmpDir(&commonCmdData, cmd, common.SetupTmpDirOptions{}) common.SetupHomeDir(&commonCmdData, cmd, common.SetupHomeDirOptions{}) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo, to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) @@ -114,17 +114,17 @@ func runRender(ctx context.Context) error { var isLocal bool switch { case cmdData.BundleDir != "": - if *commonCmdData.StagesStorage != "" { + if *commonCmdData.Repo.Address != "" { return fmt.Errorf("only one of --bundle-dir or --repo should be specified, but both provided") } - if *commonCmdData.FinalStagesStorage != "" { + if *commonCmdData.FinalRepo.Address != "" { return fmt.Errorf("only one of --bundle-dir or --final-repo should be specified, but both provided") } isLocal = true - case *commonCmdData.StagesStorage == storage.LocalStorageAddress: + case *commonCmdData.Repo.Address == storage.LocalStorageAddress: return fmt.Errorf("--repo %s is not allowed, specify remote storage address", storage.LocalStorageAddress) - case *commonCmdData.StagesStorage != "": + case *commonCmdData.Repo.Address != "": isLocal = false default: return fmt.Errorf("either --bundle-dir or --repo required") diff --git a/cmd/werf/cleanup/cleanup.go b/cmd/werf/cleanup/cleanup.go index 07e98bae5a..b4798c3c1f 100644 --- a/cmd/werf/cleanup/cleanup.go +++ b/cmd/werf/cleanup/cleanup.go @@ -64,8 +64,8 @@ It is safe to run this command periodically (daily is enough) by automated clean common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultCleanupParallelTasksLimit) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and delete images from the specified repo") diff --git a/cmd/werf/common/common.go b/cmd/werf/common/common.go index 68aea229f1..24bdf65bdb 100644 --- a/cmd/werf/common/common.go +++ b/cmd/werf/common/common.go @@ -64,11 +64,8 @@ type CmdData struct { SecretValues *[]string IgnoreSecretKey *bool - CommonRepoData *RepoData - StagesStorage *string - - CommonFinalRepoData *RepoData - FinalStagesStorage *string + Repo *RepoData + FinalRepo *RepoData SecondaryStagesStorage *[]string CacheStagesStorage *[]string @@ -390,34 +387,6 @@ func GetFirstExistingEnvVarAsString(envNames ...string) string { return "" } -func SetupCommonRepoData(cmdData *CmdData, cmd *cobra.Command) { - cmdData.CommonRepoData = &RepoData{IsCommon: true} - - SetupImplementationForRepoData(cmdData.CommonRepoData, cmd, "repo-implementation", []string{"WERF_REPO_IMPLEMENTATION"}) // legacy - SetupContainerRegistryForRepoData(cmdData.CommonRepoData, cmd, "repo-container-registry", []string{"WERF_REPO_CONTAINER_REGISTRY"}) - SetupDockerHubUsernameForRepoData(cmdData.CommonRepoData, cmd, "repo-docker-hub-username", []string{"WERF_REPO_DOCKER_HUB_USERNAME"}) - SetupDockerHubPasswordForRepoData(cmdData.CommonRepoData, cmd, "repo-docker-hub-password", []string{"WERF_REPO_DOCKER_HUB_PASSWORD"}) - SetupDockerHubTokenForRepoData(cmdData.CommonRepoData, cmd, "repo-docker-hub-token", []string{"WERF_REPO_DOCKER_HUB_TOKEN"}) - SetupGithubTokenForRepoData(cmdData.CommonRepoData, cmd, "repo-github-token", []string{"WERF_REPO_GITHUB_TOKEN"}) - SetupHarborUsernameForRepoData(cmdData.CommonRepoData, cmd, "repo-harbor-username", []string{"WERF_REPO_HARBOR_USERNAME"}) - SetupHarborPasswordForRepoData(cmdData.CommonRepoData, cmd, "repo-harbor-password", []string{"WERF_REPO_HARBOR_PASSWORD"}) - SetupQuayTokenForRepoData(cmdData.CommonRepoData, cmd, "repo-quay-token", []string{"WERF_REPO_QUAY_TOKEN"}) -} - -func SetupCommonFinalRepoData(cmdData *CmdData, cmd *cobra.Command) { - cmdData.CommonFinalRepoData = &RepoData{} - - cmdData.CommonFinalRepoData.Implementation = new(string) // legacy - SetupContainerRegistryForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-container-registry", []string{"WERF_FINAL_REPO_CONTAINER_REGISTRY"}) - SetupDockerHubUsernameForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-docker-hub-username", []string{"WERF_FINAL_REPO_DOCKER_HUB_USERNAME"}) - SetupDockerHubPasswordForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-docker-hub-password", []string{"WERF_FINAL_REPO_DOCKER_HUB_PASSWORD"}) - SetupDockerHubTokenForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-docker-hub-token", []string{"WERF_FINAL_REPO_DOCKER_HUB_TOKEN"}) - SetupGithubTokenForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-github-token", []string{"WERF_FINAL_REPO_GITHUB_TOKEN"}) - SetupHarborUsernameForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-harbor-username", []string{"WERF_FINAL_REPO_HARBOR_USERNAME"}) - SetupHarborPasswordForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-harbor-password", []string{"WERF_FINAL_REPO_HARBOR_PASSWORD"}) - SetupQuayTokenForRepoData(cmdData.CommonFinalRepoData, cmd, "final-repo-quay-token", []string{"WERF_FINAL_REPO_QUAY_TOKEN"}) -} - func SetupSecondaryStagesStorageOptions(cmdData *CmdData, cmd *cobra.Command) { cmdData.SecondaryStagesStorage = new([]string) cmd.Flags().StringArrayVarP(cmdData.SecondaryStagesStorage, "secondary-repo", "", []string{}, `Specify one or multiple secondary read-only repos with images that will be used as a cache. @@ -450,26 +419,20 @@ func SetupCacheStagesStorageOptions(cmdData *CmdData, cmd *cobra.Command) { Also, can be specified with $WERF_CACHE_REPO_* (e.g. $WERF_CACHE_REPO_1=..., $WERF_CACHE_REPO_2=...)`) } -func SetupStagesStorageOptions(cmdData *CmdData, cmd *cobra.Command) { +func SetupRepoOptions(cmdData *CmdData, cmd *cobra.Command) { SetupInsecureRegistry(cmdData, cmd) SetupSkipTlsVerifyRegistry(cmdData, cmd) - SetupCommonRepoData(cmdData, cmd) - setupStagesStorage(cmdData, cmd) -} - -func SetupFinalStagesStorageOptions(cmdData *CmdData, cmd *cobra.Command) { - SetupCommonFinalRepoData(cmdData, cmd) - setupFinalStagesStorage(cmdData, cmd) + SetupRepo(cmdData, cmd) } -func setupStagesStorage(cmdData *CmdData, cmd *cobra.Command) { - cmdData.StagesStorage = new(string) - cmd.Flags().StringVarP(cmdData.StagesStorage, "repo", "", os.Getenv("WERF_REPO"), fmt.Sprintf("Docker Repo to store stages (default $WERF_REPO)")) +func SetupRepo(cmdData *CmdData, cmd *cobra.Command) { + cmdData.Repo = NewRepoData("repo", RepoDataOptions{}) + cmdData.Repo.SetupCmd(cmd) } -func setupFinalStagesStorage(cmdData *CmdData, cmd *cobra.Command) { - cmdData.FinalStagesStorage = new(string) - cmd.Flags().StringVarP(cmdData.FinalStagesStorage, "final-repo", "", os.Getenv("WERF_FINAL_REPO"), fmt.Sprintf("Docker Repo to store only those stages which are going to be used by the Kubernetes cluster, in other word final images (default $WERF_FINAL_REPO)")) +func SetupFinalRepo(cmdData *CmdData, cmd *cobra.Command) { + cmdData.FinalRepo = NewRepoData("final-repo", RepoDataOptions{}) + cmdData.FinalRepo.SetupCmd(cmd) } func SetupStatusProgressPeriod(cmdData *CmdData, cmd *cobra.Command) { @@ -975,27 +938,23 @@ func GetParallelTasksLimit(cmdData *CmdData) (int64, error) { } func GetStagesStorageAddress(cmdData *CmdData) (string, error) { - if *cmdData.StagesStorage == "" || *cmdData.StagesStorage == storage.LocalStorageAddress { + if *cmdData.Repo.Address == "" || *cmdData.Repo.Address == storage.LocalStorageAddress { return "", fmt.Errorf("--repo=ADDRESS param required") } - return *cmdData.StagesStorage, nil + return *cmdData.Repo.Address, nil } func GetOptionalStagesStorageAddress(cmdData *CmdData) string { - if *cmdData.StagesStorage == "" { + if *cmdData.Repo.Address == "" { return storage.LocalStorageAddress } - return *cmdData.StagesStorage + return *cmdData.Repo.Address } -func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) (storage.StagesStorage, error) { - return storage.NewStagesStorage( - storage.LocalStorageAddress, - containerBackend, - storage.StagesStorageOptions{}, - ) +func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) storage.StagesStorage { + return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)) } func GetStagesStorage(stagesStorageAddress string, containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.StagesStorage, error) { @@ -1004,74 +963,28 @@ func GetStagesStorage(stagesStorageAddress string, containerBackend container_ba return nil, fmt.Errorf(`"--repo" should be specified and not equal ":local" for Buildah container backend`) } } - - if err := ValidateRepoContainerRegistry(cmdData.CommonRepoData.GetContainerRegistry()); err != nil { - return nil, err - } - - return storage.NewStagesStorage( - stagesStorageAddress, - containerBackend, - storage.StagesStorageOptions{ - RepoStagesStorageOptions: storage.RepoStagesStorageOptions{ - ContainerRegistry: cmdData.CommonRepoData.GetContainerRegistry(), - DockerRegistryOptions: docker_registry.DockerRegistryOptions{ - InsecureRegistry: *cmdData.InsecureRegistry, - SkipTlsVerifyRegistry: *cmdData.SkipTlsVerifyRegistry, - DockerHubUsername: *cmdData.CommonRepoData.DockerHubUsername, - DockerHubPassword: *cmdData.CommonRepoData.DockerHubPassword, - DockerHubToken: *cmdData.CommonRepoData.DockerHubToken, - GitHubToken: *cmdData.CommonRepoData.GitHubToken, - HarborUsername: *cmdData.CommonRepoData.HarborUsername, - HarborPassword: *cmdData.CommonRepoData.HarborPassword, - QuayToken: *cmdData.CommonRepoData.QuayToken, - }, - }, - }, - ) + return cmdData.Repo.CreateStagesStorage(containerBackend, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) } func GetOptionalFinalStagesStorage(containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.StagesStorage, error) { - finalRepoAddress := *cmdData.FinalStagesStorage - if finalRepoAddress == "" { + if *cmdData.FinalRepo.Address == "" { return nil, nil } - - if err := ValidateRepoContainerRegistry(cmdData.CommonFinalRepoData.GetContainerRegistry()); err != nil { - return nil, err - } - - return storage.NewStagesStorage( - finalRepoAddress, - containerBackend, - storage.StagesStorageOptions{ - RepoStagesStorageOptions: storage.RepoStagesStorageOptions{ - ContainerRegistry: cmdData.CommonFinalRepoData.GetContainerRegistry(), - DockerRegistryOptions: docker_registry.DockerRegistryOptions{ - InsecureRegistry: *cmdData.InsecureRegistry, - SkipTlsVerifyRegistry: *cmdData.SkipTlsVerifyRegistry, - DockerHubUsername: *cmdData.CommonFinalRepoData.DockerHubUsername, - DockerHubPassword: *cmdData.CommonFinalRepoData.DockerHubPassword, - DockerHubToken: *cmdData.CommonFinalRepoData.DockerHubToken, - GitHubToken: *cmdData.CommonFinalRepoData.GitHubToken, - HarborUsername: *cmdData.CommonFinalRepoData.HarborUsername, - HarborPassword: *cmdData.CommonFinalRepoData.HarborPassword, - QuayToken: *cmdData.CommonFinalRepoData.QuayToken, - }, - }, - }, - ) + return cmdData.FinalRepo.CreateStagesStorage(containerBackend, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) } func GetCacheStagesStorageList(containerBackend container_backend.ContainerBackend, cmdData *CmdData) ([]storage.StagesStorage, error) { var res []storage.StagesStorage for _, address := range GetCacheStagesStorage(cmdData) { - repoStagesStorage, err := storage.NewStagesStorage(address, containerBackend, storage.StagesStorageOptions{}) + repoData := NewRepoData("cache-repo", RepoDataOptions{}) + repoData.Address = &address + + storage, err := repoData.CreateStagesStorage(containerBackend, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) if err != nil { - return nil, fmt.Errorf("unable to create cache stages storage at %s: %w", address, err) + return nil, fmt.Errorf("unable to create cache stages storage in %s: %w", address, err) } - res = append(res, repoStagesStorage) + res = append(res, storage) } return res, nil @@ -1080,22 +993,21 @@ func GetCacheStagesStorageList(containerBackend container_backend.ContainerBacke func GetSecondaryStagesStorageList(stagesStorage storage.StagesStorage, containerBackend container_backend.ContainerBackend, cmdData *CmdData) ([]storage.StagesStorage, error) { var res []storage.StagesStorage - if _, matched := containerBackend.(*container_backend.DockerServerBackend); matched { + if dockerBackend, matched := containerBackend.(*container_backend.DockerServerBackend); matched { if stagesStorage.Address() != storage.LocalStorageAddress { - localStagesStorage, err := storage.NewStagesStorage(storage.LocalStorageAddress, containerBackend, storage.StagesStorageOptions{}) - if err != nil { - return nil, fmt.Errorf("unable to create local secondary stages storage: %w", err) - } - res = append(res, localStagesStorage) + res = append(res, storage.NewDockerServerStagesStorage(dockerBackend)) } } for _, address := range GetSecondaryStagesStorage(cmdData) { - repoStagesStorage, err := storage.NewStagesStorage(address, containerBackend, storage.StagesStorageOptions{}) + repoData := NewRepoData("secondary-repo", RepoDataOptions{}) + repoData.Address = &address + + storage, err := repoData.CreateStagesStorage(containerBackend, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) if err != nil { - return nil, fmt.Errorf("unable to create secondary stages storage at %s: %w", address, err) + return nil, fmt.Errorf("unable to create secondary stages storage in %s: %w", address, err) } - res = append(res, repoStagesStorage) + res = append(res, storage) } return res, nil @@ -1468,19 +1380,6 @@ func DockerRegistryInit(ctx context.Context, cmdData *CmdData) error { return docker_registry.Init(ctx, *cmdData.InsecureRegistry, *cmdData.SkipTlsVerifyRegistry) } -func ValidateRepoContainerRegistry(containerRegistry string) error { - supportedValues := docker_registry.ImplementationList() - supportedValues = append(supportedValues, "auto", "") - - for _, supportedContainerRegistry := range supportedValues { - if supportedContainerRegistry == containerRegistry { - return nil - } - } - - return fmt.Errorf("specified container registry %q is not supported", containerRegistry) -} - func ValidateMinimumNArgs(minArgs int, args []string, cmd *cobra.Command) error { if len(args) < minArgs { PrintHelp(cmd) diff --git a/cmd/werf/common/conveyor_options.go b/cmd/werf/common/conveyor_options.go index 19c771e567..ff5889d4b3 100644 --- a/cmd/werf/common/conveyor_options.go +++ b/cmd/werf/common/conveyor_options.go @@ -84,7 +84,7 @@ func getCustomTagFuncList(commonCmdData *CmdData, giterminismManager giterminism return nil, nil } - if *commonCmdData.StagesStorage == "" || *commonCmdData.StagesStorage == storage.LocalStorageAddress { + if *commonCmdData.Repo.Address == "" || *commonCmdData.Repo.Address == storage.LocalStorageAddress { return nil, fmt.Errorf("custom tags can only be used with remote storage: --repo=ADDRESS param required") } diff --git a/cmd/werf/common/repo_data.go b/cmd/werf/common/repo_data.go new file mode 100644 index 0000000000..ceb7595850 --- /dev/null +++ b/cmd/werf/common/repo_data.go @@ -0,0 +1,339 @@ +package common + +import ( + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/container_backend" + "github.com/werf/werf/pkg/docker_registry" + "github.com/werf/werf/pkg/storage" +) + +func (repoData *RepoData) CreateDockerRegistry(insecureRegistry, skipTlsVerifyRegistry bool) (docker_registry.Interface, error) { + cr := repoData.GetContainerRegistry() + if err := ValidateRepoContainerRegistry(cr); err != nil { + return nil, err + } + + regOpts := repoData.GetDockerRegistryOptions(insecureRegistry, skipTlsVerifyRegistry) + dockerRegistry, err := docker_registry.NewDockerRegistry(*repoData.Address, cr, regOpts) + if err != nil { + return nil, fmt.Errorf("error creating container registry accessor for repo %q: %w", *repoData.Address, err) + } + + return dockerRegistry, nil +} + +func (repoData *RepoData) CreateStagesStorage(containerBackend container_backend.ContainerBackend, insecureRegistry, skipTlsVerifyRegistry bool) (storage.StagesStorage, error) { + if *repoData.Address == storage.LocalStorageAddress { + return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)), nil + } else { + dockerRegistry, err := repoData.CreateDockerRegistry(insecureRegistry, skipTlsVerifyRegistry) + if err != nil { + return nil, fmt.Errorf("error creating container registry accessor for repo %s: %w", *repoData.Address, err) + } + return storage.NewRepoStagesStorage(*repoData.Address, containerBackend, dockerRegistry), nil + } +} + +type RepoData struct { + Name string + + Address *string + Implementation *string // legacy + ContainerRegistry *string + DockerHubUsername *string + DockerHubPassword *string + DockerHubToken *string + GitHubToken *string + HarborUsername *string + HarborPassword *string + QuayToken *string + + RepoDataOptions +} + +type RepoDataOptions struct { + OnlyAddress bool +} + +func NewRepoData(name string, opts RepoDataOptions) *RepoData { + return &RepoData{Name: name, RepoDataOptions: opts} +} + +func (d *RepoData) GetContainerRegistry() string { + if d.OnlyAddress { + return "" + } + + switch { + case *d.ContainerRegistry != "": + return *d.ContainerRegistry + case *d.Implementation != "": + logboek.Context(GetContext()).Warn().LogLn("DEPRECATION WARNING: The option --repo-implementation ($WERF_REPO_IMPLEMENTATION) is renamed to --repo-container-registry ($WERF_REPO_CONTAINER_REGISTRY) and will be removed in v1.3!") + return *d.Implementation + default: + return "" + } +} + +func (d *RepoData) GetDockerRegistryOptions(insecureRegistry, skipTlsVerifyRegistry bool) docker_registry.DockerRegistryOptions { + opts := docker_registry.DockerRegistryOptions{ + InsecureRegistry: insecureRegistry, + SkipTlsVerifyRegistry: skipTlsVerifyRegistry, + } + + if !d.OnlyAddress { + opts.DockerHubUsername = *d.DockerHubUsername + opts.DockerHubPassword = *d.DockerHubPassword + opts.DockerHubToken = *d.DockerHubToken + opts.GitHubToken = *d.GitHubToken + opts.HarborUsername = *d.HarborUsername + opts.HarborPassword = *d.HarborPassword + opts.QuayToken = *d.QuayToken + } + + return opts + + return docker_registry.DockerRegistryOptions{ + DockerHubUsername: *d.DockerHubUsername, + DockerHubPassword: *d.DockerHubPassword, + DockerHubToken: *d.DockerHubToken, + GitHubToken: *d.GitHubToken, + HarborUsername: *d.HarborUsername, + HarborPassword: *d.HarborPassword, + QuayToken: *d.QuayToken, + } +} + +func (repoData *RepoData) SetupCmd(cmd *cobra.Command) { + repoNameUpper := strings.ToUpper(strings.ReplaceAll(repoData.Name, "-", "_")) + makeEnvVar := func(variableSuffix string) string { + return fmt.Sprintf("WERF_%s_%s", repoNameUpper, variableSuffix) + } + makeOpt := func(optSuffix string) string { + return fmt.Sprintf("%s-%s", repoData.Name, optSuffix) + } + + repoData.SetupAddressForRepoData(cmd, repoData.Name, []string{fmt.Sprintf("WERF_%s", repoNameUpper)}) + if repoData.OnlyAddress { + return + } + + repoData.SetupImplementationForRepoData(cmd, makeOpt("implementation"), []string{makeEnvVar("IMPLEMENTATION")}) // legacy + repoData.SetupContainerRegistryForRepoData(cmd, makeOpt("container-registry"), []string{makeEnvVar("CONTAINER_REGISTRY")}) + repoData.SetupDockerHubUsernameForRepoData(cmd, makeOpt("docker-hub-username"), []string{makeEnvVar("DOCKER_HUB_USERNAME")}) + repoData.SetupDockerHubPasswordForRepoData(cmd, makeOpt("docker-hub-password"), []string{makeEnvVar("DOCKER_HUB_PASSWORD")}) + repoData.SetupDockerHubTokenForRepoData(cmd, makeOpt("docker-hub-token"), []string{makeEnvVar("DOCKER_HUB_TOKEN")}) + repoData.SetupGithubTokenForRepoData(cmd, makeOpt("github-token"), []string{makeEnvVar("GITHUB_TOKEN")}) + repoData.SetupHarborUsernameForRepoData(cmd, makeOpt("harbor-username"), []string{makeEnvVar("HARBOR_USERNAME")}) + repoData.SetupHarborPasswordForRepoData(cmd, makeOpt("harbor-password"), []string{makeEnvVar("HARBOR_PASSWORD")}) + repoData.SetupQuayTokenForRepoData(cmd, makeOpt("quay-token"), []string{makeEnvVar("QUAY_TOKEN")}) +} + +func MergeRepoData(repoDataArr ...*RepoData) *RepoData { + res := &RepoData{} + + for _, repoData := range repoDataArr { + if res.GetContainerRegistry() == "" { + value := repoData.GetContainerRegistry() + res.ContainerRegistry = &value + } + if res.DockerHubUsername == nil || *res.DockerHubUsername == "" { + res.DockerHubUsername = repoData.DockerHubUsername + } + if res.DockerHubPassword == nil || *res.DockerHubPassword == "" { + res.DockerHubPassword = repoData.DockerHubPassword + } + if res.DockerHubToken == nil || *res.DockerHubToken == "" { + res.DockerHubToken = repoData.DockerHubToken + } + if res.GitHubToken == nil || *res.GitHubToken == "" { + res.GitHubToken = repoData.GitHubToken + } + if res.HarborUsername == nil || *res.HarborUsername == "" { + res.HarborUsername = repoData.HarborUsername + } + if res.HarborPassword == nil || *res.HarborPassword == "" { + res.HarborPassword = repoData.HarborPassword + } + if res.QuayToken == nil || *res.QuayToken == "" { + res.QuayToken = repoData.QuayToken + } + } + + return res +} + +func (repoData *RepoData) SetupAddressForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("Container registry storage address (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.Address = new(string) + cmd.Flags().StringVarP( + repoData.Address, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +// legacy +func (repoData *RepoData) SetupImplementationForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + repoData.Implementation = new(string) + cmd.Flags().StringVarP( + repoData.Implementation, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + "", + ) + cmd.Flag(paramName).Hidden = true +} + +func (repoData *RepoData) SetupContainerRegistryForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usageTitle := fmt.Sprintf("Choose %s container registry implementation", repoData.Name) + + repoData.ContainerRegistry = new(string) + cmd.Flags().StringVarP( + repoData.ContainerRegistry, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + fmt.Sprintf(`%s. +The following container registries are supported: %s. +Default %s or auto mode (detect container registry by repo address).`, + usageTitle, + strings.Join(docker_registry.ImplementationList(), ", "), + strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", "), + ), + ) +} + +func (repoData *RepoData) SetupDockerHubUsernameForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Docker Hub username (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.DockerHubUsername = new(string) + cmd.Flags().StringVarP( + repoData.DockerHubUsername, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupDockerHubPasswordForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Docker Hub password (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.DockerHubPassword = new(string) + cmd.Flags().StringVarP( + repoData.DockerHubPassword, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupDockerHubTokenForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Docker Hub token (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.DockerHubToken = new(string) + cmd.Flags().StringVarP( + repoData.DockerHubToken, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupGithubTokenForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s GitHub token (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.GitHubToken = new(string) + cmd.Flags().StringVarP( + repoData.GitHubToken, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupHarborUsernameForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Harbor username (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.HarborUsername = new(string) + cmd.Flags().StringVarP( + repoData.HarborUsername, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupHarborPasswordForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s Harbor password (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.HarborPassword = new(string) + cmd.Flags().StringVarP( + repoData.HarborPassword, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func (repoData *RepoData) SetupQuayTokenForRepoData(cmd *cobra.Command, paramName string, paramEnvNames []string) { + usage := fmt.Sprintf("%s quay.io token (default %s)", repoData.Name, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) + + repoData.QuayToken = new(string) + cmd.Flags().StringVarP( + repoData.QuayToken, + paramName, + "", + getDefaultValueByParamEnvNames(paramEnvNames), + usage, + ) +} + +func getDefaultValueByParamEnvNames(paramEnvNames []string) string { + var defaultValue string + for _, paramEnvName := range paramEnvNames { + value := os.Getenv(paramEnvName) + if value != "" { + defaultValue = value + break + } + } + return defaultValue +} + +func getParamEnvNamesForUsageDescription(paramEnvNames []string) []string { + paramEnvNamesWithDollar := []string{} + for _, paramEnvName := range paramEnvNames { + paramEnvNamesWithDollar = append(paramEnvNamesWithDollar, "$"+paramEnvName) + } + return paramEnvNamesWithDollar +} + +func ValidateRepoContainerRegistry(containerRegistry string) error { + supportedValues := docker_registry.ImplementationList() + supportedValues = append(supportedValues, "auto", "") + + for _, supportedContainerRegistry := range supportedValues { + if supportedContainerRegistry == containerRegistry { + return nil + } + } + + return fmt.Errorf("specified container registry %q is not supported", containerRegistry) +} diff --git a/cmd/werf/common/storage_repo_data.go b/cmd/werf/common/storage_repo_data.go deleted file mode 100644 index 2a3cbee776..0000000000 --- a/cmd/werf/common/storage_repo_data.go +++ /dev/null @@ -1,256 +0,0 @@ -package common - -import ( - "fmt" - "os" - "strings" - - "github.com/spf13/cobra" - - "github.com/werf/logboek" - "github.com/werf/werf/pkg/docker_registry" -) - -type RepoData struct { - IsCommon bool - DesignationStorageName string - - Implementation *string // legacy - ContainerRegistry *string - DockerHubUsername *string - DockerHubPassword *string - DockerHubToken *string - GitHubToken *string - HarborUsername *string - HarborPassword *string - QuayToken *string -} - -func (d *RepoData) GetContainerRegistry() string { - switch { - case *d.ContainerRegistry != "": - return *d.ContainerRegistry - case *d.Implementation != "": - logboek.Context(GetContext()).Warn().LogLn("DEPRECATION WARNING: The option --repo-implementation ($WERF_REPO_IMPLEMENTATION) is renamed to --repo-container-registry ($WERF_REPO_CONTAINER_REGISTRY) and will be removed in v1.3!") - return *d.Implementation - default: - return "" - } -} - -func MergeRepoData(repoDataArr ...*RepoData) *RepoData { - res := &RepoData{} - - for _, repoData := range repoDataArr { - if res.GetContainerRegistry() == "" { - value := repoData.GetContainerRegistry() - res.ContainerRegistry = &value - } - if res.DockerHubUsername == nil || *res.DockerHubUsername == "" { - res.DockerHubUsername = repoData.DockerHubUsername - } - if res.DockerHubPassword == nil || *res.DockerHubPassword == "" { - res.DockerHubPassword = repoData.DockerHubPassword - } - if res.DockerHubToken == nil || *res.DockerHubToken == "" { - res.DockerHubToken = repoData.DockerHubToken - } - if res.GitHubToken == nil || *res.GitHubToken == "" { - res.GitHubToken = repoData.GitHubToken - } - if res.HarborUsername == nil || *res.HarborUsername == "" { - res.HarborUsername = repoData.HarborUsername - } - if res.HarborPassword == nil || *res.HarborPassword == "" { - res.HarborPassword = repoData.HarborPassword - } - if res.QuayToken == nil || *res.QuayToken == "" { - res.QuayToken = repoData.QuayToken - } - } - - return res -} - -// legacy -func SetupImplementationForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - repoData.Implementation = new(string) - cmd.Flags().StringVarP( - repoData.Implementation, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - "", - ) - cmd.Flag(paramName).Hidden = true -} - -func SetupContainerRegistryForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usageTitle string - if repoData.IsCommon { - usageTitle = "Choose repo container registry" - } else { - usageTitle = fmt.Sprintf("Choose repo container registry for %s", repoData.DesignationStorageName) - } - - repoData.ContainerRegistry = new(string) - cmd.Flags().StringVarP( - repoData.ContainerRegistry, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - fmt.Sprintf(`%s. -The following container registries are supported: %s. -Default %s or auto mode (detect container registry by repo address).`, - usageTitle, - strings.Join(docker_registry.ImplementationList(), ", "), - strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", "), - ), - ) -} - -func SetupDockerHubUsernameForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Docker Hub username (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Docker Hub username for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.DockerHubUsername = new(string) - cmd.Flags().StringVarP( - repoData.DockerHubUsername, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupDockerHubPasswordForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Docker Hub password (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Docker Hub password for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.DockerHubPassword = new(string) - cmd.Flags().StringVarP( - repoData.DockerHubPassword, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupDockerHubTokenForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Docker Hub token (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Docker Hub token for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.DockerHubToken = new(string) - cmd.Flags().StringVarP( - repoData.DockerHubToken, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupGithubTokenForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("GitHub token (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("GitHub token for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.GitHubToken = new(string) - cmd.Flags().StringVarP( - repoData.GitHubToken, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupHarborUsernameForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Harbor username (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Harbor username for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.HarborUsername = new(string) - cmd.Flags().StringVarP( - repoData.HarborUsername, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupHarborPasswordForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("Harbor password (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("Harbor password for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.HarborPassword = new(string) - cmd.Flags().StringVarP( - repoData.HarborPassword, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func SetupQuayTokenForRepoData(repoData *RepoData, cmd *cobra.Command, paramName string, paramEnvNames []string) { - var usage string - if repoData.IsCommon { - usage = fmt.Sprintf("quay.io token (default %s)", strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } else { - usage = fmt.Sprintf("quay.io token for %s (default %s)", repoData.DesignationStorageName, strings.Join(getParamEnvNamesForUsageDescription(paramEnvNames), ", ")) - } - - repoData.QuayToken = new(string) - cmd.Flags().StringVarP( - repoData.QuayToken, - paramName, - "", - getDefaultValueByParamEnvNames(paramEnvNames), - usage, - ) -} - -func getDefaultValueByParamEnvNames(paramEnvNames []string) string { - var defaultValue string - for _, paramEnvName := range paramEnvNames { - value := os.Getenv(paramEnvName) - if value != "" { - defaultValue = value - break - } - } - return defaultValue -} - -func getParamEnvNamesForUsageDescription(paramEnvNames []string) []string { - paramEnvNamesWithDollar := []string{} - for _, paramEnvName := range paramEnvNames { - paramEnvNamesWithDollar = append(paramEnvNamesWithDollar, "$"+paramEnvName) - } - return paramEnvNamesWithDollar -} diff --git a/cmd/werf/compose/main.go b/cmd/werf/compose/main.go index 5ff5434f32..34972e643e 100644 --- a/cmd/werf/compose/main.go +++ b/cmd/werf/compose/main.go @@ -210,8 +210,8 @@ services: common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSkipBuild(&commonCmdData, cmd) diff --git a/cmd/werf/converge/converge.go b/cmd/werf/converge/converge.go index 6866beb4b0..af0f45d215 100644 --- a/cmd/werf/converge/converge.go +++ b/cmd/werf/converge/converge.go @@ -102,8 +102,8 @@ werf converge --repo registry.mydomain.com/web --env production`, common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo, to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/dismiss/dismiss.go b/cmd/werf/dismiss/dismiss.go index 5ea6b54946..dd44d82a15 100644 --- a/cmd/werf/dismiss/dismiss.go +++ b/cmd/werf/dismiss/dismiss.go @@ -83,8 +83,8 @@ Read more info about Helm Release name, Kubernetes Namespace and how to change i common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSynchronization(&commonCmdData, cmd) common.SetupRelease(&commonCmdData, cmd) diff --git a/cmd/werf/export/export.go b/cmd/werf/export/export.go index 8e31bdd5fc..a3413761e4 100644 --- a/cmd/werf/export/export.go +++ b/cmd/werf/export/export.go @@ -78,8 +78,8 @@ All meta-information related to werf is removed from the exported images, and th common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSkipBuild(&commonCmdData, cmd) diff --git a/cmd/werf/helm/get_autogenerated_values.go b/cmd/werf/helm/get_autogenerated_values.go index 92eaf94b14..c5cc8f13a0 100644 --- a/cmd/werf/helm/get_autogenerated_values.go +++ b/cmd/werf/helm/get_autogenerated_values.go @@ -63,8 +63,8 @@ These values includes project name, docker images ids and other`), common.SetupSecondaryStagesStorageOptions(&getAutogeneratedValuedCmdData, cmd) common.SetupCacheStagesStorageOptions(&getAutogeneratedValuedCmdData, cmd) - common.SetupStagesStorageOptions(&getAutogeneratedValuedCmdData, cmd) - common.SetupFinalStagesStorageOptions(&getAutogeneratedValuedCmdData, cmd) + common.SetupRepoOptions(&getAutogeneratedValuedCmdData, cmd) + common.SetupFinalRepo(&getAutogeneratedValuedCmdData, cmd) common.SetupSynchronization(&getAutogeneratedValuedCmdData, cmd) common.SetupKubeConfig(&getAutogeneratedValuedCmdData, cmd) diff --git a/cmd/werf/host/purge/purge.go b/cmd/werf/host/purge/purge.go index 54de1cdfa3..c50c52dac2 100644 --- a/cmd/werf/host/purge/purge.go +++ b/cmd/werf/host/purge/purge.go @@ -106,10 +106,7 @@ func runReset(ctx context.Context) error { return err } } else { - stagesStorage, err := common.GetLocalStagesStorage(containerBackend) - if err != nil { - return err - } + stagesStorage := common.GetLocalStagesStorage(containerBackend) synchronization, err := common.GetSynchronization(ctx, &commonCmdData, projectName, stagesStorage) if err != nil { return err diff --git a/cmd/werf/kube_run/kube_run.go b/cmd/werf/kube_run/kube_run.go index 13f09dea29..bca1b3a3ec 100644 --- a/cmd/werf/kube_run/kube_run.go +++ b/cmd/werf/kube_run/kube_run.go @@ -142,8 +142,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSkipBuild(&commonCmdData, cmd) diff --git a/cmd/werf/main.go b/cmd/werf/main.go index d4cb1d23a6..639a8a8ce0 100644 --- a/cmd/werf/main.go +++ b/cmd/werf/main.go @@ -12,6 +12,7 @@ import ( "github.com/werf/logboek" "github.com/werf/werf/cmd/werf/build" bundle_apply "github.com/werf/werf/cmd/werf/bundle/apply" + bundle_copy "github.com/werf/werf/cmd/werf/bundle/copy" bundle_download "github.com/werf/werf/cmd/werf/bundle/download" bundle_export "github.com/werf/werf/cmd/werf/bundle/export" bundle_publish "github.com/werf/werf/cmd/werf/bundle/publish" @@ -186,6 +187,7 @@ func bundleCmd() *cobra.Command { bundle_export.NewCmd(), bundle_download.NewCmd(), bundle_render.NewCmd(), + bundle_copy.NewCmd(), ) return cmd diff --git a/cmd/werf/managed_images/add/add.go b/cmd/werf/managed_images/add/add.go index 0c7c47af5d..766fb89035 100644 --- a/cmd/werf/managed_images/add/add.go +++ b/cmd/werf/managed_images/add/add.go @@ -55,8 +55,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read and write images to the specified repo") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/managed_images/ls/ls.go b/cmd/werf/managed_images/ls/ls.go index 5da3b26e2f..4beb5fdd76 100644 --- a/cmd/werf/managed_images/ls/ls.go +++ b/cmd/werf/managed_images/ls/ls.go @@ -51,8 +51,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read images from the specified repo") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/managed_images/rm/rm.go b/cmd/werf/managed_images/rm/rm.go index f6347d9e4f..aef5314398 100644 --- a/cmd/werf/managed_images/rm/rm.go +++ b/cmd/werf/managed_images/rm/rm.go @@ -55,8 +55,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read and write images to the specified repo") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/purge/purge.go b/cmd/werf/purge/purge.go index 616be72296..0592587d50 100644 --- a/cmd/werf/purge/purge.go +++ b/cmd/werf/purge/purge.go @@ -57,8 +57,8 @@ WARNING: Images that are being used in the Kubernetes cluster will also be delet common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultCleanupParallelTasksLimit) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to delete images from the specified repo") diff --git a/cmd/werf/render/render.go b/cmd/werf/render/render.go index 5113234daf..8dfcfe4e8f 100644 --- a/cmd/werf/render/render.go +++ b/cmd/werf/render/render.go @@ -89,8 +89,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read, pull and push images into the specified repo and to pull base images") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/cmd/werf/run/run.go b/cmd/werf/run/run.go index 269c999687..b024bbd7e4 100644 --- a/cmd/werf/run/run.go +++ b/cmd/werf/run/run.go @@ -124,8 +124,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupSkipBuild(&commonCmdData, cmd) diff --git a/cmd/werf/stage/image/main.go b/cmd/werf/stage/image/main.go index 28f800c262..416af583da 100644 --- a/cmd/werf/stage/image/main.go +++ b/cmd/werf/stage/image/main.go @@ -64,8 +64,8 @@ func NewCmd() *cobra.Command { common.SetupSecondaryStagesStorageOptions(&commonCmdData, cmd) common.SetupCacheStagesStorageOptions(&commonCmdData, cmd) - common.SetupStagesStorageOptions(&commonCmdData, cmd) - common.SetupFinalStagesStorageOptions(&commonCmdData, cmd) + common.SetupRepoOptions(&commonCmdData, cmd) + common.SetupFinalRepo(&commonCmdData, cmd) common.SetupDockerConfig(&commonCmdData, cmd, "Command needs granted permissions to read and pull images from the specified stages storage") common.SetupInsecureRegistry(&commonCmdData, cmd) diff --git a/pkg/deploy/bundles/copy.go b/pkg/deploy/bundles/copy.go new file mode 100644 index 0000000000..14009be43b --- /dev/null +++ b/pkg/deploy/bundles/copy.go @@ -0,0 +1,128 @@ +package bundles + +import ( + "context" + "fmt" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "helm.sh/helm/v3/pkg/chart" + "helm.sh/helm/v3/pkg/chartutil" + "sigs.k8s.io/yaml" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/deploy/bundles/registry" + "github.com/werf/werf/pkg/docker_registry" + "github.com/werf/werf/pkg/util" +) + +func Copy(ctx context.Context, fromAddr, toAddr string, bundlesRegistryClient *registry.Client, fromRegistry docker_registry.Interface) error { + fromRef, err := registry.ParseReference(fromAddr) + if err != nil { + return fmt.Errorf("unable to parse source address %q: %w", fromAddr, err) + } + toRef, err := registry.ParseReference(toAddr) + if err != nil { + return fmt.Errorf("unable to parse destination address %q: %w", toAddr, err) + } + + if err := logboek.Context(ctx).LogProcess("Pulling bundle %s", fromRef.FullName()).DoError(func() error { + if err := bundlesRegistryClient.PullChartToCache(fromRef); err != nil { + return fmt.Errorf("unable to pull bundle %s: %w", fromRef.FullName(), err) + } + return nil + }); err != nil { + return err + } + + var ch *chart.Chart + if err := logboek.Context(ctx).LogProcess("Loading bundle %s", fromRef.FullName()).DoError(func() error { + ch, err = bundlesRegistryClient.LoadChart(fromRef) + if err != nil { + return fmt.Errorf("unable to load pulled bundle %s: %w", fromRef.FullName(), err) + } + return nil + }); err != nil { + return err + } + + { + d, err := yaml.Marshal(ch.Values) + logboek.Context(ctx).Debug().LogF("Values before change (%v):\n%s\n---\n", err, d) + } + + if werfVals, ok := ch.Values["werf"].(map[string]interface{}); ok { + if imageVals, ok := werfVals["image"].(map[string]interface{}); ok { + newImageVals := make(map[string]interface{}) + + for imageName, v := range imageVals { + if image, ok := v.(string); ok { + ref, err := registry.ParseReference(image) + if err != nil { + return fmt.Errorf("unable to parse bundle image %s: %w", image, err) + } + + ref.Repo = toRef.Repo + + // TODO: copy images in parallel + if image != ref.FullName() { + if err := logboek.Context(ctx).LogProcess("Copy image").DoError(func() error { + logboek.Context(ctx).Default().LogFDetails("Source: %s\n", image) + logboek.Context(ctx).Default().LogFDetails("Destination: %s\n", ref.FullName()) + + if err := fromRegistry.MutateAndPushImage(ctx, image, ref.FullName(), func(cfg v1.Config) (v1.Config, error) { return cfg, nil }); err != nil { + return fmt.Errorf("error copying image %s into %s: %w", image, ref.FullName(), err) + } + return nil + }); err != nil { + return err + } + } + + newImageVals[imageName] = ref.FullName() + } else { + return fmt.Errorf("unexpected value .Values.werf.image.%s=%v", imageName, v) + } + } + + werfVals["image"] = newImageVals + } + + werfVals["repo"] = toRef.Repo + } + + valuesRaw, err := yaml.Marshal(ch.Values) + if err != nil { + return fmt.Errorf("unable to marshal changed bundle values: %w", err) + } + logboek.Context(ctx).Debug().LogF("Values after change (%v):\n%s\n---\n", err, valuesRaw) + + for _, f := range ch.Raw { + if f.Name == chartutil.ValuesfileName { + f.Data = valuesRaw + break + } + } + + ch.Metadata.Name = util.Reverse(strings.SplitN(util.Reverse(toRef.Repo), "/", 2)[0]) + + if err := logboek.Context(ctx).LogProcess("Saving bundle %s", toRef.FullName()).DoError(func() error { + if err := bundlesRegistryClient.SaveChart(ch, toRef); err != nil { + return fmt.Errorf("unable to save bundle %s to the local chart helm cache: %w", toRef.FullName(), err) + } + return nil + }); err != nil { + return err + } + + if err := logboek.Context(ctx).LogProcess("Pushing bundle %s", toRef.FullName()).DoError(func() error { + if err := bundlesRegistryClient.PushChart(toRef); err != nil { + return fmt.Errorf("unable to push bundle %s: %w", toRef.FullName(), err) + } + return nil + }); err != nil { + return err + } + + return nil +} diff --git a/pkg/deploy/helm/chart_extender/werf_chart.go b/pkg/deploy/helm/chart_extender/werf_chart.go index c55761630b..4fbcfe5477 100644 --- a/pkg/deploy/helm/chart_extender/werf_chart.go +++ b/pkg/deploy/helm/chart_extender/werf_chart.go @@ -293,7 +293,7 @@ func (wc *WerfChart) CreateNewBundle(ctx context.Context, destDir, chartVersion return nil, fmt.Errorf("unable to construct bundle input values: %w", err) } - valsData, err = json.MarshalIndent(vals, "", " ") + valsData, err = yaml.Marshal(vals) if err != nil { return nil, fmt.Errorf("unable to prepare values: %w", err) } @@ -313,7 +313,7 @@ func (wc *WerfChart) CreateNewBundle(ctx context.Context, destDir, chartVersion logboek.Context(ctx).Debug().LogF("Saving bundle values:\n%s\n---\n", valsData) valuesFile := filepath.Join(destDir, "values.yaml") - if err := ioutil.WriteFile(valuesFile, append(valsData, []byte("\n")...), os.ModePerm); err != nil { + if err := ioutil.WriteFile(valuesFile, valsData, os.ModePerm); err != nil { return nil, fmt.Errorf("unable to write %q: %w", valuesFile, err) } diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index 7e0bf1a15d..48ebda9638 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -64,22 +64,12 @@ type RepoStagesStorage struct { ContainerBackend container_backend.ContainerBackend } -type RepoStagesStorageOptions struct { - docker_registry.DockerRegistryOptions - ContainerRegistry string -} - -func NewRepoStagesStorage(repoAddress string, containerBackend container_backend.ContainerBackend, options RepoStagesStorageOptions) (*RepoStagesStorage, error) { - dockerRegistry, err := docker_registry.NewDockerRegistry(repoAddress, options.ContainerRegistry, options.DockerRegistryOptions) - if err != nil { - return nil, fmt.Errorf("error creating container registry accessor for repo %q: %w", repoAddress, err) - } - +func NewRepoStagesStorage(repoAddress string, containerBackend container_backend.ContainerBackend, dockerRegistry docker_registry.Interface) *RepoStagesStorage { return &RepoStagesStorage{ RepoAddress: repoAddress, DockerRegistry: dockerRegistry, ContainerBackend: containerBackend, - }, nil + } } func (storage *RepoStagesStorage) ConstructStageImageName(_, digest string, uniqueID int64) string { diff --git a/pkg/storage/stages_storage.go b/pkg/storage/stages_storage.go index 894deb28e0..bf8f1e7c79 100644 --- a/pkg/storage/stages_storage.go +++ b/pkg/storage/stages_storage.go @@ -89,15 +89,3 @@ func (rec *ClientIDRecord) String() string { type ImageMetadata struct { ContentDigest string } - -type StagesStorageOptions struct { - RepoStagesStorageOptions -} - -func NewStagesStorage(stagesStorageAddress string, containerBackend container_backend.ContainerBackend, options StagesStorageOptions) (StagesStorage, error) { - if stagesStorageAddress == LocalStorageAddress { - return NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)), nil - } else { // Docker registry based stages storage - return NewRepoStagesStorage(stagesStorageAddress, containerBackend, options.RepoStagesStorageOptions) - } -} diff --git a/test/pkg/utils/storage.go b/test/pkg/utils/storage.go index c4814f9473..07468efd2f 100644 --- a/test/pkg/utils/storage.go +++ b/test/pkg/utils/storage.go @@ -11,19 +11,13 @@ import ( ) func NewStagesStorage(stagesStorageAddress string, implementationName string, dockerRegistryOptions docker_registry.DockerRegistryOptions) storage.StagesStorage { - s, err := storage.NewStagesStorage( - stagesStorageAddress, - &container_backend.DockerServerBackend{}, - storage.StagesStorageOptions{ - RepoStagesStorageOptions: storage.RepoStagesStorageOptions{ - DockerRegistryOptions: dockerRegistryOptions, - ContainerRegistry: implementationName, - }, - }, - ) - Ω(err).ShouldNot(HaveOccurred()) - - return s + if stagesStorageAddress == storage.LocalStorageAddress { + return storage.NewDockerServerStagesStorage(&container_backend.DockerServerBackend{}) + } else { + dockerRegistry, err := docker_registry.NewDockerRegistry(stagesStorageAddress, implementationName, dockerRegistryOptions) + Expect(err).ShouldNot(HaveOccurred()) + return storage.NewRepoStagesStorage(stagesStorageAddress, &container_backend.DockerServerBackend{}, dockerRegistry) + } } func StagesCount(ctx context.Context, stagesStorage storage.StagesStorage) int {