Skip to content

Commit

Permalink
feat(bundle): introduce bundle archive format, implement copy command…
Browse files Browse the repository at this point in the history
… to convert archive to registry and vice versa

* `werf bundle copy` command:
    * `--repo` param has been deprecated in favor of new `--from=ADDR` param
    * `--tag` param has been deprecated in favor of new `--from=ADDR` param
    * `--to-tag` param has been deprecated in favor of `--to=ADDR` param new format
    * New `--from=ADDR` and `--to=ADDR` params accept addr in different formats:
        * `archive:PATH_TO_ARCHIVE.tar.gz` — bundle archive
        * `docker://registry.example.com/group/project` — remote bundle in container registry
        * without explicitly specified schema address parsed also as a remote bundle in container registry
* Bundle archive has following structure:
    * `chart.tar.gz` — regular helm chart archive
    * `images/TAG.tar.gz` — dir contains all werf images referenced from chart values `.Values.werf.image.NAME`, named by tags

Refs werf/3p-helm#216

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Sep 2, 2022
1 parent 3a7c68d commit 345cdf0
Show file tree
Hide file tree
Showing 12 changed files with 895 additions and 87 deletions.
94 changes: 68 additions & 26 deletions cmd/werf/bundle/copy/copy.go
Expand Up @@ -8,17 +8,23 @@ import (
"github.com/spf13/cobra"
helm_v3 "helm.sh/helm/v3/cmd/helm"

"github.com/werf/logboek"
"github.com/werf/werf/cmd/werf/common"
"github.com/werf/werf/pkg/deploy/bundles"
"github.com/werf/werf/pkg/docker_registry"
"github.com/werf/werf/pkg/werf"
"github.com/werf/werf/pkg/werf/global_warnings"
)

var cmdData struct {
Repo *common.RepoData
// TODO(2.0): Legacy {
Repo string
Tag string
To *common.RepoData
ToTag string
// TODO(2.0): } Legacy

From string
To string
}

var commonCmdData common.CmdData
Expand Down Expand Up @@ -57,22 +63,16 @@ func NewCmd(ctx context.Context) *cobra.Command {
common.SetupInsecureHelmDependencies(&commonCmdData, cmd)
common.SetupSkipTlsVerifyRegistry(&commonCmdData, cmd)

cmdData.Repo = common.NewRepoData("repo", common.RepoDataOptions{OnlyAddress: true})
cmdData.Repo.SetupCmd(cmd)

cmdData.To = common.NewRepoData("to", common.RepoDataOptions{OnlyAddress: true})
cmdData.To.SetupCmd(cmd)

common.SetupLogOptions(&commonCmdData, cmd)
common.SetupLogProjectDir(&commonCmdData, cmd)
common.SetupPlatform(&commonCmdData, cmd)

defaultTag := os.Getenv("WERF_TAG")
if defaultTag == "" {
defaultTag = "latest"
}
cmd.Flags().StringVarP(&cmdData.Tag, "tag", "", defaultTag, "Provide from tag version of the bundle to copy ($WERF_TAG or latest by default)")
cmd.Flags().StringVarP(&cmdData.ToTag, "to-tag", "", "", "Provide to tag version of the bundle to copy ($WERF_TO_TAG or same as --tag by default)")
cmd.Flags().StringVarP(&cmdData.Repo, "repo", "", os.Getenv("WERF_REPO"), "Deprecated param, use --from=ADDR instead. Source address of bundle which should be copied.")
cmd.Flags().StringVarP(&cmdData.Tag, "tag", "", os.Getenv("WERF_TAG"), "Deprecated param, use --from=REPO:TAG instead. Provide from tag version of the bundle to copy ($WERF_TAG or latest by default).")
cmd.Flags().StringVarP(&cmdData.ToTag, "to-tag", "", os.Getenv("WERF_TO_TAG"), "Deprecated param, use --to=REPO:TAG instead. Provide to tag version of the bundle to copy ($WERF_TO_TAG or same as --tag by default).")

cmd.Flags().StringVarP(&cmdData.From, "from", "", os.Getenv("WERF_FROM"), "Source address of the bundle to copy, specify bundle archive using schema `archive:PATH_TO_ARCHIVE.tar.gz`, specify remote bundle with schema `[docker://]REPO:TAG` or without schema.")
cmd.Flags().StringVarP(&cmdData.To, "to", "", os.Getenv("WERF_TO"), "Destination address of the bundle to copy, specify bundle archive using schema `archive:PATH_TO_ARCHIVE.tar.gz`, specify remote bundle with schema `[docker://]REPO:TAG` or without schema.")

return cmd
}
Expand All @@ -93,26 +93,68 @@ func runCopy(ctx context.Context) error {
return err
}

if *cmdData.Repo.Address == "" {
return fmt.Errorf("--repo=ADDRESS param required")
fromAddrRaw := cmdData.From
if fromAddrRaw != "" && cmdData.Repo != "" {
return fmt.Errorf("unable to use --repo=ADDRESS and --from=ADDRESS params at the same time: please specify only --from=ADDRESS param, --repo param has been deprecated")
} else if cmdData.Repo != "" {
logboek.Context(ctx).Warn().LogF("Please use --from=ADDRESS param instead of deprecated --repo=ADDRESS param\n")
fromAddrRaw = cmdData.Repo
}
if fromAddrRaw == "" {
return fmt.Errorf("--from=ADDRESS param required")
}
if *cmdData.To.Address == "" {

toAddrRaw := cmdData.To
if toAddrRaw == "" {
return fmt.Errorf("--to=ADDRESS param required")
}

fromRegistry, err := cmdData.Repo.CreateDockerRegistry(ctx, *commonCmdData.InsecureRegistry, *commonCmdData.SkipTlsVerifyRegistry)
fromAddr, err := bundles.ParseAddr(fromAddrRaw)
if err != nil {
return fmt.Errorf("error creating container registry accessor for repo %s: %w", *cmdData.Repo.Address, err)
return fmt.Errorf("invalid from addr %q: %w", fromAddrRaw, err)
}

fromTag := cmdData.Tag
toTag := cmdData.ToTag
if toTag == "" {
toTag = fromTag
toAddr, err := bundles.ParseAddr(toAddrRaw)
if err != nil {
return fmt.Errorf("invalid to addr %q: %w", toAddrRaw, err)
}

fromRef := fmt.Sprintf("%s:%s", *cmdData.Repo.Address, fromTag)
toRef := fmt.Sprintf("%s:%s", *cmdData.To.Address, toTag)
var fromRegistry, toRegistry docker_registry.Interface

return bundles.Copy(ctx, fromRef, toRef, bundlesRegistryClient, fromRegistry)
if fromAddr.RegistryAddress != nil {
// TODO(2.0): remove legacy compatibility param
if cmdData.Tag != "" {
logboek.Context(ctx).Warn().LogF("Please use --from=REPO:TAG tag specification instead of deprecated --tag=TAG param\n")
fromAddr.RegistryAddress.Tag = cmdData.Tag
}

fromRegistry, err = common.CreateDockerRegistry(fromAddr.RegistryAddress.Repo, *commonCmdData.InsecureRegistry, *commonCmdData.SkipTlsVerifyRegistry)
if err != nil {
return err
}
}

if toAddr.RegistryAddress != nil {
// TODO(2.0): remove legacy compatibility param
if cmdData.ToTag != "" {
logboek.Context(ctx).Warn().LogF("Please use --to=REPO:TAG tag specification instead of deprecated --to-tag=TAG param\n")
toAddr.RegistryAddress.Tag = cmdData.ToTag
}

toRegistry, err = common.CreateDockerRegistry(toAddr.RegistryAddress.Repo, *commonCmdData.InsecureRegistry, *commonCmdData.SkipTlsVerifyRegistry)
if err != nil {
return err
}
}

return logboek.Context(ctx).LogProcess("Copy bundle").DoError(func() error {
logboek.Context(ctx).LogFDetails("From: %s\n", fromAddr.String())
logboek.Context(ctx).LogFDetails("To: %s\n", toAddr.String())

return bundles.Copy(ctx, fromAddr, toAddr, bundles.CopyOptions{
BundlesRegistryClient: bundlesRegistryClient,
FromRegistry: fromRegistry,
ToRegistry: toRegistry,
})
})
}
14 changes: 14 additions & 0 deletions cmd/werf/common/repo_data.go
Expand Up @@ -14,6 +14,20 @@ import (
"github.com/werf/werf/pkg/storage"
)

func CreateDockerRegistry(addr string, insecureRegistry, skipTlsVerifyRegistry bool) (docker_registry.Interface, error) {
regOpts := docker_registry.DockerRegistryOptions{
InsecureRegistry: insecureRegistry,
SkipTlsVerifyRegistry: skipTlsVerifyRegistry,
}

dockerRegistry, err := docker_registry.NewDockerRegistry(addr, "", regOpts)
if err != nil {
return nil, fmt.Errorf("error creating container registry accessor for repo %q: %w", addr, err)
}

return dockerRegistry, nil
}

func (repoData *RepoData) CreateDockerRegistry(ctx context.Context, insecureRegistry, skipTlsVerifyRegistry bool) (docker_registry.Interface, error) {
addr, err := repoData.GetAddress()
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Expand Up @@ -314,6 +314,6 @@ replace k8s.io/helm => github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f

replace github.com/deislabs/oras => github.com/werf/third-party-oras v0.9.1-0.20210927171747-6d045506f4c8

replace helm.sh/helm/v3 => github.com/werf/3p-helm/v3 v3.0.0-20220823144404-27eb1367786c
replace helm.sh/helm/v3 => github.com/werf/3p-helm/v3 v3.0.0-20220902145201-6265178e3c32

replace github.com/go-git/go-git/v5 => github.com/ZauberNerd/go-git/v5 v5.4.3-0.20220315170230-29ec1bc1e5db
4 changes: 2 additions & 2 deletions go.sum
Expand Up @@ -2050,8 +2050,8 @@ github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59b
github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg2K1HrLU=
github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/werf/3p-helm/v3 v3.0.0-20220823144404-27eb1367786c h1:/cMYGHrCXCohS+2Thjj+7fiVVG1a/LmGC4BxZOK1sYY=
github.com/werf/3p-helm/v3 v3.0.0-20220823144404-27eb1367786c/go.mod h1:NxtE2KObf2PrzDl6SIamPFPKyAqWi10iWuvKlQn/Yao=
github.com/werf/3p-helm/v3 v3.0.0-20220902145201-6265178e3c32 h1:Z3XL0banUFF7IkIzY82onwWA2qiTvuQ0pBMERA/scis=
github.com/werf/3p-helm/v3 v3.0.0-20220902145201-6265178e3c32/go.mod h1:NxtE2KObf2PrzDl6SIamPFPKyAqWi10iWuvKlQn/Yao=
github.com/werf/copy-recurse v0.2.4 h1:kEyGUKhgS8WdEOjInNQKgk4lqPWzP2AgR27F3dcGsVc=
github.com/werf/copy-recurse v0.2.4/go.mod h1:KVHSQ90p19xflWW0B7BJhLBwmSbEtuxIaBnjlUYRPhk=
github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f h1:81YscYTF9mmTf0ULOsCmm42YWQp+qWDzWi1HjWniZrg=
Expand Down
75 changes: 75 additions & 0 deletions pkg/deploy/bundles/addr.go
@@ -0,0 +1,75 @@
package bundles

import (
"fmt"
"strings"

"github.com/werf/werf/pkg/deploy/bundles/registry"
)

const (
ArchiveSchema = "archive:"
RegistrySchema = "docker://"
)

type Addr struct {
*ArchiveAddress
*RegistryAddress
}

func (addr *Addr) String() string {
if addr.RegistryAddress != nil {
return addr.RegistryAddress.FullName()
}
if addr.ArchiveAddress != nil {
return addr.ArchiveAddress.Path
}
return ""
}

type ArchiveAddress struct {
Path string
}

type RegistryAddress struct {
*registry.Reference
}

func ParseAddr(addr string) (*Addr, error) {
switch {
case strings.HasPrefix(addr, ArchiveSchema):
return &Addr{ArchiveAddress: parseArchiveAddress(addr)}, nil
case strings.HasPrefix(addr, RegistrySchema):
if regAddr, err := parseRegistryAddress(addr); err != nil {
return nil, fmt.Errorf("unable to parse registry address %q: %w", addr, err)
} else {
return &Addr{RegistryAddress: regAddr}, nil
}
default:
if regAddr, err := parseRegistryAddress(addr); err != nil {
return nil, fmt.Errorf("unable to parse registry address %q: %w", addr, err)
} else {
return &Addr{RegistryAddress: regAddr}, nil
}
}
}

func parseRegistryAddress(addr string) (*RegistryAddress, error) {
cleanAddr := strings.TrimPrefix(addr, RegistrySchema)

ref, err := registry.ParseReference(cleanAddr)
if err != nil {
return nil, err
}

if ref.Tag == "" {
ref.Tag = "latest"
}

return &RegistryAddress{Reference: ref}, nil
}

func parseArchiveAddress(addr string) *ArchiveAddress {
path := strings.TrimPrefix(addr, ArchiveSchema)
return &ArchiveAddress{Path: path}
}
44 changes: 44 additions & 0 deletions pkg/deploy/bundles/addr_test.go
@@ -0,0 +1,44 @@
package bundles

import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

var _ = Describe("Bundle addr", func() {
It("should parse different bundle address schemas", func() {
{
addr, err := ParseAddr("registry.werf.io/group/image:mytag")
Expect(err).NotTo(HaveOccurred())
Expect(addr.RegistryAddress).NotTo(BeNil())
Expect(addr.ArchiveAddress).To(BeNil())
Expect(addr.Repo).To(Equal("registry.werf.io/group/image"))
Expect(addr.Tag).To(Equal("mytag"))
}

{
addr, err := ParseAddr("registry.werf.io/group/image")
Expect(err).NotTo(HaveOccurred())
Expect(addr.RegistryAddress).NotTo(BeNil())
Expect(addr.ArchiveAddress).To(BeNil())
Expect(addr.Repo).To(Equal("registry.werf.io/group/image"))
Expect(addr.Tag).To(Equal("latest"))
}

{
addr, err := ParseAddr("archive:path/to/file.tar.gz")
Expect(err).NotTo(HaveOccurred())
Expect(addr.RegistryAddress).To(BeNil())
Expect(addr.ArchiveAddress).NotTo(BeNil())
Expect(addr.Path).To(Equal("path/to/file.tar.gz"))
}

{
addr, err := ParseAddr("archive:/absolute/path/to/file.tar.gz")
Expect(err).NotTo(HaveOccurred())
Expect(addr.RegistryAddress).To(BeNil())
Expect(addr.ArchiveAddress).NotTo(BeNil())
Expect(addr.Path).To(Equal("/absolute/path/to/file.tar.gz"))
}
})
})

0 comments on commit 345cdf0

Please sign in to comment.