Skip to content

Commit

Permalink
fix(bundles): refactor bundle copy implementation
Browse files Browse the repository at this point in the history
Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Sep 6, 2022
1 parent ae8c259 commit 63d13d0
Show file tree
Hide file tree
Showing 6 changed files with 371 additions and 318 deletions.
4 changes: 2 additions & 2 deletions cmd/werf/bundle/copy/copy.go
Expand Up @@ -153,8 +153,8 @@ func runCopy(ctx context.Context) error {

return bundles.Copy(ctx, fromAddr, toAddr, bundles.CopyOptions{
BundlesRegistryClient: bundlesRegistryClient,
FromRegistry: fromRegistry,
ToRegistry: toRegistry,
FromRegistryClient: fromRegistry,
ToRegistryClient: toRegistry,
})
})
}
35 changes: 35 additions & 0 deletions pkg/deploy/bundles/bundle_accessor.go
@@ -0,0 +1,35 @@
package bundles

import (
"context"
"fmt"

"helm.sh/helm/v3/pkg/chart"

"github.com/werf/werf/pkg/docker_registry"
)

type BundleAccessor interface {
ReadChart(ctx context.Context) (*chart.Chart, error)
WriteChart(ctx context.Context, ch *chart.Chart) error

CopyTo(ctx context.Context, to BundleAccessor) error
CopyFromArchive(ctx context.Context, fromArchive *BundleArchive) error
CopyFromRemote(ctx context.Context, fromRemote *RemoteBundle) error
}

type BundleAccessorOptions struct {
BundlesRegistryClient BundlesRegistryClient
RegistryClient docker_registry.Interface
}

func NewBundleAccessor(addr *Addr, opts BundleAccessorOptions) BundleAccessor {
switch {
case addr.RegistryAddress != nil:
return NewRemoteBundle(addr.RegistryAddress, opts.BundlesRegistryClient, opts.RegistryClient)
case addr.ArchiveAddress != nil:
return NewBundleArchive(addr.ArchiveAddress.Path)
default:
panic(fmt.Sprintf("invalid address given %#v", addr))
}
}
113 changes: 98 additions & 15 deletions pkg/deploy/bundles/bundle.go → pkg/deploy/bundles/bundle_archive.go
Expand Up @@ -4,45 +4,37 @@ import (
"archive/tar"
"bytes"
"compress/gzip"
"context"
"fmt"
"io"
"os"
"time"

"github.com/google/uuid"
"github.com/otiai10/copy"
"helm.sh/helm/v3/pkg/chart"

bundles_registry "github.com/werf/werf/pkg/deploy/bundles/registry"
"github.com/werf/logboek"
"github.com/werf/werf/pkg/image"
)

const (
chartArchiveFileName = "chart.tar.gz"
)

type Bundle interface{}

type BundleArchiveWriter interface {
type BundleWriter interface {
Open() error
WriteChartArchive(data []byte) error
WriteImageArchive(imageTag string, data []byte) error
Save() error
}

type BundleArchiveReader interface {
type BundleReader interface {
ReadChartArchive() ([]byte, error)
GetImageArchiveOpener(imageTag string) *ImageArchiveOpener
ReadImageArchive(imageTag string) (*ImageArchiveReadCloser, error)
}

type RemoteBundle struct {
RegistryClient *bundles_registry.Client
}

func NewRemoteBundle(registryClient *bundles_registry.Client) *RemoteBundle {
return &RemoteBundle{
RegistryClient: registryClient,
}
}

type BundleArchive struct {
Path string

Expand Down Expand Up @@ -265,6 +257,97 @@ func (bundle *BundleArchive) ReadImageArchive(imageTag string) (*ImageArchiveRea
}
}

func (bundle *BundleArchive) ReadChart(ctx context.Context) (*chart.Chart, error) {
chartBytes, err := bundle.ReadChartArchive()
if err != nil {
return nil, fmt.Errorf("unable to read chart archive: %w", err)
}

ch, err := BytesToChart(chartBytes)
if err != nil {
return nil, fmt.Errorf("unable to parse chart archive from bundle archive: %w", err)
}

return ch, nil
}

func (bundle *BundleArchive) WriteChart(ctx context.Context, ch *chart.Chart) error {
chartBytes, err := ChartToBytes(ch)
if err != nil {
return fmt.Errorf("unable to dump chart to archive: %w", err)
}

if err := bundle.WriteChartArchive(chartBytes); err != nil {
return fmt.Errorf("unable to write chart archive into bundle archive: %w", err)
}

return nil
}

func (bundle *BundleArchive) CopyTo(ctx context.Context, to BundleAccessor) error {
return to.CopyFromArchive(ctx, bundle)
}

func (bundle *BundleArchive) CopyFromArchive(ctx context.Context, fromArchive *BundleArchive) error {
if err := copy.Copy(fromArchive.Path, bundle.Path); err != nil {
return fmt.Errorf("unable to copy file %q to %q: %w", fromArchive.Path, bundle.Path, err)
}
return nil
}

func (bundle *BundleArchive) CopyFromRemote(ctx context.Context, fromRemote *RemoteBundle) error {
ch, err := fromRemote.ReadChart(ctx)
if err != nil {
return fmt.Errorf("unable to read chart from remote bundle: %w", err)
}

if err := bundle.Open(); err != nil {
return fmt.Errorf("unable to open target bundle archive: %w", err)
}

if err := logboek.Context(ctx).LogProcess("Saving bundle %s into archive", fromRemote.RegistryAddress.FullName()).DoError(func() error {
return bundle.WriteChart(ctx, ch)
}); err != nil {
return err
}

if werfVals, ok := ch.Values["werf"].(map[string]interface{}); ok {
if imageVals, ok := werfVals["image"].(map[string]interface{}); ok {
for imageName, v := range imageVals {
if imageRef, ok := v.(string); ok {
logboek.Context(ctx).Default().LogFDetails("Saving image %s\n", imageRef)

_, tag := image.ParseRepositoryAndTag(imageRef)

// TODO: maybe save into tmp file archive OR read resulting image size from the registry before pulling
imageBytes := bytes.NewBuffer(nil)
zipper := gzip.NewWriter(imageBytes)

if err := fromRemote.RegistryClient.PullImageArchive(ctx, zipper, imageRef); err != nil {
return fmt.Errorf("error pulling image %q archive: %w", imageRef, err)
}

if err := zipper.Close(); err != nil {
return fmt.Errorf("unable to close gzip writer: %w", err)
}

if err := bundle.WriteImageArchive(tag, imageBytes.Bytes()); err != nil {
return fmt.Errorf("error writing image %q into bundle archive: %w", imageRef, err)
}
} else {
return fmt.Errorf("unexpected value .Values.werf.image.%s=%v", imageName, v)
}
}
}
}

if err := bundle.Save(); err != nil {
return fmt.Errorf("error saving destination bundle archive: %w", err)
}

return nil
}

type ImageArchiveOpener struct {
Archive *BundleArchive
ImageTag string
Expand Down
14 changes: 14 additions & 0 deletions pkg/deploy/bundles/bundles_registry_client.go
@@ -0,0 +1,14 @@
package bundles

import (
"helm.sh/helm/v3/pkg/chart"

bundles_registry "github.com/werf/werf/pkg/deploy/bundles/registry"
)

type BundlesRegistryClient interface {
PullChartToCache(ref *bundles_registry.Reference) error
LoadChart(ref *bundles_registry.Reference) (*chart.Chart, error)
SaveChart(ch *chart.Chart, ref *bundles_registry.Reference) error
PushChart(ref *bundles_registry.Reference) error
}

0 comments on commit 63d13d0

Please sign in to comment.