Skip to content

Commit

Permalink
fix(helm-dependencies): automatically fill ~/.werf/local_cache on 'we…
Browse files Browse the repository at this point in the history 
…rf helm dependency update' command

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Feb 17, 2023
1 parent 3addc23 commit b094521
Show file tree
Hide file tree
Showing 3 changed files with 124 additions and 38 deletions.
27 changes: 26 additions & 1 deletion cmd/werf/helm/helm.go
View file
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"fmt"
"os"
"path/filepath"
"time"

"github.com/spf13/cobra"
Expand Down Expand Up @@ -76,11 +77,35 @@ func NewCmd(ctx context.Context) (*cobra.Command, error) {
common.SetupInsecureHelmDependencies(&_commonCmdData, cmd)
common.SetupDockerConfig(&_commonCmdData, cmd, "")

dependencyCmd := helm_v3.NewDependencyCmd(actionConfig, os.Stdout)
for _, depCmd := range dependencyCmd.Commands() {
if depCmd.Name() == "update" {
oldRunE := depCmd.RunE
depCmd.RunE = func(cmd *cobra.Command, args []string) error {
if err := oldRunE(cmd, args); err != nil {
return err
}

chartpath := "."
if len(args) > 0 {
chartpath = filepath.Clean(args[0])
}

ch, err := loader.LoadDir(chartpath)
if err != nil {
return fmt.Errorf("error loading chart %q: %w", chartpath, err)
}

return chart_extender.CopyChartDependenciesIntoCache(cmd.Context(), chartpath, ch)
}
}
}

cmd.AddCommand(
helm2.ReplaceHelmUninstallDocs(helm_v3.NewUninstallCmd(actionConfig, os.Stdout, helm_v3.UninstallCmdOptions{
StagesSplitter: helm.NewStagesSplitter(),
})),
helm2.ReplaceHelmDependencyDocs(helm_v3.NewDependencyCmd(actionConfig, os.Stdout)),
helm2.ReplaceHelmDependencyDocs(dependencyCmd),
helm2.ReplaceHelmGetDocs(helm_v3.NewGetCmd(actionConfig, os.Stdout)),
helm2.ReplaceHelmHistoryDocs(helm_v3.NewHistoryCmd(actionConfig, os.Stdout)),
NewLintCmd(actionConfig, wc),
Expand Down
112 changes: 97 additions & 15 deletions pkg/deploy/helm/chart_extender/chart_dependencies_loader.go
View file
Expand Up @@ -5,10 +5,12 @@ import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"strings"

"github.com/otiai10/copy"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"helm.sh/helm/v3/pkg/chart"
Expand All @@ -20,13 +22,14 @@ import (

"github.com/werf/lockgate"
"github.com/werf/logboek"
"github.com/werf/logboek/pkg/types"
"github.com/werf/werf/pkg/deploy/helm/command_helpers"
"github.com/werf/werf/pkg/util"
"github.com/werf/werf/pkg/werf"
)

func GetChartDependenciesCacheDir(lockChecksum string) string {
return filepath.Join(werf.GetLocalCacheDir(), "helm_chart_dependencies", "1", lockChecksum)
func GetChartDependenciesCacheDir() string {
return filepath.Join(werf.GetLocalCacheDir(), "helm_chart_dependencies", "1")
}

func LoadMetadata(files []*chart.ChartExtenderBufferedFile) (*chart.Metadata, error) {
Expand Down Expand Up @@ -60,14 +63,54 @@ func LoadMetadata(files []*chart.ChartExtenderBufferedFile) (*chart.Metadata, er
return metadata, nil
}

func GetPreparedChartDependenciesDir(ctx context.Context, metadataFile, metadataLockFile *chart.ChartExtenderBufferedFile, helmEnvSettings *cli.EnvSettings, registryClient *registry.Client, buildChartDependenciesOpts command_helpers.BuildChartDependenciesOptions) (string, error) {
depsDir := GetChartDependenciesCacheDir(util.Sha256Hash(string(metadataLockFile.Data)))
func CopyChartDependenciesIntoCache(ctx context.Context, chartDir string, ch *chart.Chart) error {
metadataBytes, err := yaml.Marshal(ch.Metadata)
if err != nil {
return fmt.Errorf("unable to marshal chart metadata into yaml: %w", err)
}

metadataLockBytes, err := yaml.Marshal(ch.Lock)
if err != nil {
return fmt.Errorf("unable to marshal chart metadata lock into yaml: %w", err)
}

_, err = prepareDependenciesDir(ctx, metadataBytes, metadataLockBytes, func(tmpDepsDir string) error {
var archivesDependencies []*chart.File

FindArchivesDependencies:
for _, f := range ch.Raw {
if strings.HasPrefix(f.Name, "charts/") {
for _, depLock := range ch.Lock.Dependencies {
if filepath.Base(f.Name) == MakeDependencyArchiveName(depLock.Name, depLock.Version) {
archivesDependencies = append(archivesDependencies, f)
continue FindArchivesDependencies
}
}
}
}

for _, depArch := range archivesDependencies {
srcPath := filepath.Join(chartDir, depArch.Name)
destPath := filepath.Join(tmpDepsDir, depArch.Name)
if err := copy.Copy(srcPath, destPath); err != nil {
return fmt.Errorf("unable to copy %q into cache %q: %w", srcPath, tmpDepsDir, err)
}
}

return nil
}, logboek.Context(ctx).Info())

return err
}

func prepareDependenciesDir(ctx context.Context, metadataBytes, metadataLockBytes []byte, prepareFunc func(tmpDepsDir string) error, logger types.ManagerInterface) (string, error) {
depsDir := filepath.Join(GetChartDependenciesCacheDir(), util.Sha256Hash(string(metadataLockBytes)))

_, err := os.Stat(depsDir)
switch {
case os.IsNotExist(err):
if err := logboek.Context(ctx).Default().LogProcess("Building chart dependencies").DoError(func() error {
logboek.Context(ctx).Default().LogF("Using chart dependencies directory: %s\n", depsDir)
if err := logger.LogProcess("Preparing chart dependencies").DoError(func() error {
logger.LogF("Using chart dependencies directory: %s\n", depsDir)
_, lock, err := werf.AcquireHostLock(ctx, depsDir, lockgate.AcquireOptions{})
if err != nil {
return fmt.Errorf("error acquiring lock for %q: %w", depsDir, err)
Expand All @@ -85,32 +128,67 @@ func GetPreparedChartDependenciesDir(ctx context.Context, metadataFile, metadata

tmpDepsDir := fmt.Sprintf("%s.tmp.%s", depsDir, uuid.NewV4().String())

buildChartDependenciesOpts.LoadOptions = &loader.LoadOptions{
ChartExtender: NewWerfChartStub(ctx, buildChartDependenciesOpts.IgnoreInvalidAnnotationsAndLabels),
SubchartExtenderFactoryFunc: nil,
if err := createChartDependenciesDir(tmpDepsDir, metadataBytes, metadataLockBytes); err != nil {
return err
}

if err := command_helpers.BuildChartDependenciesInDir(ctx, metadataFile, metadataLockFile, tmpDepsDir, helmEnvSettings, registryClient, buildChartDependenciesOpts); err != nil {
return fmt.Errorf("error building chart dependencies: %w", err)
if err := prepareFunc(tmpDepsDir); err != nil {
return err
}

if err := os.Rename(tmpDepsDir, depsDir); err != nil {
return fmt.Errorf("error renaming %q to %q: %w", tmpDepsDir, depsDir, err)
}

return nil
}); err != nil {
return "", err
}
case err != nil:
return "", fmt.Errorf("error accessing %q: %w", depsDir, err)
default:
logboek.Context(ctx).Default().LogF("Using cached chart dependencies directory: %s\n", depsDir)
logger.LogF("Using cached chart dependencies directory: %s\n", depsDir)
}

return depsDir, nil
}

func createChartDependenciesDir(destDir string, metadataBytes, metadataLockBytes []byte) error {
if err := os.MkdirAll(destDir, os.ModePerm); err != nil {
return fmt.Errorf("error creating dir %q: %w", destDir, err)
}

files := []*chart.ChartExtenderBufferedFile{
{Name: "Chart.yaml", Data: metadataBytes},
{Name: "Chart.lock", Data: metadataLockBytes},
}

for _, file := range files {
if file == nil {
continue
}

path := filepath.Join(destDir, file.Name)
if err := ioutil.WriteFile(path, file.Data, 0o644); err != nil {
return fmt.Errorf("error writing %q: %w", path, err)
}
}

return nil
}

func GetPreparedChartDependenciesDir(ctx context.Context, metadataFile, metadataLockFile *chart.ChartExtenderBufferedFile, helmEnvSettings *cli.EnvSettings, registryClient *registry.Client, buildChartDependenciesOpts command_helpers.BuildChartDependenciesOptions) (string, error) {
return prepareDependenciesDir(ctx, metadataFile.Data, metadataLockFile.Data, func(tmpDepsDir string) error {
buildChartDependenciesOpts.LoadOptions = &loader.LoadOptions{
ChartExtender: NewWerfChartStub(ctx, buildChartDependenciesOpts.IgnoreInvalidAnnotationsAndLabels),
SubchartExtenderFactoryFunc: nil,
}
if err := command_helpers.BuildChartDependenciesInDir(ctx, tmpDepsDir, helmEnvSettings, registryClient, buildChartDependenciesOpts); err != nil {
return fmt.Errorf("error building chart dependencies: %w", err)
}
return nil
}, logboek.Context(ctx).Default())
}

type ChartDependenciesConfiguration struct {
ChartMetadata *chart.Metadata
ChartMetadataLock *chart.Lock
Expand Down Expand Up @@ -142,7 +220,7 @@ func (conf *ChartDependenciesConfiguration) GetExternalDependenciesFiles(loadedC
metadata.APIVersion = "v2"

var externalDependenciesNames []string
var isExternalDependency = func(depName string) bool {
isExternalDependency := func(depName string) bool {
for _, externalDepName := range externalDependenciesNames {
if depName == externalDepName {
return true
Expand All @@ -159,7 +237,7 @@ FindExternalDependencies:

for _, loadedFile := range loadedChartFiles {
if strings.HasPrefix(loadedFile.Name, "charts/") {
if filepath.Base(loadedFile.Name) == fmt.Sprintf("%s-%s.tgz", depLock.Name, depLock.Version) {
if filepath.Base(loadedFile.Name) == MakeDependencyArchiveName(depLock.Name, depLock.Version) {
continue FindExternalDependencies
}
}
Expand Down Expand Up @@ -343,3 +421,7 @@ func HashReq(req, lock []*chart.Dependency) (string, error) {
s, err := provenance.Digest(bytes.NewBuffer(data))
return "sha256:" + s, err
}

func MakeDependencyArchiveName(depName, depVersion string) string {
return fmt.Sprintf("%s-%s.tgz", depName, depVersion)
}
23 changes: 1 addition & 22 deletions pkg/deploy/helm/command_helpers/build_chart_dependencies.go
View file
Expand Up @@ -3,11 +3,7 @@ package command_helpers
import (
"context"
"fmt"
"io/ioutil"
"os"
"path/filepath"

"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/downloader"
Expand All @@ -25,26 +21,9 @@ type BuildChartDependenciesOptions struct {
IgnoreInvalidAnnotationsAndLabels bool
}

func BuildChartDependenciesInDir(ctx context.Context, chartFile, chartLockFile *chart.ChartExtenderBufferedFile, targetDir string, helmEnvSettings *cli.EnvSettings, registryClient *registry.Client, opts BuildChartDependenciesOptions) error {
func BuildChartDependenciesInDir(ctx context.Context, targetDir string, helmEnvSettings *cli.EnvSettings, registryClient *registry.Client, opts BuildChartDependenciesOptions) error {
logboek.Context(ctx).Debug().LogF("-- BuildChartDependenciesInDir\n")

if err := os.MkdirAll(targetDir, os.ModePerm); err != nil {
return fmt.Errorf("error creating dir %q: %w", targetDir, err)
}

files := []*chart.ChartExtenderBufferedFile{chartFile, chartLockFile}

for _, file := range files {
if file == nil {
continue
}

path := filepath.Join(targetDir, file.Name)
if err := ioutil.WriteFile(path, file.Data, 0o644); err != nil {
return fmt.Errorf("error writing %q: %w", path, err)
}
}

man := &downloader.Manager{
Out: logboek.Context(ctx).OutStream(),
ChartPath: targetDir,
Expand Down

0 comments on commit b094521

Please sign in to comment.