/
chart_dependencies_loader.go
319 lines (264 loc) · 10.7 KB
/
chart_dependencies_loader.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
package chart_extender
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os"
"path/filepath"
"strings"
"github.com/pkg/errors"
uuid "github.com/satori/go.uuid"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/chart/loader"
"helm.sh/helm/v3/pkg/cli"
"helm.sh/helm/v3/pkg/provenance"
"helm.sh/helm/v3/pkg/registry"
"sigs.k8s.io/yaml"
"github.com/werf/lockgate"
"github.com/werf/logboek"
"github.com/werf/werf/pkg/deploy/helm/command_helpers"
"github.com/werf/werf/pkg/util"
"github.com/werf/werf/pkg/werf"
)
func GetChartDependenciesCacheDir(lockChecksum string) string {
return filepath.Join(werf.GetLocalCacheDir(), "helm_chart_dependencies", "1", lockChecksum)
}
func LoadMetadata(files []*chart.ChartExtenderBufferedFile) (*chart.Metadata, error) {
var metadata *chart.Metadata
for _, f := range files {
if f.Name == "Chart.yaml" {
metadata = new(chart.Metadata)
if err := yaml.Unmarshal(f.Data, metadata); err != nil {
return nil, errors.Wrap(err, "cannot load Chart.yaml")
}
if metadata.APIVersion == "" {
metadata.APIVersion = chart.APIVersionV1
}
break
}
}
for _, f := range files {
if f.Name == "requirements.yaml" {
if err := yaml.Unmarshal(f.Data, metadata); err != nil {
return nil, errors.Wrap(err, "cannot load requirements.yaml")
}
break
}
}
return metadata, nil
}
func GetPreparedChartDependenciesDir(ctx context.Context, metadataFile, metadataLockFile *chart.ChartExtenderBufferedFile, helmEnvSettings *cli.EnvSettings, registryClient *registry.Client, buildChartDependenciesOpts command_helpers.BuildChartDependenciesOptions) (string, error) {
depsDir := GetChartDependenciesCacheDir(util.Sha256Hash(string(metadataLockFile.Data)))
_, err := os.Stat(depsDir)
switch {
case os.IsNotExist(err):
if err := logboek.Context(ctx).Default().LogProcess("Building chart dependencies").DoError(func() error {
logboek.Context(ctx).Default().LogF("Using chart dependencies directory: %s\n", depsDir)
_, lock, err := werf.AcquireHostLock(ctx, depsDir, lockgate.AcquireOptions{})
if err != nil {
return fmt.Errorf("error acquiring lock for %q: %w", depsDir, err)
}
defer werf.ReleaseHostLock(lock)
tmpDepsDir := fmt.Sprintf("%s.tmp.%s", depsDir, uuid.NewV4().String())
buildChartDependenciesOpts.LoadOptions = &loader.LoadOptions{
ChartExtender: NewWerfChartStub(ctx, buildChartDependenciesOpts.IgnoreInvalidAnnotationsAndLabels),
SubchartExtenderFactoryFunc: nil,
}
if err := command_helpers.BuildChartDependenciesInDir(ctx, metadataFile, metadataLockFile, tmpDepsDir, helmEnvSettings, registryClient, buildChartDependenciesOpts); err != nil {
return fmt.Errorf("error building chart dependencies: %w", err)
}
if err := os.Rename(tmpDepsDir, depsDir); err != nil {
return fmt.Errorf("error renaming %q to %q: %w", tmpDepsDir, depsDir, err)
}
return nil
}); err != nil {
return "", err
}
case err != nil:
return "", fmt.Errorf("error accessing %q: %w", depsDir, err)
default:
logboek.Context(ctx).Default().LogF("Using cached chart dependencies directory: %s\n", depsDir)
}
return depsDir, nil
}
type ChartDependenciesConfiguration struct {
ChartMetadata *chart.Metadata
ChartMetadataLock *chart.Lock
}
func NewChartDependenciesConfiguration(chartMetadata *chart.Metadata, chartMetadataLock *chart.Lock) *ChartDependenciesConfiguration {
return &ChartDependenciesConfiguration{ChartMetadata: chartMetadata, ChartMetadataLock: chartMetadataLock}
}
func (conf *ChartDependenciesConfiguration) GetExternalDependenciesFiles() (bool, *chart.ChartExtenderBufferedFile, *chart.ChartExtenderBufferedFile, error) {
metadataBytes, err := yaml.Marshal(conf.ChartMetadata)
if err != nil {
return false, nil, nil, fmt.Errorf("unable to marshal original chart metadata into yaml: %w", err)
}
metadata := new(chart.Metadata)
if err := yaml.Unmarshal(metadataBytes, metadata); err != nil {
return false, nil, nil, fmt.Errorf("unable to unmarshal original chart metadata yaml: %w", err)
}
metadataLockBytes, err := yaml.Marshal(conf.ChartMetadataLock)
if err != nil {
return false, nil, nil, fmt.Errorf("unable to marshal original chart metadata lock into yaml: %w", err)
}
metadataLock := new(chart.Lock)
if err := yaml.Unmarshal(metadataLockBytes, metadataLock); err != nil {
return false, nil, nil, fmt.Errorf("unable to unmarshal original chart metadata lock yaml: %w", err)
}
metadata.APIVersion = "v2"
var localDependenciesNames []string
var filteredLockDependencies []*chart.Dependency
for _, depLock := range metadataLock.Dependencies {
if depLock.Repository == "" || strings.HasPrefix(depLock.Repository, "file://") {
localDependenciesNames = append(localDependenciesNames, depLock.Name)
continue
}
filteredLockDependencies = append(filteredLockDependencies, depLock)
}
metadataLock.Dependencies = filteredLockDependencies
var filteredDependencies []*chart.Dependency
FilterOutLocalDependencies:
for _, dep := range metadata.Dependencies {
for _, localDepName := range localDependenciesNames {
if localDepName == dep.Name {
continue FilterOutLocalDependencies
}
}
filteredDependencies = append(filteredDependencies, dep)
}
metadata.Dependencies = filteredDependencies
if len(metadata.Dependencies) == 0 {
return false, nil, nil, nil
}
// Set resolved repository from the lock file
for _, dep := range metadata.Dependencies {
for _, depLock := range metadataLock.Dependencies {
if dep.Name == depLock.Name {
dep.Repository = depLock.Repository
break
}
}
}
if newDigest, err := HashReq(metadata.Dependencies, metadataLock.Dependencies); err != nil {
return false, nil, nil, fmt.Errorf("unable to calculate external dependencies Chart.yaml digest: %w", err)
} else {
metadataLock.Digest = newDigest
}
metadataFile := &chart.ChartExtenderBufferedFile{Name: "Chart.yaml"}
if data, err := yaml.Marshal(metadata); err != nil {
return false, nil, nil, fmt.Errorf("unable to marshal chart metadata file with external dependencies: %w", err)
} else {
metadataFile.Data = data
}
metadataLockFile := &chart.ChartExtenderBufferedFile{Name: "Chart.lock"}
if data, err := yaml.Marshal(metadataLock); err != nil {
return false, nil, nil, fmt.Errorf("unable to marshal chart metadata lock file with external dependencies: %w", err)
} else {
metadataLockFile.Data = data
}
return true, metadataFile, metadataLockFile, nil
}
func LoadChartDependencies(ctx context.Context, loadChartDirFunc func(ctx context.Context, dir string) ([]*chart.ChartExtenderBufferedFile, error), chartDir string, loadedChartFiles []*chart.ChartExtenderBufferedFile, helmEnvSettings *cli.EnvSettings, registryClient *registry.Client, buildChartDependenciesOpts command_helpers.BuildChartDependenciesOptions) ([]*chart.ChartExtenderBufferedFile, error) {
res := loadedChartFiles
var chartMetadata *chart.Metadata
var chartMetadataLock *chart.Lock
for _, f := range loadedChartFiles {
switch f.Name {
case "Chart.yaml":
chartMetadata = new(chart.Metadata)
if err := yaml.Unmarshal(f.Data, chartMetadata); err != nil {
return nil, errors.Wrap(err, "cannot load Chart.yaml")
}
if chartMetadata.APIVersion == "" {
chartMetadata.APIVersion = chart.APIVersionV1
}
case "Chart.lock":
chartMetadataLock = new(chart.Lock)
if err := yaml.Unmarshal(f.Data, chartMetadataLock); err != nil {
return nil, errors.Wrap(err, "cannot load Chart.lock")
}
}
}
for _, f := range loadedChartFiles {
switch f.Name {
case "requirements.yaml":
if chartMetadata == nil {
chartMetadata = new(chart.Metadata)
}
if err := yaml.Unmarshal(f.Data, chartMetadata); err != nil {
return nil, errors.Wrap(err, "cannot load requirements.yaml")
}
case "requirements.lock":
if chartMetadataLock == nil {
chartMetadataLock = new(chart.Lock)
}
if err := yaml.Unmarshal(f.Data, chartMetadataLock); err != nil {
return nil, errors.Wrap(err, "cannot load requirements.lock")
}
}
}
if chartMetadata == nil {
return res, nil
}
if chartMetadataLock == nil {
if len(chartMetadata.Dependencies) > 0 {
logboek.Context(ctx).Error().LogLn("Cannot build chart dependencies and preload charts without lock file (.helm/Chart.lock or .helm/requirements.lock)")
logboek.Context(ctx).Error().LogLn("It is recommended to add Chart.lock file to your project repository or remove chart dependencies.")
logboek.Context(ctx).Error().LogLn()
logboek.Context(ctx).Error().LogLn("To generate a lock file run 'werf helm dependency update .helm' and commit resulting .helm/Chart.lock or .helm/requirements.lock (it is not required to commit whole .helm/charts directory, better add it to the .gitignore).")
logboek.Context(ctx).Error().LogLn()
}
return res, nil
}
conf := NewChartDependenciesConfiguration(chartMetadata, chartMetadataLock)
// Append virtually loaded files from custom dependency repositories in the local filesystem,
// pretending these files are located in the charts/ dir as designed in the Helm.
for _, chartDep := range chartMetadataLock.Dependencies {
if !strings.HasPrefix(chartDep.Repository, "file://") {
continue
}
relativeLocalChartPath := strings.TrimPrefix(chartDep.Repository, "file://")
localChartPath := filepath.Join(chartDir, relativeLocalChartPath)
chartFiles, err := loadChartDirFunc(ctx, localChartPath)
if err != nil {
return nil, fmt.Errorf("unable to load custom subchart dir %q: %w", localChartPath, err)
}
for _, f := range chartFiles {
f.Name = filepath.Join("charts", chartDep.Name, f.Name)
}
res = append(res, chartFiles...)
}
haveExternalDependencies, metadataFile, metadataLockFile, err := conf.GetExternalDependenciesFiles()
if err != nil {
return nil, fmt.Errorf("unable to get external dependencies chart configuration files: %w", err)
}
if !haveExternalDependencies {
return res, nil
}
depsDir, err := GetPreparedChartDependenciesDir(ctx, metadataFile, metadataLockFile, helmEnvSettings, registryClient, buildChartDependenciesOpts)
if err != nil {
return nil, fmt.Errorf("error preparing chart dependencies: %w", err)
}
localFiles, err := loader.GetFilesFromLocalFilesystem(depsDir)
if err != nil {
return nil, err
}
for _, f := range localFiles {
if strings.HasPrefix(f.Name, "charts/") {
f1 := new(chart.ChartExtenderBufferedFile)
*f1 = chart.ChartExtenderBufferedFile(*f)
res = append(res, f1)
logboek.Context(ctx).Debug().LogF("-- LoadChartDependencies: loading subchart %q from the dependencies dir %q\n", f.Name, depsDir)
}
}
return res, nil
}
func HashReq(req, lock []*chart.Dependency) (string, error) {
data, err := json.Marshal([2][]*chart.Dependency{req, lock})
if err != nil {
return "", err
}
s, err := provenance.Digest(bytes.NewBuffer(data))
return "sha256:" + s, err
}