diff --git a/go.mod b/go.mod index d874013411..b082baeb4e 100644 --- a/go.mod +++ b/go.mod @@ -67,6 +67,7 @@ require ( go.opentelemetry.io/otel/sdk v1.7.0 go.opentelemetry.io/otel/trace v1.7.0 golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 + golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 golang.org/x/net v0.0.0-20220728211354-c7608f3a8462 gopkg.in/errgo.v2 v2.1.0 gopkg.in/ini.v1 v1.66.2 diff --git a/go.sum b/go.sum index 7ed5e5252f..23078186dd 100644 --- a/go.sum +++ b/go.sum @@ -2299,6 +2299,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= diff --git a/pkg/build/image/build_context_archive.go b/pkg/build/image/build_context_archive.go index b38b9bbb60..df553908e2 100644 --- a/pkg/build/image/build_context_archive.go +++ b/pkg/build/image/build_context_archive.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "github.com/werf/logboek" "github.com/werf/werf/pkg/container_backend" @@ -69,6 +70,10 @@ func (a *BuildContextArchive) Path() string { } func (a *BuildContextArchive) ExtractOrGetExtractedDir(ctx context.Context) (string, error) { + if a.path == "" { + panic("extract should not be called before create") + } + if a.extractionDir != "" { return a.extractionDir, nil } @@ -107,12 +112,25 @@ func (a *BuildContextArchive) CleanupExtractedDir(ctx context.Context) { } func (a *BuildContextArchive) CalculatePathsChecksum(ctx context.Context, paths []string) (string, error) { + sort.Strings(paths) + paths = util.UniqStrings(paths) + dir, err := a.ExtractOrGetExtractedDir(ctx) if err != nil { return "", fmt.Errorf("unable to access context directory: %w", err) } - _ = dir + var pathsHashes []string + for _, path := range paths { + p := filepath.Join(dir, path) + + hash, err := util.HashContentsAndPathsRecurse(p) + if err != nil { + return "", fmt.Errorf("unable to calculate hash: %w", err) + } + + pathsHashes = append(pathsHashes, hash) + } - return "", nil + return util.Sha256Hash(pathsHashes...), nil } diff --git a/pkg/build/stage/instruction/add.go b/pkg/build/stage/instruction/add.go index 56b3b56463..82ce8c266e 100644 --- a/pkg/build/stage/instruction/add.go +++ b/pkg/build/stage/instruction/add.go @@ -3,6 +3,9 @@ package instruction import ( "context" "fmt" + "strings" + + "github.com/containers/buildah/copier" "github.com/werf/werf/pkg/build/stage" "github.com/werf/werf/pkg/config" @@ -34,11 +37,20 @@ func (stg *Add) GetDependencies(ctx context.Context, c stage.Conveyor, cb contai args = append(args, "Chown", stg.instruction.Data.Chown) args = append(args, "Chmod", stg.instruction.Data.Chmod) - pathsChecksum, err := buildContextArchive.CalculatePathsChecksum(ctx, stg.instruction.Data.Src) - if err != nil { - return "", fmt.Errorf("unable to calculate build context paths checksum: %w", err) + var fileGlobSrc []string + for _, src := range stg.instruction.Data.Src { + if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { + fileGlobSrc = append(fileGlobSrc, src) + } + } + + if len(fileGlobSrc) > 0 { + if srcChecksum, err := calculateBuildContextGlobsChecksum(ctx, fileGlobSrc, true, buildContextArchive); err != nil { + return "", fmt.Errorf("unable to calculate build context globs checksum: %w", err) + } else { + args = append(args, "SrcChecksum", srcChecksum) + } } - args = append(args, "SrcChecksum", pathsChecksum) // TODO(staged-dockerfile): support http src and --checksum option: https://docs.docker.com/engine/reference/builder/#verifying-a-remote-file-checksum-add---checksumchecksum-http-src-dest // TODO(staged-dockerfile): support git ref: https://docs.docker.com/engine/reference/builder/#adding-a-git-repository-add-git-ref-dir @@ -47,3 +59,36 @@ func (stg *Add) GetDependencies(ctx context.Context, c stage.Conveyor, cb contai return util.Sha256Hash(args...), nil } + +func calculateBuildContextGlobsChecksum(ctx context.Context, fileGlobs []string, checkForArchives bool, buildContextArchive container_backend.BuildContextArchiver) (string, error) { + contextDir, err := buildContextArchive.ExtractOrGetExtractedDir(ctx) + if err != nil { + return "", fmt.Errorf("unable to get build context dir: %w", err) + } + + globStats, err := copier.Stat(contextDir, contextDir, copier.StatOptions{CheckForArchives: checkForArchives}, fileGlobs) + if err != nil { + return "", fmt.Errorf("unable to stat globs: %w", err) + } + if len(globStats) == 0 { + return "", fmt.Errorf("no glob matches for globs: %v", fileGlobs) + } + + var matches []string + for _, globStat := range globStats { + if globStat.Error != "" { + return "", fmt.Errorf("unable to stat glob %q: %w", globStat.Glob, globStat.Error) + } + + for _, match := range globStat.Globbed { + matches = append(matches, match) + } + } + + pathsChecksum, err := buildContextArchive.CalculatePathsChecksum(ctx, matches) + if err != nil { + return "", fmt.Errorf("unable to calculate build context paths checksum: %w", err) + } + + return pathsChecksum, nil +} diff --git a/pkg/build/stage/instruction/copy.go b/pkg/build/stage/instruction/copy.go index ddccb11f51..8965d6b763 100644 --- a/pkg/build/stage/instruction/copy.go +++ b/pkg/build/stage/instruction/copy.go @@ -2,6 +2,7 @@ package instruction import ( "context" + "fmt" "github.com/werf/werf/pkg/build/stage" "github.com/werf/werf/pkg/config" @@ -45,6 +46,16 @@ func (stg *Copy) GetDependencies(ctx context.Context, c stage.Conveyor, cb conta args = append(args, "Chmod", stg.instruction.Data.Chmod) args = append(args, "ExpandedFrom", stg.backendInstruction.From) + if stg.UsesBuildContext() { + if srcChecksum, err := calculateBuildContextGlobsChecksum(ctx, stg.instruction.Data.Src, false, buildContextArchive); err != nil { + return "", fmt.Errorf("unable to calculate build context globs checksum: %w", err) + } else { + args = append(args, "SrcChecksum", srcChecksum) + } + } + + // TODO(ilya-lesikov): should checksum of files from other image be calculated if --from specified? + // TODO(staged-dockerfile): support --link option: https://docs.docker.com/engine/reference/builder/#copy---link return util.Sha256Hash(args...), nil diff --git a/pkg/build/stage/instruction/run.go b/pkg/build/stage/instruction/run.go index 7847c15736..aee0c554a4 100644 --- a/pkg/build/stage/instruction/run.go +++ b/pkg/build/stage/instruction/run.go @@ -28,5 +28,8 @@ func (stg *Run) GetDependencies(ctx context.Context, c stage.Conveyor, cb contai args = append(args, "Instruction", stg.instruction.Data.Name()) args = append(args, append([]string{"Command"}, stg.instruction.Data.Command...)...) + + // TODO(ilya-lesikov): should bind mount with context as src be counted as dependency? + return util.Sha256Hash(args...), nil } diff --git a/pkg/buildah/common.go b/pkg/buildah/common.go index 97e9bc9b13..cac1f1a4c6 100644 --- a/pkg/buildah/common.go +++ b/pkg/buildah/common.go @@ -110,8 +110,9 @@ type ConfigOpts struct { type CopyOpts struct { CommonOpts - Chown string - Chmod string + Chown string + Chmod string + Ignores []string } type AddOpts struct { @@ -120,6 +121,7 @@ type AddOpts struct { ContextDir string Chown string Chmod string + Ignores []string } type ( diff --git a/pkg/buildah/native_linux.go b/pkg/buildah/native_linux.go index b64f7c2713..93377b040f 100644 --- a/pkg/buildah/native_linux.go +++ b/pkg/buildah/native_linux.go @@ -574,10 +574,7 @@ func (b *NativeBuildah) Copy(ctx context.Context, container, contextDir string, Chmod: opts.Chmod, PreserveOwnership: false, ContextDir: contextDir, - // TODO(ilya-lesikov): ignore file? - Excludes: nil, - // TODO(ilya-lesikov): ignore file? - IgnoreFile: "", + Excludes: opts.Ignores, }, absSrc...); err != nil { return fmt.Errorf("error copying files to %q: %w", dst, err) } @@ -610,10 +607,7 @@ func (b *NativeBuildah) Add(ctx context.Context, container string, src []string, Chown: opts.Chown, PreserveOwnership: false, ContextDir: opts.ContextDir, - // TODO(ilya-lesikov): ignore file? - Excludes: nil, - // TODO(ilya-lesikov): ignore file? - IgnoreFile: "", + Excludes: opts.Ignores, }, expandedSrc...); err != nil { return fmt.Errorf("error adding files to %q: %w", dst, err) } diff --git a/pkg/dockerfile/frontend/buildkit_dockerfile.go b/pkg/dockerfile/frontend/buildkit_dockerfile.go index d1bb973cbf..d419c60ff7 100644 --- a/pkg/dockerfile/frontend/buildkit_dockerfile.go +++ b/pkg/dockerfile/frontend/buildkit_dockerfile.go @@ -108,11 +108,17 @@ func extractSrcAndDst(sourcesAndDest instructions.SourcesAndDest) ([]string, str // /home/user1/go/pkg/mod/github.com/moby/buildkit@v0.8.2/frontend/dockerfile/parser/parser.go:250 var src []string for _, s := range sourcesAndDest[0 : len(sourcesAndDest)-1] { - s, _ = strconv.Unquote(s) + if unquoted, err := strconv.Unquote(s); err == nil { + s = unquoted + } + src = append(src, s) } - dst, _ := strconv.Unquote(sourcesAndDest[len(sourcesAndDest)-1]) + dst := sourcesAndDest[len(sourcesAndDest)-1] + if unquoted, err := strconv.Unquote(dst); err == nil { + dst = unquoted + } return src, dst } diff --git a/pkg/util/hashsum.go b/pkg/util/hashsum.go index d4599f0bde..4e453fc13d 100644 --- a/pkg/util/hashsum.go +++ b/pkg/util/hashsum.go @@ -3,10 +3,14 @@ package util import ( "crypto/sha256" "fmt" + "io" + "os" + "path/filepath" "strings" "github.com/spaolacci/murmur3" "golang.org/x/crypto/sha3" + "golang.org/x/mod/sumdb/dirhash" ) // LegacyMurmurHash function returns a hash of non-fixed length (1-8 symbols) @@ -31,6 +35,32 @@ func Sha256Hash(args ...string) string { return fmt.Sprintf("%x", sum) } +// For file: hash contents of file with its name. +// For directory: hash contents of all files in directory, along with their relative filenames. +func HashContentsAndPathsRecurse(path string) (string, error) { + path = filepath.Clean(path) + + fi, err := os.Stat(path) + if err != nil { + return "", fmt.Errorf("unable to stat %q: %w", path, err) + } + + var hash string + if fi.IsDir() { + if hash, err = dirhash.HashDir(path, "/", dirhash.Hash1); err != nil { + return "", fmt.Errorf("unable to calculate hash for dir %q: %w", path, err) + } + } else { + if hash, err = dirhash.Hash1([]string{filepath.Base(path)}, func(_ string) (io.ReadCloser, error) { + return os.Open(path) + }); err != nil { + return "", fmt.Errorf("unable to calculate hash for file %q: %w", path, err) + } + } + + return hash, nil +} + func prepareHashArgs(args ...string) string { return strings.Join(args, ":::") }