feat(staged-dockerfile): use contents of Copy/Add sources in checksum

Signed-off-by: Ilya Lesikov <ilya@lesikov.com>

go.mod

@@ -67,6 +67,7 @@ require (
 	go.opentelemetry.io/otel/sdk v1.7.0
 	go.opentelemetry.io/otel/trace v1.7.0
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
+	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3
 	golang.org/x/net v0.0.0-20220728211354-c7608f3a8462
 	gopkg.in/errgo.v2 v2.1.0
 	gopkg.in/ini.v1 v1.66.2

go.sum

@@ -2299,6 +2299,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
+golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
 golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

pkg/build/image/build_context_archive.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"sort"
 
 	"github.com/werf/logboek"
 	"github.com/werf/werf/pkg/container_backend"
@@ -69,6 +70,10 @@ func (a *BuildContextArchive) Path() string {
 }
 
 func (a *BuildContextArchive) ExtractOrGetExtractedDir(ctx context.Context) (string, error) {
+	if a.path == "" {
+		panic("extract should not be called before create")
+	}
+
 	if a.extractionDir != "" {
 		return a.extractionDir, nil
 	}
@@ -107,12 +112,25 @@ func (a *BuildContextArchive) CleanupExtractedDir(ctx context.Context) {
 }
 
 func (a *BuildContextArchive) CalculatePathsChecksum(ctx context.Context, paths []string) (string, error) {
+	sort.Strings(paths)
+	paths = util.UniqStrings(paths)
+
 	dir, err := a.ExtractOrGetExtractedDir(ctx)
 	if err != nil {
 		return "", fmt.Errorf("unable to access context directory: %w", err)
 	}
 
-	_ = dir
+	var pathsHashes []string
+	for _, path := range paths {
+		p := filepath.Join(dir, path)
+
+		hash, err := util.HashContentsAndPathsRecurse(p)
+		if err != nil {
+			return "", fmt.Errorf("unable to calculate hash: %w", err)
+		}
+
+		pathsHashes = append(pathsHashes, hash)
+	}
 
-	return "", nil
+	return util.Sha256Hash(pathsHashes...), nil
 }

pkg/build/stage/instruction/add.go

@@ -3,6 +3,9 @@ package instruction
 import (
 	"context"
 	"fmt"
+	"strings"
+
+	"github.com/containers/buildah/copier"
 
 	"github.com/werf/werf/pkg/build/stage"
 	"github.com/werf/werf/pkg/config"
@@ -34,11 +37,20 @@ func (stg *Add) GetDependencies(ctx context.Context, c stage.Conveyor, cb contai
 	args = append(args, "Chown", stg.instruction.Data.Chown)
 	args = append(args, "Chmod", stg.instruction.Data.Chmod)
 
-	pathsChecksum, err := buildContextArchive.CalculatePathsChecksum(ctx, stg.instruction.Data.Src)
-	if err != nil {
-		return "", fmt.Errorf("unable to calculate build context paths checksum: %w", err)
+	var fileGlobSrc []string
+	for _, src := range stg.instruction.Data.Src {
+		if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") {
+			fileGlobSrc = append(fileGlobSrc, src)
+		}
+	}
+
+	if len(fileGlobSrc) > 0 {
+		if srcChecksum, err := calculateBuildContextGlobsChecksum(ctx, fileGlobSrc, true, buildContextArchive); err != nil {
+			return "", fmt.Errorf("unable to calculate build context globs checksum: %w", err)
+		} else {
+			args = append(args, "SrcChecksum", srcChecksum)
+		}
 	}
-	args = append(args, "SrcChecksum", pathsChecksum)
 
 	// TODO(staged-dockerfile): support http src and --checksum option: https://docs.docker.com/engine/reference/builder/#verifying-a-remote-file-checksum-add---checksumchecksum-http-src-dest
 	// TODO(staged-dockerfile): support git ref: https://docs.docker.com/engine/reference/builder/#adding-a-git-repository-add-git-ref-dir
@@ -47,3 +59,36 @@ func (stg *Add) GetDependencies(ctx context.Context, c stage.Conveyor, cb contai
 
 	return util.Sha256Hash(args...), nil
 }
+
+func calculateBuildContextGlobsChecksum(ctx context.Context, fileGlobs []string, checkForArchives bool, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
+	contextDir, err := buildContextArchive.ExtractOrGetExtractedDir(ctx)
+	if err != nil {
+		return "", fmt.Errorf("unable to get build context dir: %w", err)
+	}
+
+	globStats, err := copier.Stat(contextDir, contextDir, copier.StatOptions{CheckForArchives: checkForArchives}, fileGlobs)
+	if err != nil {
+		return "", fmt.Errorf("unable to stat globs: %w", err)
+	}
+	if len(globStats) == 0 {
+		return "", fmt.Errorf("no glob matches for globs: %v", fileGlobs)
+	}
+
+	var matches []string
+	for _, globStat := range globStats {
+		if globStat.Error != "" {
+			return "", fmt.Errorf("unable to stat glob %q: %w", globStat.Glob, globStat.Error)
+		}
+
+		for _, match := range globStat.Globbed {
+			matches = append(matches, match)
+		}
+	}
+
+	pathsChecksum, err := buildContextArchive.CalculatePathsChecksum(ctx, matches)
+	if err != nil {
+		return "", fmt.Errorf("unable to calculate build context paths checksum: %w", err)
+	}
+
+	return pathsChecksum, nil
+}

pkg/build/stage/instruction/copy.go

@@ -2,6 +2,7 @@ package instruction
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/werf/werf/pkg/build/stage"
 	"github.com/werf/werf/pkg/config"
@@ -45,6 +46,16 @@ func (stg *Copy) GetDependencies(ctx context.Context, c stage.Conveyor, cb conta
 	args = append(args, "Chmod", stg.instruction.Data.Chmod)
 	args = append(args, "ExpandedFrom", stg.backendInstruction.From)
 
+	if stg.UsesBuildContext() {
+		if srcChecksum, err := calculateBuildContextGlobsChecksum(ctx, stg.instruction.Data.Src, false, buildContextArchive); err != nil {
+			return "", fmt.Errorf("unable to calculate build context globs checksum: %w", err)
+		} else {
+			args = append(args, "SrcChecksum", srcChecksum)
+		}
+	}
+
+	// TODO(ilya-lesikov): should checksum of files from other image be calculated if --from specified?
+
 	// TODO(staged-dockerfile): support --link option: https://docs.docker.com/engine/reference/builder/#copy---link
 
 	return util.Sha256Hash(args...), nil

pkg/build/stage/instruction/run.go

@@ -28,5 +28,8 @@ func (stg *Run) GetDependencies(ctx context.Context, c stage.Conveyor, cb contai
 
 	args = append(args, "Instruction", stg.instruction.Data.Name())
 	args = append(args, append([]string{"Command"}, stg.instruction.Data.Command...)...)
+
+	// TODO(ilya-lesikov): should bind mount with context as src be counted as dependency?
+
 	return util.Sha256Hash(args...), nil
 }

pkg/buildah/common.go

@@ -110,8 +110,9 @@ type ConfigOpts struct {
 type CopyOpts struct {
 	CommonOpts
 
-	Chown string
-	Chmod string
+	Chown   string
+	Chmod   string
+	Ignores []string
 }
 
 type AddOpts struct {
@@ -120,6 +121,7 @@ type AddOpts struct {
 	ContextDir string
 	Chown      string
 	Chmod      string
+	Ignores    []string
 }
 
 type (

pkg/buildah/native_linux.go

@@ -574,10 +574,7 @@ func (b *NativeBuildah) Copy(ctx context.Context, container, contextDir string,
 		Chmod:             opts.Chmod,
 		PreserveOwnership: false,
 		ContextDir:        contextDir,
-		// TODO(ilya-lesikov): ignore file?
-		Excludes: nil,
-		// TODO(ilya-lesikov): ignore file?
-		IgnoreFile: "",
+		Excludes:          opts.Ignores,
 	}, absSrc...); err != nil {
 		return fmt.Errorf("error copying files to %q: %w", dst, err)
 	}
@@ -610,10 +607,7 @@ func (b *NativeBuildah) Add(ctx context.Context, container string, src []string,
 		Chown:             opts.Chown,
 		PreserveOwnership: false,
 		ContextDir:        opts.ContextDir,
-		// TODO(ilya-lesikov): ignore file?
-		Excludes: nil,
-		// TODO(ilya-lesikov): ignore file?
-		IgnoreFile: "",
+		Excludes:          opts.Ignores,
 	}, expandedSrc...); err != nil {
 		return fmt.Errorf("error adding files to %q: %w", dst, err)
 	}

pkg/dockerfile/frontend/buildkit_dockerfile.go

@@ -108,11 +108,17 @@ func extractSrcAndDst(sourcesAndDest instructions.SourcesAndDest) ([]string, str
 	//  /home/user1/go/pkg/mod/github.com/moby/buildkit@v0.8.2/frontend/dockerfile/parser/parser.go:250
 	var src []string
 	for _, s := range sourcesAndDest[0 : len(sourcesAndDest)-1] {
-		s, _ = strconv.Unquote(s)
+		if unquoted, err := strconv.Unquote(s); err == nil {
+			s = unquoted
+		}
+
 		src = append(src, s)
 	}
 
-	dst, _ := strconv.Unquote(sourcesAndDest[len(sourcesAndDest)-1])
+	dst := sourcesAndDest[len(sourcesAndDest)-1]
+	if unquoted, err := strconv.Unquote(dst); err == nil {
+		dst = unquoted
+	}
 
 	return src, dst
 }

pkg/util/hashsum.go

@@ -3,10 +3,14 @@ package util
 import (
 	"crypto/sha256"
 	"fmt"
+	"io"
+	"os"
+	"path/filepath"
 	"strings"
 
 	"github.com/spaolacci/murmur3"
 	"golang.org/x/crypto/sha3"
+	"golang.org/x/mod/sumdb/dirhash"
 )
 
 // LegacyMurmurHash function returns a hash of non-fixed length (1-8 symbols)
@@ -31,6 +35,32 @@ func Sha256Hash(args ...string) string {
 	return fmt.Sprintf("%x", sum)
 }
 
+// For file: hash contents of file with its name.
+// For directory: hash contents of all files in directory, along with their relative filenames.
+func HashContentsAndPathsRecurse(path string) (string, error) {
+	path = filepath.Clean(path)
+
+	fi, err := os.Stat(path)
+	if err != nil {
+		return "", fmt.Errorf("unable to stat %q: %w", path, err)
+	}
+
+	var hash string
+	if fi.IsDir() {
+		if hash, err = dirhash.HashDir(path, "/", dirhash.Hash1); err != nil {
+			return "", fmt.Errorf("unable to calculate hash for dir %q: %w", path, err)
+		}
+	} else {
+		if hash, err = dirhash.Hash1([]string{filepath.Base(path)}, func(_ string) (io.ReadCloser, error) {
+			return os.Open(path)
+		}); err != nil {
+			return "", fmt.Errorf("unable to calculate hash for file %q: %w", path, err)
+		}
+	}
+
+	return hash, nil
+}
+
 func prepareHashArgs(args ...string) string {
 	return strings.Join(args, ":::")
 }