feat(stapel-to-buildah): implemented dependencies checksum using buil…

…dah container backend Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
werf · Apr 15, 2022 · 9596f6d · 9596f6d
1 parent b59afde
commit 9596f6d
Show file tree

Hide file tree

Showing 21 changed files with 188 additions and 61 deletions.
diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go
@@ -567,7 +567,7 @@ func extractLegacyStageImage(stageImage *stage.StageImage) *container_backend.Le
 
 func (phase *BuildPhase) calculateStage(ctx context.Context, img *Image, stg stage.Interface) (bool, func(), error) {
 	// FIXME(stapel-to-buildah): store StageImage-s everywhere in stage and build pkgs
-	stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg))
+	stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg))
 	if err != nil {
 		return false, nil, err
 	}

diff --git a/pkg/build/stage/base.go b/pkg/build/stage/base.go
@@ -126,7 +126,7 @@ func (s *BaseStage) FetchDependencies(_ context.Context, _ Conveyor, _ container
 	return nil
 }
 
-func (s *BaseStage) GetDependencies(_ context.Context, _ Conveyor, _, _ *StageImage) (string, error) {
+func (s *BaseStage) GetDependencies(_ context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	panic("method must be implemented!")
 }
 

diff --git a/pkg/build/stage/before_install.go b/pkg/build/stage/before_install.go
@@ -27,7 +27,7 @@ type BeforeInstallStage struct {
 	*UserStage
 }
 
-func (s *BeforeInstallStage) GetDependencies(ctx context.Context, _ Conveyor, _, _ *StageImage) (string, error) {
+func (s *BeforeInstallStage) GetDependencies(ctx context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	return s.builder.BeforeInstallChecksum(ctx), nil
 }
 

diff --git a/pkg/build/stage/before_setup.go b/pkg/build/stage/before_setup.go
@@ -28,7 +28,7 @@ type BeforeSetupStage struct {
 	*UserWithGitPatchStage
 }
 
-func (s *BeforeSetupStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
+func (s *BeforeSetupStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, BeforeSetup)
 	if err != nil {
 		return "", err

diff --git a/pkg/build/stage/conveyor.go b/pkg/build/stage/conveyor.go
@@ -30,7 +30,7 @@ type Conveyor interface {
 
 	GiterminismManager() giterminism_manager.Interface
 
-	UseLegacyStapelBuilder(cr container_backend.ContainerBackend) bool
+	UseLegacyStapelBuilder(cb container_backend.ContainerBackend) bool
 }
 
 type VirtualMergeOptions struct {

diff --git a/pkg/build/stage/dependencies.go b/pkg/build/stage/dependencies.go
@@ -67,14 +67,14 @@ type DependenciesStage struct {
 	dependencies []*config.Dependency
 }
 
-func (s *DependenciesStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
+func (s *DependenciesStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	var args []string
 
 	for ind, elm := range s.imports {
 		var sourceChecksum string
 		var err error
 		if err := logboek.Context(ctx).Info().LogProcess("Getting import %d source checksum ...", ind).DoError(func() error {
-			sourceChecksum, err = s.getImportSourceChecksum(ctx, c, elm)
+			sourceChecksum, err = s.getImportSourceChecksum(ctx, c, cb, elm)
 			return err
 		}); err != nil {
 			return "", fmt.Errorf("unable to get import %d source checksum: %w", ind, err)
@@ -217,15 +217,15 @@ func (s *DependenciesStage) PrepareImage(ctx context.Context, c Conveyor, cr con
 	}
 }
 
-func (s *DependenciesStage) getImportSourceChecksum(ctx context.Context, c Conveyor, importElm *config.Import) (string, error) {
+func (s *DependenciesStage) getImportSourceChecksum(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, importElm *config.Import) (string, error) {
 	importSourceID := getImportSourceID(c, importElm)
 	importMetadata, err := c.GetImportMetadata(ctx, s.projectName, importSourceID)
 	if err != nil {
 		return "", fmt.Errorf("unable to get import metadata: %w", err)
 	}
 
 	if importMetadata == nil {
-		checksum, err := s.generateImportChecksum(ctx, c, importElm)
+		checksum, err := s.generateImportChecksum(ctx, c, cb, importElm)
 		if err != nil {
 			return "", fmt.Errorf("unable to generate import source checksum: %w", err)
 		}
@@ -245,59 +245,83 @@ func (s *DependenciesStage) getImportSourceChecksum(ctx context.Context, c Conve
 	return importMetadata.Checksum, nil
 }
 
-func (s *DependenciesStage) generateImportChecksum(ctx context.Context, c Conveyor, importElm *config.Import) (string, error) {
+func (s *DependenciesStage) generateImportChecksum(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, importElm *config.Import) (string, error) {
 	if err := fetchSourceImageDockerImage(ctx, c, importElm); err != nil {
 		return "", fmt.Errorf("unable to fetch source image: %w", err)
 	}
 
 	sourceImageDockerImageName := getSourceImageDockerImageName(c, importElm)
-	importSourceID := getImportSourceID(c, importElm)
 
-	stapelContainerName, err := stapel.GetOrCreateContainer(ctx)
-	if err != nil {
-		return "", err
-	}
+	if c.UseLegacyStapelBuilder(cb) {
+		importSourceID := getImportSourceID(c, importElm)
 
-	importHostTmpDir := filepath.Join(s.imageTmpDir, string(s.Name()), "imports", importSourceID)
-	importContainerDir := s.containerWerfDir
+		stapelContainerName, err := stapel.GetOrCreateContainer(ctx)
+		if err != nil {
+			return "", err
+		}
 
-	importScriptHostTmpPath := filepath.Join(importHostTmpDir, "script.sh")
-	resultChecksumHostTmpPath := filepath.Join(importHostTmpDir, "checksum")
-	importScriptContainerPath := path.Join(importContainerDir, "script.sh")
-	resultChecksumContainerPath := path.Join(importContainerDir, "checksum")
+		importHostTmpDir := filepath.Join(s.imageTmpDir, string(s.Name()), "imports", importSourceID)
+		importContainerDir := s.containerWerfDir
 
-	command := generateChecksumCommand(importElm.Add, importElm.IncludePaths, importElm.ExcludePaths, resultChecksumContainerPath)
-	if err := stapel.CreateScript(importScriptHostTmpPath, []string{command}); err != nil {
-		return "", fmt.Errorf("unable to create script: %w", err)
-	}
+		importScriptHostTmpPath := filepath.Join(importHostTmpDir, "script.sh")
+		resultChecksumHostTmpPath := filepath.Join(importHostTmpDir, "checksum")
+		importScriptContainerPath := path.Join(importContainerDir, "script.sh")
+		resultChecksumContainerPath := path.Join(importContainerDir, "checksum")
 
-	runArgs := []string{
-		"--rm",
-		"--user=0:0",
-		"--workdir=/",
-		fmt.Sprintf("--volumes-from=%s", stapelContainerName),
-		fmt.Sprintf("--volume=%s:%s", importHostTmpDir, importContainerDir),
-		fmt.Sprintf("--entrypoint=%s", stapel.BashBinPath()),
-		sourceImageDockerImageName,
-		importScriptContainerPath,
-	}
+		command := generateChecksumCommand(importElm.Add, importElm.IncludePaths, importElm.ExcludePaths, resultChecksumContainerPath)
+		if err := stapel.CreateScript(importScriptHostTmpPath, []string{command}); err != nil {
+			return "", fmt.Errorf("unable to create script: %w", err)
+		}
 
-	if debugImportSourceChecksum() {
-		fmt.Println(runArgs)
-	}
+		runArgs := []string{
+			"--rm",
+			"--user=0:0",
+			"--workdir=/",
+			fmt.Sprintf("--volumes-from=%s", stapelContainerName),
+			fmt.Sprintf("--volume=%s:%s", importHostTmpDir, importContainerDir),
+			fmt.Sprintf("--entrypoint=%s", stapel.BashBinPath()),
+			sourceImageDockerImageName,
+			importScriptContainerPath,
+		}
 
-	if output, err := docker.CliRun_RecordedOutput(ctx, runArgs...); err != nil {
-		logboek.Context(ctx).Error().LogF("%s", output)
-		return "", err
-	}
+		if debugImportSourceChecksum() {
+			fmt.Println(runArgs)
+		}
 
-	data, err := ioutil.ReadFile(resultChecksumHostTmpPath)
-	if err != nil {
-		return "", fmt.Errorf("unable to read file with import source checksum: %w", err)
-	}
+		if output, err := docker.CliRun_RecordedOutput(ctx, runArgs...); err != nil {
+			logboek.Context(ctx).Error().LogF("%s", output)
+			return "", err
+		}
 
-	checksum := strings.TrimSpace(string(data))
-	return checksum, nil
+		data, err := ioutil.ReadFile(resultChecksumHostTmpPath)
+		if err != nil {
+			return "", fmt.Errorf("unable to read file with import source checksum: %w", err)
+		}
+
+		checksum := strings.TrimSpace(string(data))
+
+		return checksum, nil
+	} else {
+		var checksum string
+		var err error
+
+		logboek.Context(ctx).Debug().LogProcess("Calculating dependency import checksum").Do(func() {
+			checksum, err = cb.CalculateDependencyImportChecksum(ctx, container_backend.DependencyImportSpec{
+				ImageName:    sourceImageDockerImageName,
+				FromPath:     importElm.Add,
+				ToPath:       importElm.To,
+				IncludePaths: importElm.IncludePaths,
+				ExcludePaths: importElm.ExcludePaths,
+				Owner:        importElm.Owner,
+				Group:        importElm.Group,
+			})
+		})
+
+		if err != nil {
+			return "", fmt.Errorf("unable to calculate dependency import checksum in %s: %w", sourceImageDockerImageName, err)
+		}
+		return checksum, nil
+	}
 }
 
 func generateChecksumCommand(from string, includePaths, excludePaths []string, resultChecksumPath string) string {

diff --git a/pkg/build/stage/dependencies_test.go b/pkg/build/stage/dependencies_test.go
@@ -30,7 +30,7 @@ var _ = Describe("DependenciesStage", func() {
 				Builder: stageBuilder,
 			}
 
-			digest, err := stage.GetDependencies(ctx, conveyor, nil, stageImage)
+			digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage)
 			Expect(err).To(Succeed())
 			Expect(digest).To(Equal(data.ExpectedDigest))
 

diff --git a/pkg/build/stage/docker_instructions.go b/pkg/build/stage/docker_instructions.go
@@ -30,7 +30,7 @@ type DockerInstructionsStage struct {
 	instructions *config.Docker
 }
 
-func (s *DockerInstructionsStage) GetDependencies(_ context.Context, _ Conveyor, _, _ *StageImage) (string, error) {
+func (s *DockerInstructionsStage) GetDependencies(_ context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	var args []string
 
 	args = append(args, s.instructions.Volume...)

diff --git a/pkg/build/stage/dockerfile.go b/pkg/build/stage/dockerfile.go
@@ -411,7 +411,7 @@ func isUnsupportedMediaTypeError(err error) bool {
 
 var errImageNotExistLocally = errors.New("IMAGE_NOT_EXIST_LOCALLY")
 
-func (s *DockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
+func (s *DockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c)
 
 	resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash)

diff --git a/pkg/build/stage/dockerfile_test.go b/pkg/build/stage/dockerfile_test.go
@@ -78,7 +78,7 @@ var _ = Describe("DockerfileStage", func() {
 				Builder: stageBuilder,
 			}
 
-			digest, err := stage.GetDependencies(ctx, conveyor, nil, stageImage)
+			digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage)
 			Expect(err).To(Succeed())
 			Expect(digest).To(Equal(data.TestDependencies.ExpectedDigest))
 

diff --git a/pkg/build/stage/from.go b/pkg/build/stage/from.go
@@ -47,7 +47,7 @@ type FromStage struct {
 	cacheVersion                 string
 }
 
-func (s *FromStage) GetDependencies(_ context.Context, c Conveyor, prevImage, _ *StageImage) (string, error) {
+func (s *FromStage) GetDependencies(_ context.Context, c Conveyor, _ container_backend.ContainerBackend, prevImage, _ *StageImage) (string, error) {
 	var args []string
 
 	if s.cacheVersion != "" {

diff --git a/pkg/build/stage/git_archive.go b/pkg/build/stage/git_archive.go
@@ -47,7 +47,7 @@ func (s *GitArchiveStage) SelectSuitableStage(ctx context.Context, c Conveyor, s
 }
 
 // TODO: 1.3 add git mapping type (dir, file, ...) to gitArchive stage digest
-func (s *GitArchiveStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
+func (s *GitArchiveStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	var args []string
 	for _, gitMapping := range s.gitMappings {
 		if gitMapping.IsLocal() {

diff --git a/pkg/build/stage/git_cache.go b/pkg/build/stage/git_cache.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/werf/werf/pkg/container_backend"
 	"github.com/werf/werf/pkg/image"
 	"github.com/werf/werf/pkg/util"
 )
@@ -45,7 +46,7 @@ func (s *GitCacheStage) IsEmpty(ctx context.Context, c Conveyor, prevBuiltImage
 	return isEmpty, nil
 }
 
-func (s *GitCacheStage) GetDependencies(ctx context.Context, c Conveyor, _, prevBuiltImage *StageImage) (string, error) {
+func (s *GitCacheStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, prevBuiltImage *StageImage) (string, error) {
 	patchSize, err := s.gitMappingsPatchSize(ctx, c, prevBuiltImage)
 	if err != nil {
 		return "", err

diff --git a/pkg/build/stage/git_latest_patch.go b/pkg/build/stage/git_latest_patch.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 
+	"github.com/werf/werf/pkg/container_backend"
 	"github.com/werf/werf/pkg/image"
 	"github.com/werf/werf/pkg/util"
 )
@@ -49,7 +50,7 @@ func (s *GitLatestPatchStage) IsEmpty(ctx context.Context, c Conveyor, prevBuilt
 	return isEmpty, nil
 }
 
-func (s *GitLatestPatchStage) GetDependencies(ctx context.Context, c Conveyor, _, prevBuiltImage *StageImage) (string, error) {
+func (s *GitLatestPatchStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, prevBuiltImage *StageImage) (string, error) {
 	var args []string
 
 	for _, gitMapping := range s.gitMappings {

diff --git a/pkg/build/stage/install.go b/pkg/build/stage/install.go
@@ -28,7 +28,7 @@ type InstallStage struct {
 	*UserWithGitPatchStage
 }
 
-func (s *InstallStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
+func (s *InstallStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, Install)
 	if err != nil {
 		return "", err

diff --git a/pkg/build/stage/interface.go b/pkg/build/stage/interface.go
@@ -14,11 +14,11 @@ type Interface interface {
 
 	IsEmpty(ctx context.Context, c Conveyor, prevBuiltImage *StageImage) (bool, error)
 
-	FetchDependencies(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error
-	GetDependencies(ctx context.Context, c Conveyor, prevImage *StageImage, prevBuiltImage *StageImage) (string, error)
+	FetchDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error
+	GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage *StageImage, prevBuiltImage *StageImage) (string, error)
 	GetNextStageDependencies(ctx context.Context, c Conveyor) (string, error)
 
-	PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error
+	PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error
 
 	PreRunHook(context.Context, Conveyor) error
 

diff --git a/pkg/build/stage/setup.go b/pkg/build/stage/setup.go
@@ -28,7 +28,7 @@ type SetupStage struct {
 	*UserWithGitPatchStage
 }
 
-func (s *SetupStage) GetDependencies(ctx context.Context, c Conveyor, _, _ *StageImage) (string, error) {
+func (s *SetupStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
 	stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, Setup)
 	if err != nil {
 		return "", err