feat(stapel-to-buildah): git archive stage implementation

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>

pkg/build/stage/git.go

@@ -30,14 +30,14 @@ func (s *GitStage) PrepareImage(ctx context.Context, c Conveyor, cr container_ba
 		return err
 	}
 
-	if c.UseLegacyStapelBuilder(cr) {
-		if c.GiterminismManager().Dev() {
-			stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(map[string]string{imagePkg.WerfDevLabel: "true"})
+	if c.GiterminismManager().Dev() {
+		addLabels := map[string]string{imagePkg.WerfDevLabel: "true"}
+		if c.UseLegacyStapelBuilder(cr) {
+			stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(addLabels)
+		} else {
+			stageImage.Builder.StapelStageBuilder().AddLabels(addLabels)
 		}
-
-		return nil
-	} else {
-		// TODO(stapel-to-buildah)
-		panic("not implemented")
 	}
+
+	return nil
 }

pkg/build/stage/git_archive.go

@@ -69,25 +69,26 @@ func (s *GitArchiveStage) GetNextStageDependencies(ctx context.Context, c Convey
 }
 
 func (s *GitArchiveStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
+	if !c.UseLegacyStapelBuilder(cr) {
+		stageImage.Builder.StapelStageBuilder().SetStageType(container_backend.DataArchivesStage)
+	}
+
 	if err := s.GitStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
 		return err
 	}
 
-	if c.UseLegacyStapelBuilder(cr) {
-		for _, gitMapping := range s.gitMappings {
-			if err := gitMapping.ApplyArchiveCommand(ctx, c, stageImage); err != nil {
-				return err
-			}
+	for _, gitMapping := range s.gitMappings {
+		if err := gitMapping.PrepareArchiveForImage(ctx, c, cr, stageImage); err != nil {
+			return fmt.Errorf("unable to prepare git mapping %s for image stage: %w", gitMapping.Name, err)
 		}
+	}
 
+	if c.UseLegacyStapelBuilder(cr) {
 		stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetArchivesCacheDir(), s.ContainerArchivesDir))
 		stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", s.ScriptsDir, s.ContainerScriptsDir))
-
-		return nil
-	} else {
-		// TODO(stapel-to-buildah)
-		panic("not implemented")
 	}
+
+	return nil
 }
 
 func (s *GitArchiveStage) IsEmpty(ctx context.Context, c Conveyor, stageImage *StageImage) (bool, error) {

pkg/build/stage/git_mapping.go

@@ -14,6 +14,7 @@ import (
 	"sync"
 
 	"github.com/werf/logboek"
+	"github.com/werf/werf/pkg/container_backend"
 	"github.com/werf/werf/pkg/git_repo"
 	"github.com/werf/werf/pkg/path_matcher"
 	"github.com/werf/werf/pkg/stapel"
@@ -237,7 +238,7 @@ func (gm *GitMapping) applyPatchCommand(patchFile *ContainerFileDescriptor, arch
 	return commands, nil
 }
 
-func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, prevBuiltImage, stageImage *StageImage) error {
+func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
 	fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage)
 	if err != nil {
 		return fmt.Errorf("unable to get base commit from built image: %w", err)
@@ -257,7 +258,7 @@ func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, prevBui
 		return err
 	}
 
-	gm.AddGitCommitToImageLabels(stageImage, toCommitInfo)
+	gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo)
 
 	return nil
 }
@@ -296,21 +297,24 @@ func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (Imag
 	return res, nil
 }
 
-func (gm *GitMapping) AddGitCommitToImageLabels(stageImage *StageImage, commitInfo ImageCommitInfo) {
-	stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
-		gm.ImageGitCommitLabel(): commitInfo.Commit,
-	})
+func (gm *GitMapping) AddGitCommitToImageLabels(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage, commitInfo ImageCommitInfo) {
+	addLabels := make(map[string]string)
+	addLabels[gm.ImageGitCommitLabel()] = commitInfo.Commit
 
 	if commitInfo.VirtualMerge {
-		stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
-			gm.VirtualMergeLabel():           "true",
-			gm.VirtualMergeFromCommitLabel(): commitInfo.VirtualMergeFromCommit,
-			gm.VirtualMergeIntoCommitLabel(): commitInfo.VirtualMergeIntoCommit,
-		})
+		addLabels[gm.VirtualMergeLabel()] = "true"
+		addLabels[gm.VirtualMergeFromCommitLabel()] = commitInfo.VirtualMergeFromCommit
+		addLabels[gm.VirtualMergeIntoCommitLabel()] = commitInfo.VirtualMergeIntoCommit
 	} else {
-		stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{
-			gm.VirtualMergeLabel(): "false",
-		})
+		addLabels[gm.VirtualMergeLabel()] = "false"
+	}
+
+	if len(addLabels) > 0 {
+		if c.UseLegacyStapelBuilder(cb) {
+			stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(addLabels)
+		} else {
+			stageImage.Builder.StapelStageBuilder().AddLabels(addLabels)
+		}
 	}
 }
 
@@ -567,22 +571,61 @@ func (gm *GitMapping) applyArchiveCommand(archiveFile *ContainerFileDescriptor,
 	return commands, nil
 }
 
-func (gm *GitMapping) ApplyArchiveCommand(ctx context.Context, c Conveyor, stageImage *StageImage) error {
+func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage) error {
 	commitInfo, err := gm.GetLatestCommitInfo(ctx, c)
 	if err != nil {
 		return fmt.Errorf("unable to get latest commit info: %w", err)
 	}
 
-	commands, err := gm.baseApplyArchiveCommand(ctx, commitInfo.Commit, stageImage)
-	if err != nil {
-		return err
-	}
+	if c.UseLegacyStapelBuilder(cb) {
+		commands, err := gm.baseApplyArchiveCommand(ctx, commitInfo.Commit, stageImage)
+		if err != nil {
+			return err
+		}
 
-	if err := gm.applyScript(stageImage, commands); err != nil {
-		return err
+		if err := gm.applyScript(stageImage, commands); err != nil {
+			return err
+		}
+	} else {
+		archiveOpts, err := gm.makeArchiveOptions(ctx, commitInfo.Commit)
+		if err != nil {
+			return err
+		}
+
+		archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts)
+		if err != nil {
+			return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err)
+		}
+
+		var archiveType container_backend.ArchiveType
+
+		gitArchiveType, err := gm.getArchiveType(ctx, commitInfo.Commit)
+		if err != nil {
+			return fmt.Errorf("unable to determine git archive type: %w", err)
+		}
+
+		stageImage.Builder.StapelStageBuilder().AddLabels(map[string]string{gm.getArchiveTypeLabelName(): string(gitArchiveType)})
+
+		switch gitArchiveType {
+		case git_repo.FileArchive:
+			archiveType = container_backend.FileArchive
+		case git_repo.DirectoryArchive:
+			archiveType = container_backend.DirectoryArchive
+		}
+
+		f, err := os.Open(archive.GetFilePath())
+		if err != nil {
+			return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err)
+		}
+
+		stageImage.Builder.StapelStageBuilder().AddDataArchives(container_backend.DataArchive{
+			Data: f,
+			Type: archiveType,
+			To:   gm.To,
+		})
 	}
 
-	gm.AddGitCommitToImageLabels(stageImage, commitInfo)
+	gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, commitInfo)
 
 	return nil
 }

pkg/build/stage/git_patch.go

@@ -80,7 +80,7 @@ func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr contain
 func (s *GitPatchStage) prepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
 	if c.UseLegacyStapelBuilder(cr) {
 		for _, gitMapping := range s.gitMappings {
-			if err := gitMapping.ApplyPatchCommand(ctx, c, prevBuiltImage, stageImage); err != nil {
+			if err := gitMapping.ApplyPatchCommand(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
 				return err
 			}
 		}

pkg/container_backend/buildah_backend.go

@@ -14,6 +14,7 @@ import (
 	"github.com/werf/logboek"
 	"github.com/werf/werf/pkg/buildah"
 	"github.com/werf/werf/pkg/image"
+	"github.com/werf/werf/pkg/util"
 )
 
 type BuildahBackend struct {
@@ -148,6 +149,64 @@ type dependencyContainer struct {
 	Import    DependencyImport
 }
 
+func (runtime *BuildahBackend) applyDataArchives(ctx context.Context, container *containerDesc, opts BuildStapelStageOptions) error {
+	logboek.Context(ctx).Debug().LogF("Mounting container %q\n", container.Name)
+	if err := runtime.mountContainers(ctx, []*containerDesc{container}); err != nil {
+		return fmt.Errorf("unable to mount containers: %w", err)
+	}
+	defer func() {
+		logboek.Context(ctx).Debug().LogF("Unmounting container %q\n", container.Name)
+		if err := runtime.unmountContainers(ctx, []*containerDesc{container}); err != nil {
+			logboek.Context(ctx).Error().LogF("ERROR: unable to unmount containers: %s\n", err)
+		}
+	}()
+
+	for _, archive := range opts.DataArchives {
+		destPath := filepath.Join(container.RootMount, archive.To)
+
+		var extractDestPath string
+		switch archive.Type {
+		case DirectoryArchive:
+			extractDestPath = destPath
+		case FileArchive:
+			extractDestPath = filepath.Dir(destPath)
+		default:
+			return fmt.Errorf("unknown archive type %q", archive.Type)
+		}
+
+		_, err := os.Stat(destPath)
+		switch {
+		case os.IsNotExist(err):
+		case err != nil:
+			return fmt.Errorf("unable to access container path %q: %w", destPath, err)
+		default:
+			logboek.Context(ctx).Debug().LogF("Removing archive destination path %s\n", archive.To)
+			if err := os.RemoveAll(destPath); err != nil {
+				return fmt.Errorf("unable to cleanup archive destination path %s: %w", archive.To, err)
+			}
+		}
+
+		logboek.Context(ctx).Debug().LogF("Extracting archive into container path %s\n", archive.To)
+		if err := util.ExtractTar(archive.Data, extractDestPath); err != nil {
+			return fmt.Errorf("unable to extract data archive into %s: %w", archive.To, err)
+		}
+		if err := archive.Data.Close(); err != nil {
+			return fmt.Errorf("error closing archive data stream: %w", err)
+		}
+	}
+
+	for _, path := range opts.PathsToRemove {
+		destPath := filepath.Join(container.RootMount, path)
+
+		logboek.Context(ctx).Debug().LogF("Removing container path %s\n", path)
+		if err := os.RemoveAll(destPath); err != nil {
+			return fmt.Errorf("unable to remove path %s: %w", path, err)
+		}
+	}
+
+	return nil
+}
+
 func (runtime *BuildahBackend) applyDependencies(ctx context.Context, container *containerDesc, opts BuildStapelStageOptions) error {
 	var dependencies []*dependencyContainer
 
@@ -181,12 +240,12 @@ func (runtime *BuildahBackend) applyDependencies(ctx context.Context, container
 		}
 	}
 
-	logboek.Context(ctx).Debug().LogF("Mounting dependencies containers %v\n", dependenciesContainers)
+	logboek.Context(ctx).Debug().LogF("Mounting containers %v\n", append(dependenciesContainers, container))
 	if err := runtime.mountContainers(ctx, append(dependenciesContainers, container)); err != nil {
 		return fmt.Errorf("unable to mount containers: %w", err)
 	}
 	defer func() {
-		logboek.Context(ctx).Debug().LogF("Unmounting dependencies containers %v\n", dependenciesContainers)
+		logboek.Context(ctx).Debug().LogF("Unmounting containers %v\n", append(dependenciesContainers, container))
 		if err := runtime.unmountContainers(ctx, append(dependenciesContainers, container)); err != nil {
 			logboek.Context(ctx).Error().LogF("ERROR: unable to unmount containers: %s\n", err)
 		}
@@ -236,6 +295,10 @@ func (runtime *BuildahBackend) BuildStapelStage(ctx context.Context, stageType S
 		if err := runtime.applyDependencies(ctx, container, opts); err != nil {
 			return "", err
 		}
+	case DataArchivesStage:
+		if err := runtime.applyDataArchives(ctx, container, opts); err != nil {
+			return "", err
+		}
 	default:
 		return "", fmt.Errorf("unsupported stage type %q", stageType.String())
 	}

pkg/container_backend/stapelstagetype.go

@@ -1,6 +1,9 @@
 package container_backend
 
-import "fmt"
+import (
+	"fmt"
+	"io"
+)
 
 type StapelStageType int
 
@@ -10,6 +13,7 @@ const (
 	UserCommandsStage
 	DockerInstructionsStage
 	DependenciesStage
+	DataArchivesStage
 )
 
 type BuildStapelStageOptionsInterface interface {
@@ -25,6 +29,9 @@ type BuildStapelStageOptionsInterface interface {
 	SetWorkdir(workdir string) BuildStapelStageOptionsInterface
 	SetHealthcheck(healthcheck string) BuildStapelStageOptionsInterface
 
+	AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface
+	AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface
+
 	UserCommandsStage() UserCommandsStageOptionsInterface
 	DependenciesStage() DependenciesStageOptionsInterface
 }
@@ -50,10 +57,27 @@ type BuildStapelStageOptions struct {
 	Workdir      string
 	Healthcheck  string
 
+	DataArchives  []DataArchive
+	PathsToRemove []string
+
 	UserCommandsStageOptions
 	DependenciesStageOptions
 }
 
+type ArchiveType int
+
+//go:generate stringer -type=ArchiveType
+const (
+	FileArchive ArchiveType = iota
+	DirectoryArchive
+)
+
+type DataArchive struct {
+	Data io.ReadCloser
+	Type ArchiveType
+	To   string
+}
+
 func (opts *BuildStapelStageOptions) SetBaseImage(baseImage string) BuildStapelStageOptionsInterface {
 	opts.BaseImage = baseImage
 	return opts
@@ -118,6 +142,16 @@ func (opts *BuildStapelStageOptions) AddBuildVolumes(volumes ...string) BuildSta
 	return opts
 }
 
+func (opts *BuildStapelStageOptions) AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface {
+	opts.DataArchives = append(opts.DataArchives, archives...)
+	return opts
+}
+
+func (opts *BuildStapelStageOptions) AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface {
+	opts.PathsToRemove = append(opts.PathsToRemove, paths...)
+	return opts
+}
+
 func (opts *BuildStapelStageOptions) UserCommandsStage() UserCommandsStageOptionsInterface {
 	return &opts.UserCommandsStageOptions
 }