Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat(stapel-to-buildah): support git patches related stages
* Refactor container-backend paths removal method.
* Added git patch archive creation using in-memory buffer when passing patch-stage data.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
distorhead committed Apr 13, 2022
1 parent 367beb8 commit 79f71c1
Show file tree
Hide file tree
Showing 14 changed files with 444 additions and 143 deletions.
6 changes: 4 additions & 2 deletions pkg/build/build_phase.go
Expand Up @@ -46,8 +46,10 @@ type BuildOptions struct {
CustomTagFuncList []CustomTagFunc
}

type CustomTagFunc func(string, string) string
type ExportTagFunc func(string, string) string
type (
CustomTagFunc func(string, string) string
ExportTagFunc func(string, string) string
)

type IntrospectOptions struct {
Targets []IntrospectTarget
Expand Down
4 changes: 2 additions & 2 deletions pkg/build/stage/base.go
Expand Up @@ -339,7 +339,7 @@ func (s *BaseStage) addServiceMountsVolumes(mountpointsByType map[string][]strin
} else {
stageImage.Builder.StapelStageBuilder().AddBuildVolumes(volume)
if cleanupMountpoints {
stageImage.Builder.StapelStageBuilder().AddPathsToRemove(absoluteMountpoint)
stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveInsidePath, []string{absoluteMountpoint}, nil)
}
}
}
Expand Down Expand Up @@ -439,7 +439,7 @@ func (s *BaseStage) addCustomMountVolumes(mountpointsByFrom map[string][]string,
} else {
stageImage.Builder.StapelStageBuilder().AddBuildVolumes(volume)
if cleanupMountpoints {
stageImage.Builder.StapelStageBuilder().AddPathsToRemove(absoluteMountpoint)
stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveInsidePath, []string{absoluteMountpoint}, nil)
}
}
}
Expand Down
141 changes: 109 additions & 32 deletions pkg/build/stage/git_mapping.go
@@ -1,6 +1,7 @@
package stage

import (
"archive/tar"
"context"
"crypto/sha256"
"fmt"
Expand All @@ -13,6 +14,9 @@ import (
"strings"
"sync"

"github.com/djherbis/buffer"
"github.com/djherbis/nio/v3"

"github.com/werf/logboek"
"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/git_repo"
Expand Down Expand Up @@ -238,31 +242,6 @@ func (gm *GitMapping) applyPatchCommand(patchFile *ContainerFileDescriptor, arch
return commands, nil
}

func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage)
if err != nil {
return fmt.Errorf("unable to get base commit from built image: %w", err)
}

toCommitInfo, err := gm.GetLatestCommitInfo(ctx, c)
if err != nil {
return fmt.Errorf("unable to get latest commit info: %w", err)
}

commands, err := gm.baseApplyPatchCommand(ctx, fromCommit, toCommitInfo.Commit, prevBuiltImage)
if err != nil {
return err
}

if err := gm.applyScript(stageImage, commands); err != nil {
return err
}

gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo)

return nil
}

func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (ImageCommitInfo, error) {
res := ImageCommitInfo{}

Expand Down Expand Up @@ -571,7 +550,111 @@ func (gm *GitMapping) applyArchiveCommand(archiveFile *ContainerFileDescriptor,
return commands, nil
}

func (gm *GitMapping) PreparePatchForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage)
if err != nil {
return fmt.Errorf("unable to get base commit from built image: %w", err)
}
toCommitInfo, err := gm.GetLatestCommitInfo(ctx, c)
if err != nil {
return fmt.Errorf("unable to get latest commit info: %w", err)
}

if c.UseLegacyStapelBuilder(cb) {
commands, err := gm.baseApplyPatchCommand(ctx, fromCommit, toCommitInfo.Commit, prevBuiltImage)
if err != nil {
return err
}

if err := gm.applyScript(stageImage, commands); err != nil {
return err
}
} else {
patchOpts, err := gm.makePatchOptions(ctx, fromCommit, toCommitInfo.Commit, false, false)
if err != nil {
return fmt.Errorf("unable to make patch options: %w", err)
}
logboek.Context(ctx).Debug().LogF("Creating patch from %s to %s\n", fromCommit, toCommitInfo.Commit)
patch, err := gm.GitRepo().GetOrCreatePatch(ctx, *patchOpts)
if err != nil {
return fmt.Errorf("unable to create patch: %w", err)
}
if patch.IsEmpty() {
return nil
}

archiveOpts, err := gm.makeArchiveOptions(ctx, toCommitInfo.Commit)
if err != nil {
return err
}
logboek.Context(ctx).Debug().LogF("Creating archive for commit %s\n", toCommitInfo.Commit)
archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts)
if err != nil {
return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err)
}
var archiveType container_backend.ArchiveType
gitArchiveType, err := gm.getArchiveType(ctx, toCommitInfo.Commit)
if err != nil {
return fmt.Errorf("unable to determine git archive type: %w", err)
}
switch gitArchiveType {
case git_repo.FileArchive:
archiveType = container_backend.FileArchive
case git_repo.DirectoryArchive:
archiveType = container_backend.DirectoryArchive
}

tarBuf := buffer.New(64 * 1024 * 1024)
patchArchiveReader, patchArchiveWriter := nio.Pipe(tarBuf)
f, err := os.Open(archive.GetFilePath())
if err != nil {
return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err)
}

var includePaths []string
for _, path := range patch.GetPaths() {
if util.IsStringsContainValue(patch.GetPathsToRemove(), path) {
continue
}
includePaths = append(includePaths, path)
}

go func() {
logboek.Context(ctx).Debug().LogF("Starting archive %q filtering process, includePaths: %v\n", archive.GetFilePath(), includePaths)
tw := tar.NewWriter(patchArchiveWriter)
defer func() {
if err := tw.Close(); err != nil {
logboek.Context(ctx).Error().LogF("ERROR: %s\n", err)
panic("tar writer close failed")
}
}()

if err := util.CopyTar(ctx, f, tw, util.CopyTarOptions{IncludePaths: includePaths}); err != nil {
logboek.Context(ctx).Error().LogF("ERROR: %s\n", err)
panic("tar copy failed")
}
}()

logboek.Context(ctx).Debug().LogF("Adding git patch data archive with included paths: %v\n", includePaths)
stageImage.Builder.StapelStageBuilder().AddDataArchive(patchArchiveReader, archiveType, gm.To)

logboek.Context(ctx).Debug().LogF("Adding git paths to remove: %v\n", patch.GetPathsToRemove())
var pathsToRemove []string
for _, path := range patch.GetPathsToRemove() {
pathsToRemove = append(pathsToRemove, filepath.Join(gm.To, path))
}
stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveExactPathWithEmptyParentDirs, pathsToRemove, []string{gm.To})
}

gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo)

return nil
}

func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage) error {
// FIXME: legacy stapel
// FIXME: file-archive type

commitInfo, err := gm.GetLatestCommitInfo(ctx, c)
if err != nil {
return fmt.Errorf("unable to get latest commit info: %w", err)
Expand All @@ -591,12 +674,10 @@ func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb
if err != nil {
return err
}

archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts)
if err != nil {
return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err)
}

var archiveType container_backend.ArchiveType

gitArchiveType, err := gm.getArchiveType(ctx, commitInfo.Commit)
Expand All @@ -618,11 +699,7 @@ func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb
return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err)
}

stageImage.Builder.StapelStageBuilder().AddDataArchives(container_backend.DataArchive{
Data: f,
Type: archiveType,
To: gm.To,
})
stageImage.Builder.StapelStageBuilder().AddDataArchive(f, archiveType, gm.To)
}

gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, commitInfo)
Expand Down
17 changes: 7 additions & 10 deletions pkg/build/stage/git_patch.go
Expand Up @@ -78,20 +78,17 @@ func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr contain
}

func (s *GitPatchStage) prepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
if c.UseLegacyStapelBuilder(cr) {
for _, gitMapping := range s.gitMappings {
if err := gitMapping.ApplyPatchCommand(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
return err
}
for _, gitMapping := range s.gitMappings {
if err := gitMapping.PreparePatchForImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil {
return err
}
}

if c.UseLegacyStapelBuilder(cr) {
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetPatchesCacheDir(), s.ContainerPatchesDir))
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetArchivesCacheDir(), s.ContainerArchivesDir))
stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", s.ScriptsDir, s.ContainerScriptsDir))

return nil
} else {
// TODO(stapel-to-buildah)
panic("not implemented")
}

return nil
}
53 changes: 38 additions & 15 deletions pkg/container_backend/build_stapel_stage_options.go
Expand Up @@ -21,8 +21,8 @@ type BuildStapelStageOptionsInterface interface {
AddBuildVolumes(volumes ...string) BuildStapelStageOptionsInterface
AddCommands(commands ...string) BuildStapelStageOptionsInterface

AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface
AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface
AddDataArchive(archive io.ReadCloser, archiveType ArchiveType, to string) BuildStapelStageOptionsInterface
RemoveData(removeType RemoveType, paths []string, keepParentDirs []string) BuildStapelStageOptionsInterface
AddDependencyImport(imageName, fromPath, toPath string, includePaths, excludePaths []string, owner, group string) BuildStapelStageOptionsInterface
}

Expand All @@ -42,9 +42,9 @@ type BuildStapelStageOptions struct {
BuildVolumes []string
Commands []string

DataArchives []DataArchive
PathsToRemove []string
DependenciesImports []DependencyImport
DataArchiveSpecs []DataArchiveSpec
RemoveDataSpecs []RemoveDataSpec
DependencyImportSpecs []DependencyImportSpec
}

type ArchiveType int
Expand All @@ -55,13 +55,28 @@ const (
DirectoryArchive
)

type DataArchive struct {
Data io.ReadCloser
Type ArchiveType
To string
type DataArchiveSpec struct {
Archive io.ReadCloser
Type ArchiveType
To string
}

type DependencyImport struct {
type RemoveType int

//go:generate stringer -type=RemoveType
const (
RemoveExactPath RemoveType = iota
RemoveExactPathWithEmptyParentDirs
RemoveInsidePath
)

type RemoveDataSpec struct {
Type RemoveType
Paths []string
KeepParentDirs []string
}

type DependencyImportSpec struct {
ImageName string
FromPath string
ToPath string
Expand Down Expand Up @@ -140,18 +155,26 @@ func (opts *BuildStapelStageOptions) AddCommands(commands ...string) BuildStapel
return opts
}

func (opts *BuildStapelStageOptions) AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface {
opts.DataArchives = append(opts.DataArchives, archives...)
func (opts *BuildStapelStageOptions) AddDataArchive(archive io.ReadCloser, archiveType ArchiveType, to string) BuildStapelStageOptionsInterface {
opts.DataArchiveSpecs = append(opts.DataArchiveSpecs, DataArchiveSpec{
Archive: archive,
Type: archiveType,
To: to,
})
return opts
}

func (opts *BuildStapelStageOptions) AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface {
opts.PathsToRemove = append(opts.PathsToRemove, paths...)
func (opts *BuildStapelStageOptions) RemoveData(removeType RemoveType, paths []string, keepParentDirs []string) BuildStapelStageOptionsInterface {
opts.RemoveDataSpecs = append(opts.RemoveDataSpecs, RemoveDataSpec{
Type: removeType,
Paths: paths,
KeepParentDirs: keepParentDirs,
})
return opts
}

func (opts *BuildStapelStageOptions) AddDependencyImport(imageName, fromPath, toPath string, includePaths, excludePaths []string, owner, group string) BuildStapelStageOptionsInterface {
opts.DependenciesImports = append(opts.DependenciesImports, DependencyImport{
opts.DependencyImportSpecs = append(opts.DependencyImportSpecs, DependencyImportSpec{
ImageName: imageName,
FromPath: fromPath,
ToPath: toPath,
Expand Down

0 comments on commit 79f71c1

Please sign in to comment.