diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 05c1ea159e..7e43ab46b5 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -46,8 +46,10 @@ type BuildOptions struct { CustomTagFuncList []CustomTagFunc } -type CustomTagFunc func(string, string) string -type ExportTagFunc func(string, string) string +type ( + CustomTagFunc func(string, string) string + ExportTagFunc func(string, string) string +) type IntrospectOptions struct { Targets []IntrospectTarget diff --git a/pkg/build/stage/base.go b/pkg/build/stage/base.go index 55170b909c..4ff354040c 100644 --- a/pkg/build/stage/base.go +++ b/pkg/build/stage/base.go @@ -339,7 +339,7 @@ func (s *BaseStage) addServiceMountsVolumes(mountpointsByType map[string][]strin } else { stageImage.Builder.StapelStageBuilder().AddBuildVolumes(volume) if cleanupMountpoints { - stageImage.Builder.StapelStageBuilder().AddPathsToRemove(absoluteMountpoint) + stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveInsidePath, []string{absoluteMountpoint}, nil) } } } @@ -439,7 +439,7 @@ func (s *BaseStage) addCustomMountVolumes(mountpointsByFrom map[string][]string, } else { stageImage.Builder.StapelStageBuilder().AddBuildVolumes(volume) if cleanupMountpoints { - stageImage.Builder.StapelStageBuilder().AddPathsToRemove(absoluteMountpoint) + stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveInsidePath, []string{absoluteMountpoint}, nil) } } } diff --git a/pkg/build/stage/git_mapping.go b/pkg/build/stage/git_mapping.go index 4068a611da..003ad62b02 100644 --- a/pkg/build/stage/git_mapping.go +++ b/pkg/build/stage/git_mapping.go @@ -1,6 +1,7 @@ package stage import ( + "archive/tar" "context" "crypto/sha256" "fmt" @@ -13,6 +14,9 @@ import ( "strings" "sync" + "github.com/djherbis/buffer" + "github.com/djherbis/nio/v3" + "github.com/werf/logboek" "github.com/werf/werf/pkg/container_backend" "github.com/werf/werf/pkg/git_repo" @@ -238,31 +242,6 @@ func (gm *GitMapping) applyPatchCommand(patchFile *ContainerFileDescriptor, arch return commands, nil } -func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage) - if err != nil { - return fmt.Errorf("unable to get base commit from built image: %w", err) - } - - toCommitInfo, err := gm.GetLatestCommitInfo(ctx, c) - if err != nil { - return fmt.Errorf("unable to get latest commit info: %w", err) - } - - commands, err := gm.baseApplyPatchCommand(ctx, fromCommit, toCommitInfo.Commit, prevBuiltImage) - if err != nil { - return err - } - - if err := gm.applyScript(stageImage, commands); err != nil { - return err - } - - gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo) - - return nil -} - func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (ImageCommitInfo, error) { res := ImageCommitInfo{} @@ -571,7 +550,111 @@ func (gm *GitMapping) applyArchiveCommand(archiveFile *ContainerFileDescriptor, return commands, nil } +func (gm *GitMapping) PreparePatchForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { + fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage) + if err != nil { + return fmt.Errorf("unable to get base commit from built image: %w", err) + } + toCommitInfo, err := gm.GetLatestCommitInfo(ctx, c) + if err != nil { + return fmt.Errorf("unable to get latest commit info: %w", err) + } + + if c.UseLegacyStapelBuilder(cb) { + commands, err := gm.baseApplyPatchCommand(ctx, fromCommit, toCommitInfo.Commit, prevBuiltImage) + if err != nil { + return err + } + + if err := gm.applyScript(stageImage, commands); err != nil { + return err + } + } else { + patchOpts, err := gm.makePatchOptions(ctx, fromCommit, toCommitInfo.Commit, false, false) + if err != nil { + return fmt.Errorf("unable to make patch options: %w", err) + } + logboek.Context(ctx).Debug().LogF("Creating patch from %s to %s\n", fromCommit, toCommitInfo.Commit) + patch, err := gm.GitRepo().GetOrCreatePatch(ctx, *patchOpts) + if err != nil { + return fmt.Errorf("unable to create patch: %w", err) + } + if patch.IsEmpty() { + return nil + } + + archiveOpts, err := gm.makeArchiveOptions(ctx, toCommitInfo.Commit) + if err != nil { + return err + } + logboek.Context(ctx).Debug().LogF("Creating archive for commit %s\n", toCommitInfo.Commit) + archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts) + if err != nil { + return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err) + } + var archiveType container_backend.ArchiveType + gitArchiveType, err := gm.getArchiveType(ctx, toCommitInfo.Commit) + if err != nil { + return fmt.Errorf("unable to determine git archive type: %w", err) + } + switch gitArchiveType { + case git_repo.FileArchive: + archiveType = container_backend.FileArchive + case git_repo.DirectoryArchive: + archiveType = container_backend.DirectoryArchive + } + + tarBuf := buffer.New(64 * 1024 * 1024) + patchArchiveReader, patchArchiveWriter := nio.Pipe(tarBuf) + f, err := os.Open(archive.GetFilePath()) + if err != nil { + return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err) + } + + var includePaths []string + for _, path := range patch.GetPaths() { + if util.IsStringsContainValue(patch.GetPathsToRemove(), path) { + continue + } + includePaths = append(includePaths, path) + } + + go func() { + logboek.Context(ctx).Debug().LogF("Starting archive %q filtering process, includePaths: %v\n", archive.GetFilePath(), includePaths) + tw := tar.NewWriter(patchArchiveWriter) + defer func() { + if err := tw.Close(); err != nil { + logboek.Context(ctx).Error().LogF("ERROR: %s\n", err) + panic("tar writer close failed") + } + }() + + if err := util.CopyTar(ctx, f, tw, util.CopyTarOptions{IncludePaths: includePaths}); err != nil { + logboek.Context(ctx).Error().LogF("ERROR: %s\n", err) + panic("tar copy failed") + } + }() + + logboek.Context(ctx).Debug().LogF("Adding git patch data archive with included paths: %v\n", includePaths) + stageImage.Builder.StapelStageBuilder().AddDataArchive(patchArchiveReader, archiveType, gm.To) + + logboek.Context(ctx).Debug().LogF("Adding git paths to remove: %v\n", patch.GetPathsToRemove()) + var pathsToRemove []string + for _, path := range patch.GetPathsToRemove() { + pathsToRemove = append(pathsToRemove, filepath.Join(gm.To, path)) + } + stageImage.Builder.StapelStageBuilder().RemoveData(container_backend.RemoveExactPathWithEmptyParentDirs, pathsToRemove, []string{gm.To}) + } + + gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo) + + return nil +} + func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage) error { + // FIXME: legacy stapel + // FIXME: file-archive type + commitInfo, err := gm.GetLatestCommitInfo(ctx, c) if err != nil { return fmt.Errorf("unable to get latest commit info: %w", err) @@ -591,12 +674,10 @@ func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb if err != nil { return err } - archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts) if err != nil { return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err) } - var archiveType container_backend.ArchiveType gitArchiveType, err := gm.getArchiveType(ctx, commitInfo.Commit) @@ -618,11 +699,7 @@ func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err) } - stageImage.Builder.StapelStageBuilder().AddDataArchives(container_backend.DataArchive{ - Data: f, - Type: archiveType, - To: gm.To, - }) + stageImage.Builder.StapelStageBuilder().AddDataArchive(f, archiveType, gm.To) } gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, commitInfo) diff --git a/pkg/build/stage/git_patch.go b/pkg/build/stage/git_patch.go index 06db207572..7687d3a0d7 100644 --- a/pkg/build/stage/git_patch.go +++ b/pkg/build/stage/git_patch.go @@ -78,20 +78,17 @@ func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr contain } func (s *GitPatchStage) prepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if c.UseLegacyStapelBuilder(cr) { - for _, gitMapping := range s.gitMappings { - if err := gitMapping.ApplyPatchCommand(ctx, c, cr, prevBuiltImage, stageImage); err != nil { - return err - } + for _, gitMapping := range s.gitMappings { + if err := gitMapping.PreparePatchForImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { + return err } + } + if c.UseLegacyStapelBuilder(cr) { stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetPatchesCacheDir(), s.ContainerPatchesDir)) stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetArchivesCacheDir(), s.ContainerArchivesDir)) stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", s.ScriptsDir, s.ContainerScriptsDir)) - - return nil - } else { - // TODO(stapel-to-buildah) - panic("not implemented") } + + return nil } diff --git a/pkg/container_backend/build_stapel_stage_options.go b/pkg/container_backend/build_stapel_stage_options.go index 556d219716..1e01cd1e4a 100644 --- a/pkg/container_backend/build_stapel_stage_options.go +++ b/pkg/container_backend/build_stapel_stage_options.go @@ -21,8 +21,8 @@ type BuildStapelStageOptionsInterface interface { AddBuildVolumes(volumes ...string) BuildStapelStageOptionsInterface AddCommands(commands ...string) BuildStapelStageOptionsInterface - AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface - AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface + AddDataArchive(archive io.ReadCloser, archiveType ArchiveType, to string) BuildStapelStageOptionsInterface + RemoveData(removeType RemoveType, paths []string, keepParentDirs []string) BuildStapelStageOptionsInterface AddDependencyImport(imageName, fromPath, toPath string, includePaths, excludePaths []string, owner, group string) BuildStapelStageOptionsInterface } @@ -42,9 +42,9 @@ type BuildStapelStageOptions struct { BuildVolumes []string Commands []string - DataArchives []DataArchive - PathsToRemove []string - DependenciesImports []DependencyImport + DataArchiveSpecs []DataArchiveSpec + RemoveDataSpecs []RemoveDataSpec + DependencyImportSpecs []DependencyImportSpec } type ArchiveType int @@ -55,13 +55,28 @@ const ( DirectoryArchive ) -type DataArchive struct { - Data io.ReadCloser - Type ArchiveType - To string +type DataArchiveSpec struct { + Archive io.ReadCloser + Type ArchiveType + To string } -type DependencyImport struct { +type RemoveType int + +//go:generate stringer -type=RemoveType +const ( + RemoveExactPath RemoveType = iota + RemoveExactPathWithEmptyParentDirs + RemoveInsidePath +) + +type RemoveDataSpec struct { + Type RemoveType + Paths []string + KeepParentDirs []string +} + +type DependencyImportSpec struct { ImageName string FromPath string ToPath string @@ -140,18 +155,26 @@ func (opts *BuildStapelStageOptions) AddCommands(commands ...string) BuildStapel return opts } -func (opts *BuildStapelStageOptions) AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface { - opts.DataArchives = append(opts.DataArchives, archives...) +func (opts *BuildStapelStageOptions) AddDataArchive(archive io.ReadCloser, archiveType ArchiveType, to string) BuildStapelStageOptionsInterface { + opts.DataArchiveSpecs = append(opts.DataArchiveSpecs, DataArchiveSpec{ + Archive: archive, + Type: archiveType, + To: to, + }) return opts } -func (opts *BuildStapelStageOptions) AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface { - opts.PathsToRemove = append(opts.PathsToRemove, paths...) +func (opts *BuildStapelStageOptions) RemoveData(removeType RemoveType, paths []string, keepParentDirs []string) BuildStapelStageOptionsInterface { + opts.RemoveDataSpecs = append(opts.RemoveDataSpecs, RemoveDataSpec{ + Type: removeType, + Paths: paths, + KeepParentDirs: keepParentDirs, + }) return opts } func (opts *BuildStapelStageOptions) AddDependencyImport(imageName, fromPath, toPath string, includePaths, excludePaths []string, owner, group string) BuildStapelStageOptionsInterface { - opts.DependenciesImports = append(opts.DependenciesImports, DependencyImport{ + opts.DependencyImportSpecs = append(opts.DependencyImportSpecs, DependencyImportSpec{ ImageName: imageName, FromPath: fromPath, ToPath: toPath, diff --git a/pkg/container_backend/buildah_backend.go b/pkg/container_backend/buildah_backend.go index d251da00ae..3e00ef6674 100644 --- a/pkg/container_backend/buildah_backend.go +++ b/pkg/container_backend/buildah_backend.go @@ -121,10 +121,10 @@ func (runtime *BuildahBackend) applyCommands(ctx context.Context, container *con type dependencyContainer struct { Container *containerDesc - Import DependencyImport + Import DependencyImportSpec } -func (runtime *BuildahBackend) applyDataArchives(ctx context.Context, container *containerDesc, dataArchives []DataArchive) error { +func (runtime *BuildahBackend) applyDataArchives(ctx context.Context, container *containerDesc, dataArchives []DataArchiveSpec) error { for _, archive := range dataArchives { destPath := filepath.Join(container.RootMount, archive.To) @@ -134,27 +134,29 @@ func (runtime *BuildahBackend) applyDataArchives(ctx context.Context, container extractDestPath = destPath case FileArchive: extractDestPath = filepath.Dir(destPath) + + _, err := os.Stat(destPath) + switch { + case os.IsNotExist(err): + case err != nil: + return fmt.Errorf("unable to access container path %q: %w", destPath, err) + default: + logboek.Context(ctx).Debug().LogF("Removing archive destination path %s\n", archive.To) + if err := os.RemoveAll(destPath); err != nil { + return fmt.Errorf("unable to cleanup archive destination path %s: %w", archive.To, err) + } + } default: return fmt.Errorf("unknown archive type %q", archive.Type) } - _, err := os.Stat(destPath) - switch { - case os.IsNotExist(err): - case err != nil: - return fmt.Errorf("unable to access container path %q: %w", destPath, err) - default: - logboek.Context(ctx).Debug().LogF("Removing archive destination path %s\n", archive.To) - if err := os.RemoveAll(destPath); err != nil { - return fmt.Errorf("unable to cleanup archive destination path %s: %w", archive.To, err) - } - } + logboek.Context(ctx).Debug().LogF("Apply data archive into %q\n", archive.To) logboek.Context(ctx).Debug().LogF("Extracting archive into container path %s\n", archive.To) - if err := util.ExtractTar(archive.Data, extractDestPath); err != nil { + if err := util.ExtractTar(archive.Archive, extractDestPath); err != nil { return fmt.Errorf("unable to extract data archive into %s: %w", archive.To, err) } - if err := archive.Data.Close(); err != nil { + if err := archive.Archive.Close(); err != nil { return fmt.Errorf("error closing archive data stream: %w", err) } } @@ -162,20 +164,39 @@ func (runtime *BuildahBackend) applyDataArchives(ctx context.Context, container return nil } -func (runtime *BuildahBackend) applyPathsToRemove(ctx context.Context, container *containerDesc, pathsToRemove []string) error { - for _, path := range pathsToRemove { - destPath := filepath.Join(container.RootMount, path) - - logboek.Context(ctx).Debug().LogF("Removing container path %s\n", path) - if err := os.RemoveAll(destPath); err != nil { - return fmt.Errorf("unable to remove path %s: %w", path, err) +func (runtime *BuildahBackend) applyRemoveData(ctx context.Context, container *containerDesc, removeData []RemoveDataSpec) error { + for _, spec := range removeData { + switch spec.Type { + case RemoveExactPath: + for _, path := range spec.Paths { + destPath := filepath.Join(container.RootMount, path) + if err := removeExactPath(ctx, destPath); err != nil { + return fmt.Errorf("unable to remove %q: %w", path, err) + } + } + case RemoveExactPathWithEmptyParentDirs: + for _, path := range spec.Paths { + destPath := filepath.Join(container.RootMount, path) + if err := removeExactPathWithEmptyParentDirs(ctx, destPath, spec.KeepParentDirs); err != nil { + return fmt.Errorf("unable to remove %q: %w", path, err) + } + } + case RemoveInsidePath: + for _, path := range spec.Paths { + destPath := filepath.Join(container.RootMount, path) + if err := removeInsidePath(ctx, destPath); err != nil { + return fmt.Errorf("unable to remove %q: %w", path, err) + } + } + default: + return fmt.Errorf("unknown remove operation type %q", spec.Type) } } return nil } -func (runtime *BuildahBackend) applyDependenciesImports(ctx context.Context, container *containerDesc, dependenciesImports []DependencyImport) error { +func (runtime *BuildahBackend) applyDependenciesImports(ctx context.Context, container *containerDesc, dependenciesImports []DependencyImportSpec) error { var dependencies []*dependencyContainer var dependenciesImages []string @@ -247,7 +268,7 @@ func (runtime *BuildahBackend) BuildStapelStage(ctx context.Context, opts BuildS }() // TODO(stapel-to-buildah): cleanup orphan build containers in werf-host-cleanup procedure - if len(opts.DependenciesImports)+len(opts.DataArchives)+len(opts.PathsToRemove) > 0 { + if len(opts.DependencyImportSpecs)+len(opts.DataArchiveSpecs)+len(opts.RemoveDataSpecs) > 0 { logboek.Context(ctx).Debug().LogF("Mounting build container %s\n", container.Name) if err := runtime.mountContainers(ctx, []*containerDesc{container}); err != nil { return "", fmt.Errorf("unable to mount build container %s: %w", container.Name, err) @@ -260,18 +281,18 @@ func (runtime *BuildahBackend) BuildStapelStage(ctx context.Context, opts BuildS }() } - if len(opts.DependenciesImports) > 0 { - if err := runtime.applyDependenciesImports(ctx, container, opts.DependenciesImports); err != nil { + if len(opts.DependencyImportSpecs) > 0 { + if err := runtime.applyDependenciesImports(ctx, container, opts.DependencyImportSpecs); err != nil { return "", err } } - if len(opts.DataArchives) > 0 { - if err := runtime.applyDataArchives(ctx, container, opts.DataArchives); err != nil { + if len(opts.DataArchiveSpecs) > 0 { + if err := runtime.applyDataArchives(ctx, container, opts.DataArchiveSpecs); err != nil { return "", err } } - if len(opts.PathsToRemove) > 0 { - if err := runtime.applyPathsToRemove(ctx, container, opts.PathsToRemove); err != nil { + if len(opts.RemoveDataSpecs) > 0 { + if err := runtime.applyRemoveData(ctx, container, opts.RemoveDataSpecs); err != nil { return "", err } } diff --git a/pkg/container_backend/file_utils.go b/pkg/container_backend/file_utils.go new file mode 100644 index 0000000000..acf53d581e --- /dev/null +++ b/pkg/container_backend/file_utils.go @@ -0,0 +1,106 @@ +package container_backend + +import ( + "context" + "fmt" + "os" + "path/filepath" + + "github.com/werf/logboek" +) + +func removeExactPath(ctx context.Context, path string) error { + _, err := os.Stat(path) + switch { + case os.IsNotExist(err): + case err != nil: + return fmt.Errorf("unable to access path %q: %w", path, err) + default: + logboek.Context(ctx).Debug().LogF("Removing path %s\n", path) + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("unable to remove path %s: %w", path, err) + } + } + + return nil +} + +func removeExactPathWithEmptyParentDirs(ctx context.Context, path string, keepParentDirs []string) error { + _, err := os.Stat(path) + switch { + case os.IsNotExist(err): + case err != nil: + return fmt.Errorf("unable to access path %q: %w", path, err) + default: + logboek.Context(ctx).Debug().LogF("Removing path %s\n", path) + if err := os.RemoveAll(path); err != nil { + return fmt.Errorf("unable to remove path %s: %w", path, err) + } + } + + p := path + for { + parentDir := filepath.Dir(p) + if parentDir == p { + return nil + } + p = parentDir + + for _, keepPath := range keepParentDirs { + if keepPath == p { + return nil + } + } + + _, err := os.Stat(p) + switch { + case os.IsNotExist(err): + // This may happen when initially given input path is not exists + continue + case err != nil: + return fmt.Errorf("unable to access path %q: %w", path, err) + } + + entries, err := os.ReadDir(p) + if err != nil { + return fmt.Errorf("error reading dir %q: %w", p, err) + } + if len(entries) > 0 { + return nil + } + logboek.Context(ctx).Debug().LogF("Removing empty dir %s\n", p) + if err := os.RemoveAll(p); err != nil { + return fmt.Errorf("unable to remove empty dir %q: %w", p, err) + } + } +} + +func removeInsidePath(ctx context.Context, path string) error { + stat, err := os.Stat(path) + switch { + case os.IsNotExist(err): + return nil + case err != nil: + return fmt.Errorf("unable to access path %q: %w", path, err) + } + + if !stat.IsDir() { + return nil + } + + entries, err := os.ReadDir(path) + if err != nil { + return fmt.Errorf("error reading dir %q: %w", path, err) + } + + for _, entry := range entries { + destPath := filepath.Join(path, entry.Name()) + + logboek.Context(ctx).Debug().LogF("Removing path %s\n", destPath) + if err := os.RemoveAll(destPath); err != nil { + return fmt.Errorf("unable to remove path %q: %w", destPath, err) + } + } + + return nil +} diff --git a/pkg/container_backend/removetype_string.go b/pkg/container_backend/removetype_string.go new file mode 100644 index 0000000000..848876bf9b --- /dev/null +++ b/pkg/container_backend/removetype_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=RemoveType"; DO NOT EDIT. + +package container_backend + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[RemoveExactPath-0] + _ = x[RemoveExactPathWithEmptyParentDirs-1] + _ = x[RemoveInsidePath-2] +} + +const _RemoveType_name = "RemoveExactPathRemoveExactPathWithEmptyParentDirsRemoveInsidePath" + +var _RemoveType_index = [...]uint8{0, 15, 49, 65} + +func (i RemoveType) String() string { + if i < 0 || i >= RemoveType(len(_RemoveType_index)-1) { + return "RemoveType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _RemoveType_name[_RemoveType_index[i]:_RemoveType_index[i+1]] +} diff --git a/pkg/context_manager/context_manager.go b/pkg/context_manager/context_manager.go index 65639461db..2443998871 100644 --- a/pkg/context_manager/context_manager.go +++ b/pkg/context_manager/context_manager.go @@ -116,24 +116,27 @@ func AddContextAddFilesToContextArchive(ctx context.Context, originalArchivePath destinationArchivePath := GetTmpArchivePath() pathsToExcludeFromSourceArchive := contextAddFiles - if err := util.CreateArchiveBasedOnAnotherOne(ctx, originalArchivePath, destinationArchivePath, pathsToExcludeFromSourceArchive, func(tw *tar.Writer) error { - addFilePathsToCopy, err := GetContextAddFilesPaths(projectDir, contextDir, contextAddFiles) - if err != nil { - return err - } - - for _, addFilePathToCopy := range addFilePathsToCopy { - tarEntryName, err := filepath.Rel(filepath.Join(projectDir, contextDir), addFilePathToCopy) + if err := util.CreateArchiveBasedOnAnotherOne(ctx, originalArchivePath, destinationArchivePath, util.CreateArchiveOptions{ + CopyTarOptions: util.CopyTarOptions{ExcludePaths: pathsToExcludeFromSourceArchive}, + AfterCopyFunc: func(tw *tar.Writer) error { + addFilePathsToCopy, err := GetContextAddFilesPaths(projectDir, contextDir, contextAddFiles) if err != nil { - return fmt.Errorf("unable to get context relative path for %q: %w", addFilePathToCopy, err) + return err } - tarEntryName = filepath.ToSlash(tarEntryName) - if err := util.CopyFileIntoTar(tw, tarEntryName, addFilePathToCopy); err != nil { - return fmt.Errorf("unable to add contextAddFile %q to archive %q: %w", addFilePathToCopy, destinationArchivePath, err) + + for _, addFilePathToCopy := range addFilePathsToCopy { + tarEntryName, err := filepath.Rel(filepath.Join(projectDir, contextDir), addFilePathToCopy) + if err != nil { + return fmt.Errorf("unable to get context relative path for %q: %w", addFilePathToCopy, err) + } + tarEntryName = filepath.ToSlash(tarEntryName) + if err := util.CopyFileIntoTar(tw, tarEntryName, addFilePathToCopy); err != nil { + return fmt.Errorf("unable to add contextAddFile %q to archive %q: %w", addFilePathToCopy, destinationArchivePath, err) + } + logboek.Context(ctx).Debug().LogF("Extra file was added to the current context: %q\n", tarEntryName) } - logboek.Context(ctx).Debug().LogF("Extra file was added to the current context: %q\n", tarEntryName) - } - return nil + return nil + }, }); err != nil { return "", err } diff --git a/pkg/git_repo/git_repo.go b/pkg/git_repo/git_repo.go index 0e92b4e71a..06bdc7eb4c 100644 --- a/pkg/git_repo/git_repo.go +++ b/pkg/git_repo/git_repo.go @@ -87,6 +87,7 @@ type Patch interface { HasBinary() bool GetPaths() []string GetBinaryPaths() []string + GetPathsToRemove() []string } type Archive interface { diff --git a/pkg/git_repo/patch_file.go b/pkg/git_repo/patch_file.go index 10101f94f0..f2d95f5044 100644 --- a/pkg/git_repo/patch_file.go +++ b/pkg/git_repo/patch_file.go @@ -28,3 +28,7 @@ func (p *PatchFile) GetPaths() []string { func (p *PatchFile) GetBinaryPaths() []string { return p.Descriptor.BinaryPaths } + +func (p *PatchFile) GetPathsToRemove() []string { + return p.Descriptor.PathsToRemove +} diff --git a/pkg/true_git/diff_parser.go b/pkg/true_git/diff_parser.go index 0a0c2249bc..fdbdf9ab77 100644 --- a/pkg/true_git/diff_parser.go +++ b/pkg/true_git/diff_parser.go @@ -50,6 +50,7 @@ type diffParser struct { Paths []string BinaryPaths []string + PathsToRemove []string LastSeenPaths []string state parserState @@ -403,6 +404,7 @@ func (p *diffParser) handleDeleteFilePath(line string) error { newLine := fmt.Sprintf("--- a/%s", newPath) p.state = diffBody + p.PathsToRemove = appendUnique(p.PathsToRemove, newPath) return p.writeOutLine(newLine) } diff --git a/pkg/true_git/patch.go b/pkg/true_git/patch.go index 959a01a796..d4e8063c76 100644 --- a/pkg/true_git/patch.go +++ b/pkg/true_git/patch.go @@ -20,6 +20,8 @@ type PatchOptions struct { FromCommit, ToCommit string FileRenames map[string]string // Files to rename during patching. Git repo relative paths of original files as keys, new filenames (without base path) as values. + // TODO: maybe add --path-status option for git, so that created patch will be only presented by the descriptor without content-diff + WithEntireFileContext bool WithBinary bool } @@ -45,8 +47,9 @@ func (opts PatchOptions) ID() string { } type PatchDescriptor struct { - Paths []string - BinaryPaths []string + Paths []string + BinaryPaths []string + PathsToRemove []string } func PatchWithSubmodules(ctx context.Context, out io.Writer, gitDir, workTreeCacheDir string, opts PatchOptions) (*PatchDescriptor, error) { @@ -228,8 +231,9 @@ WaitForData: } desc := &PatchDescriptor{ - Paths: p.Paths, - BinaryPaths: p.BinaryPaths, + Paths: p.Paths, + BinaryPaths: p.BinaryPaths, + PathsToRemove: p.PathsToRemove, } if debugPatch() { diff --git a/pkg/util/archive.go b/pkg/util/archive.go index cfc0c312d3..449ab805a0 100644 --- a/pkg/util/archive.go +++ b/pkg/util/archive.go @@ -17,7 +17,12 @@ import ( "github.com/werf/logboek" ) -func CreateArchiveBasedOnAnotherOne(ctx context.Context, sourceArchivePath, destinationArchivePath string, pathsToExclude []string, f func(tw *tar.Writer) error) error { +type CreateArchiveOptions struct { + CopyTarOptions + AfterCopyFunc func(tw *tar.Writer) error +} + +func CreateArchiveBasedOnAnotherOne(ctx context.Context, sourceArchivePath, destinationArchivePath string, opts CreateArchiveOptions) error { return CreateArchive(destinationArchivePath, func(tw *tar.Writer) error { source, err := os.Open(sourceArchivePath) if err != nil { @@ -25,41 +30,15 @@ func CreateArchiveBasedOnAnotherOne(ctx context.Context, sourceArchivePath, dest } defer source.Close() - tr := tar.NewReader(source) - - ArchiveCopying: - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } else if err != nil { - return fmt.Errorf("unable to read archive %q: %w", sourceArchivePath, err) - } - - for _, pathToExclude := range pathsToExclude { - if hdr.Name == filepath.ToSlash(pathToExclude) { - if debugArchiveUtil() { - logboek.Context(ctx).Debug().LogF("Source archive file was excluded: %q\n", hdr.Name) - } - - continue ArchiveCopying - } - } - - if err := tw.WriteHeader(hdr); err != nil { - return fmt.Errorf("unable to write header %q from %q archive to %q: %w", hdr.Name, sourceArchivePath, destinationArchivePath, err) - } - - if _, err := io.Copy(tw, tr); err != nil { - return fmt.Errorf("unable to copy file %q from %q archive to %q: %w", hdr.Name, sourceArchivePath, destinationArchivePath, err) - } + if err := CopyTar(ctx, source, tw, opts.CopyTarOptions); err != nil { + return err + } - if debugArchiveUtil() { - logboek.Context(ctx).Debug().LogF("Source archive file was added: %q\n", hdr.Name) - } + if opts.AfterCopyFunc != nil { + return opts.AfterCopyFunc(tw) } - return f(tw) + return nil }) } @@ -172,6 +151,63 @@ func CopyGitIndexEntryIntoTar(tw *tar.Writer, tarEntryName string, entry *index. return nil } +type CopyTarOptions struct { + IncludePaths []string + ExcludePaths []string +} + +func CopyTar(ctx context.Context, in io.Reader, tw *tar.Writer, opts CopyTarOptions) error { + tr := tar.NewReader(in) + +ArchiveCopying: + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return fmt.Errorf("unable to read archive: %w", err) + } + + for _, excPath := range opts.ExcludePaths { + if hdr.Name == filepath.ToSlash(excPath) { + if debugArchiveUtil() { + logboek.Context(ctx).Debug().LogF("Source archive file excluded: %q\n", hdr.Name) + } + + continue ArchiveCopying + } + } + + if len(opts.IncludePaths) > 0 { + for _, incPath := range opts.IncludePaths { + if hdr.Name == filepath.ToSlash(incPath) { + if debugArchiveUtil() { + logboek.Context(ctx).Debug().LogF("Source archive file included: %q\n", hdr.Name) + } + goto CopyEntry + } + } + + continue ArchiveCopying + } + + CopyEntry: + if err := tw.WriteHeader(hdr); err != nil { + return fmt.Errorf("unable to write tar header entry %q: %w", hdr.Name, err) + } + + if _, err := io.Copy(tw, tr); err != nil { + return fmt.Errorf("unable to copy tar entry %q data: %w", hdr.Name, err) + } + + if debugArchiveUtil() { + logboek.Context(ctx).Debug().LogF("Source archive file was added: %q\n", hdr.Name) + } + } + + return nil +} + func ExtractTar(tarFileReader io.Reader, dstDir string) error { tarReader := tar.NewReader(tarFileReader) for {