diff --git a/pkg/build/stage/git.go b/pkg/build/stage/git.go index 82d6702ba7..25af560725 100644 --- a/pkg/build/stage/git.go +++ b/pkg/build/stage/git.go @@ -30,14 +30,14 @@ func (s *GitStage) PrepareImage(ctx context.Context, c Conveyor, cr container_ba return err } - if c.UseLegacyStapelBuilder(cr) { - if c.GiterminismManager().Dev() { - stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(map[string]string{imagePkg.WerfDevLabel: "true"}) + if c.GiterminismManager().Dev() { + addLabels := map[string]string{imagePkg.WerfDevLabel: "true"} + if c.UseLegacyStapelBuilder(cr) { + stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(addLabels) + } else { + stageImage.Builder.StapelStageBuilder().AddLabels(addLabels) } - - return nil - } else { - // TODO(stapel-to-buildah) - panic("not implemented") } + + return nil } diff --git a/pkg/build/stage/git_archive.go b/pkg/build/stage/git_archive.go index d0a73cad21..9915133f92 100644 --- a/pkg/build/stage/git_archive.go +++ b/pkg/build/stage/git_archive.go @@ -69,25 +69,26 @@ func (s *GitArchiveStage) GetNextStageDependencies(ctx context.Context, c Convey } func (s *GitArchiveStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { + if !c.UseLegacyStapelBuilder(cr) { + stageImage.Builder.StapelStageBuilder().SetStageType(container_backend.DataArchivesStage) + } + if err := s.GitStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { return err } - if c.UseLegacyStapelBuilder(cr) { - for _, gitMapping := range s.gitMappings { - if err := gitMapping.ApplyArchiveCommand(ctx, c, stageImage); err != nil { - return err - } + for _, gitMapping := range s.gitMappings { + if err := gitMapping.PrepareArchiveForImage(ctx, c, cr, stageImage); err != nil { + return fmt.Errorf("unable to prepare git mapping %s for image stage: %w", gitMapping.Name, err) } + } + if c.UseLegacyStapelBuilder(cr) { stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetArchivesCacheDir(), s.ContainerArchivesDir)) stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", s.ScriptsDir, s.ContainerScriptsDir)) - - return nil - } else { - // TODO(stapel-to-buildah) - panic("not implemented") } + + return nil } func (s *GitArchiveStage) IsEmpty(ctx context.Context, c Conveyor, stageImage *StageImage) (bool, error) { diff --git a/pkg/build/stage/git_mapping.go b/pkg/build/stage/git_mapping.go index 11367fc3d8..4068a611da 100644 --- a/pkg/build/stage/git_mapping.go +++ b/pkg/build/stage/git_mapping.go @@ -14,6 +14,7 @@ import ( "sync" "github.com/werf/logboek" + "github.com/werf/werf/pkg/container_backend" "github.com/werf/werf/pkg/git_repo" "github.com/werf/werf/pkg/path_matcher" "github.com/werf/werf/pkg/stapel" @@ -237,7 +238,7 @@ func (gm *GitMapping) applyPatchCommand(patchFile *ContainerFileDescriptor, arch return commands, nil } -func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, prevBuiltImage, stageImage *StageImage) error { +func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { fromCommit, err := gm.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage) if err != nil { return fmt.Errorf("unable to get base commit from built image: %w", err) @@ -257,7 +258,7 @@ func (gm *GitMapping) ApplyPatchCommand(ctx context.Context, c Conveyor, prevBui return err } - gm.AddGitCommitToImageLabels(stageImage, toCommitInfo) + gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, toCommitInfo) return nil } @@ -296,21 +297,24 @@ func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (Imag return res, nil } -func (gm *GitMapping) AddGitCommitToImageLabels(stageImage *StageImage, commitInfo ImageCommitInfo) { - stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{ - gm.ImageGitCommitLabel(): commitInfo.Commit, - }) +func (gm *GitMapping) AddGitCommitToImageLabels(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage, commitInfo ImageCommitInfo) { + addLabels := make(map[string]string) + addLabels[gm.ImageGitCommitLabel()] = commitInfo.Commit if commitInfo.VirtualMerge { - stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{ - gm.VirtualMergeLabel(): "true", - gm.VirtualMergeFromCommitLabel(): commitInfo.VirtualMergeFromCommit, - gm.VirtualMergeIntoCommitLabel(): commitInfo.VirtualMergeIntoCommit, - }) + addLabels[gm.VirtualMergeLabel()] = "true" + addLabels[gm.VirtualMergeFromCommitLabel()] = commitInfo.VirtualMergeFromCommit + addLabels[gm.VirtualMergeIntoCommitLabel()] = commitInfo.VirtualMergeIntoCommit } else { - stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(map[string]string{ - gm.VirtualMergeLabel(): "false", - }) + addLabels[gm.VirtualMergeLabel()] = "false" + } + + if len(addLabels) > 0 { + if c.UseLegacyStapelBuilder(cb) { + stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(addLabels) + } else { + stageImage.Builder.StapelStageBuilder().AddLabels(addLabels) + } } } @@ -567,22 +571,61 @@ func (gm *GitMapping) applyArchiveCommand(archiveFile *ContainerFileDescriptor, return commands, nil } -func (gm *GitMapping) ApplyArchiveCommand(ctx context.Context, c Conveyor, stageImage *StageImage) error { +func (gm *GitMapping) PrepareArchiveForImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, stageImage *StageImage) error { commitInfo, err := gm.GetLatestCommitInfo(ctx, c) if err != nil { return fmt.Errorf("unable to get latest commit info: %w", err) } - commands, err := gm.baseApplyArchiveCommand(ctx, commitInfo.Commit, stageImage) - if err != nil { - return err - } + if c.UseLegacyStapelBuilder(cb) { + commands, err := gm.baseApplyArchiveCommand(ctx, commitInfo.Commit, stageImage) + if err != nil { + return err + } - if err := gm.applyScript(stageImage, commands); err != nil { - return err + if err := gm.applyScript(stageImage, commands); err != nil { + return err + } + } else { + archiveOpts, err := gm.makeArchiveOptions(ctx, commitInfo.Commit) + if err != nil { + return err + } + + archive, err := gm.GitRepo().GetOrCreateArchive(ctx, *archiveOpts) + if err != nil { + return fmt.Errorf("unable to create git archive for commit %s with path scope %s: %w", archiveOpts.Commit, archiveOpts.PathScope, err) + } + + var archiveType container_backend.ArchiveType + + gitArchiveType, err := gm.getArchiveType(ctx, commitInfo.Commit) + if err != nil { + return fmt.Errorf("unable to determine git archive type: %w", err) + } + + stageImage.Builder.StapelStageBuilder().AddLabels(map[string]string{gm.getArchiveTypeLabelName(): string(gitArchiveType)}) + + switch gitArchiveType { + case git_repo.FileArchive: + archiveType = container_backend.FileArchive + case git_repo.DirectoryArchive: + archiveType = container_backend.DirectoryArchive + } + + f, err := os.Open(archive.GetFilePath()) + if err != nil { + return fmt.Errorf("unable to open archive file %q: %w", archive.GetFilePath(), err) + } + + stageImage.Builder.StapelStageBuilder().AddDataArchives(container_backend.DataArchive{ + Data: f, + Type: archiveType, + To: gm.To, + }) } - gm.AddGitCommitToImageLabels(stageImage, commitInfo) + gm.AddGitCommitToImageLabels(ctx, c, cb, stageImage, commitInfo) return nil } diff --git a/pkg/build/stage/git_patch.go b/pkg/build/stage/git_patch.go index 9d7a458286..06db207572 100644 --- a/pkg/build/stage/git_patch.go +++ b/pkg/build/stage/git_patch.go @@ -80,7 +80,7 @@ func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr contain func (s *GitPatchStage) prepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { if c.UseLegacyStapelBuilder(cr) { for _, gitMapping := range s.gitMappings { - if err := gitMapping.ApplyPatchCommand(ctx, c, prevBuiltImage, stageImage); err != nil { + if err := gitMapping.ApplyPatchCommand(ctx, c, cr, prevBuiltImage, stageImage); err != nil { return err } } diff --git a/pkg/container_backend/buildah_backend.go b/pkg/container_backend/buildah_backend.go index 13f06c1643..d508201937 100644 --- a/pkg/container_backend/buildah_backend.go +++ b/pkg/container_backend/buildah_backend.go @@ -14,6 +14,7 @@ import ( "github.com/werf/logboek" "github.com/werf/werf/pkg/buildah" "github.com/werf/werf/pkg/image" + "github.com/werf/werf/pkg/util" ) type BuildahBackend struct { @@ -148,6 +149,64 @@ type dependencyContainer struct { Import DependencyImport } +func (runtime *BuildahBackend) applyDataArchives(ctx context.Context, container *containerDesc, opts BuildStapelStageOptions) error { + logboek.Context(ctx).Debug().LogF("Mounting container %q\n", container.Name) + if err := runtime.mountContainers(ctx, []*containerDesc{container}); err != nil { + return fmt.Errorf("unable to mount containers: %w", err) + } + defer func() { + logboek.Context(ctx).Debug().LogF("Unmounting container %q\n", container.Name) + if err := runtime.unmountContainers(ctx, []*containerDesc{container}); err != nil { + logboek.Context(ctx).Error().LogF("ERROR: unable to unmount containers: %s\n", err) + } + }() + + for _, archive := range opts.DataArchives { + destPath := filepath.Join(container.RootMount, archive.To) + + var extractDestPath string + switch archive.Type { + case DirectoryArchive: + extractDestPath = destPath + case FileArchive: + extractDestPath = filepath.Dir(destPath) + default: + return fmt.Errorf("unknown archive type %q", archive.Type) + } + + _, err := os.Stat(destPath) + switch { + case os.IsNotExist(err): + case err != nil: + return fmt.Errorf("unable to access container path %q: %w", destPath, err) + default: + logboek.Context(ctx).Debug().LogF("Removing archive destination path %s\n", archive.To) + if err := os.RemoveAll(destPath); err != nil { + return fmt.Errorf("unable to cleanup archive destination path %s: %w", archive.To, err) + } + } + + logboek.Context(ctx).Debug().LogF("Extracting archive into container path %s\n", archive.To) + if err := util.ExtractTar(archive.Data, extractDestPath); err != nil { + return fmt.Errorf("unable to extract data archive into %s: %w", archive.To, err) + } + if err := archive.Data.Close(); err != nil { + return fmt.Errorf("error closing archive data stream: %w", err) + } + } + + for _, path := range opts.PathsToRemove { + destPath := filepath.Join(container.RootMount, path) + + logboek.Context(ctx).Debug().LogF("Removing container path %s\n", path) + if err := os.RemoveAll(destPath); err != nil { + return fmt.Errorf("unable to remove path %s: %w", path, err) + } + } + + return nil +} + func (runtime *BuildahBackend) applyDependencies(ctx context.Context, container *containerDesc, opts BuildStapelStageOptions) error { var dependencies []*dependencyContainer @@ -181,12 +240,12 @@ func (runtime *BuildahBackend) applyDependencies(ctx context.Context, container } } - logboek.Context(ctx).Debug().LogF("Mounting dependencies containers %v\n", dependenciesContainers) + logboek.Context(ctx).Debug().LogF("Mounting containers %v\n", append(dependenciesContainers, container)) if err := runtime.mountContainers(ctx, append(dependenciesContainers, container)); err != nil { return fmt.Errorf("unable to mount containers: %w", err) } defer func() { - logboek.Context(ctx).Debug().LogF("Unmounting dependencies containers %v\n", dependenciesContainers) + logboek.Context(ctx).Debug().LogF("Unmounting containers %v\n", append(dependenciesContainers, container)) if err := runtime.unmountContainers(ctx, append(dependenciesContainers, container)); err != nil { logboek.Context(ctx).Error().LogF("ERROR: unable to unmount containers: %s\n", err) } @@ -236,6 +295,10 @@ func (runtime *BuildahBackend) BuildStapelStage(ctx context.Context, stageType S if err := runtime.applyDependencies(ctx, container, opts); err != nil { return "", err } + case DataArchivesStage: + if err := runtime.applyDataArchives(ctx, container, opts); err != nil { + return "", err + } default: return "", fmt.Errorf("unsupported stage type %q", stageType.String()) } diff --git a/pkg/container_backend/stapelstagetype.go b/pkg/container_backend/stapelstagetype.go index d5a98ed317..03ed79e5bb 100644 --- a/pkg/container_backend/stapelstagetype.go +++ b/pkg/container_backend/stapelstagetype.go @@ -1,6 +1,9 @@ package container_backend -import "fmt" +import ( + "fmt" + "io" +) type StapelStageType int @@ -10,6 +13,7 @@ const ( UserCommandsStage DockerInstructionsStage DependenciesStage + DataArchivesStage ) type BuildStapelStageOptionsInterface interface { @@ -25,6 +29,9 @@ type BuildStapelStageOptionsInterface interface { SetWorkdir(workdir string) BuildStapelStageOptionsInterface SetHealthcheck(healthcheck string) BuildStapelStageOptionsInterface + AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface + AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface + UserCommandsStage() UserCommandsStageOptionsInterface DependenciesStage() DependenciesStageOptionsInterface } @@ -50,10 +57,27 @@ type BuildStapelStageOptions struct { Workdir string Healthcheck string + DataArchives []DataArchive + PathsToRemove []string + UserCommandsStageOptions DependenciesStageOptions } +type ArchiveType int + +//go:generate stringer -type=ArchiveType +const ( + FileArchive ArchiveType = iota + DirectoryArchive +) + +type DataArchive struct { + Data io.ReadCloser + Type ArchiveType + To string +} + func (opts *BuildStapelStageOptions) SetBaseImage(baseImage string) BuildStapelStageOptionsInterface { opts.BaseImage = baseImage return opts @@ -118,6 +142,16 @@ func (opts *BuildStapelStageOptions) AddBuildVolumes(volumes ...string) BuildSta return opts } +func (opts *BuildStapelStageOptions) AddDataArchives(archives ...DataArchive) BuildStapelStageOptionsInterface { + opts.DataArchives = append(opts.DataArchives, archives...) + return opts +} + +func (opts *BuildStapelStageOptions) AddPathsToRemove(paths ...string) BuildStapelStageOptionsInterface { + opts.PathsToRemove = append(opts.PathsToRemove, paths...) + return opts +} + func (opts *BuildStapelStageOptions) UserCommandsStage() UserCommandsStageOptionsInterface { return &opts.UserCommandsStageOptions }