From 9d72ad1edbf4d365987e8fdcc4d8b5122dbb51fd Mon Sep 17 00:00:00 2001 From: Ilya Lesikov Date: Mon, 17 Oct 2022 18:34:40 +0300 Subject: [PATCH] feat(staged-dockerfile): make and extract context archive only once Signed-off-by: Ilya Lesikov --- pkg/build/base_phase.go | 4 +- pkg/build/build_phase.go | 25 ++- pkg/build/conveyor.go | 6 +- pkg/build/image/build_context_archive.go | 163 ++++++++++++++++++ pkg/build/image/image.go | 10 ++ pkg/build/phase.go | 2 +- pkg/build/stage/base.go | 18 +- pkg/build/stage/before_install.go | 8 +- pkg/build/stage/before_setup.go | 8 +- pkg/build/stage/dependencies.go | 10 +- pkg/build/stage/dependencies_test.go | 4 +- pkg/build/stage/from.go | 20 +-- pkg/build/stage/full_dockerfile.go | 50 +----- pkg/build/stage/full_dockerfile_test.go | 8 +- pkg/build/stage/git.go | 6 +- pkg/build/stage/git_archive.go | 10 +- pkg/build/stage/git_cache.go | 2 +- pkg/build/stage/git_latest_patch.go | 2 +- pkg/build/stage/git_patch.go | 6 +- pkg/build/stage/install.go | 8 +- pkg/build/stage/instruction/run.go | 5 +- pkg/build/stage/interface.go | 6 +- pkg/build/stage/setup.go | 8 +- pkg/build/stage/stapel_docker_instructions.go | 12 +- pkg/build/stage/user_with_git_patch.go | 6 +- pkg/container_backend/archive_interface.go | 16 ++ .../build_context/build_context.go | 54 ------ pkg/container_backend/buildah_backend.go | 39 ++--- .../docker_server_backend.go | 16 +- pkg/container_backend/instruction.go | 3 +- pkg/container_backend/instruction/add.go | 10 +- pkg/container_backend/instruction/cmd.go | 4 +- pkg/container_backend/instruction/copy.go | 10 +- .../instruction/entrypoint.go | 4 +- pkg/container_backend/instruction/env.go | 4 +- pkg/container_backend/instruction/expose.go | 4 +- pkg/container_backend/instruction/label.go | 4 +- pkg/container_backend/instruction/on_build.go | 4 +- pkg/container_backend/instruction/run.go | 5 +- pkg/container_backend/instruction/shell.go | 4 +- .../instruction/stop_signal.go | 4 +- pkg/container_backend/instruction/user.go | 4 +- pkg/container_backend/instruction/volume.go | 4 +- pkg/container_backend/instruction/workdir.go | 4 +- pkg/container_backend/interface.go | 9 +- .../perf_check_container_backend.go | 5 +- .../stage_builder/dockerfile_builder.go | 21 +-- .../stage_builder/dockerfile_stage_builder.go | 20 +-- 48 files changed, 379 insertions(+), 280 deletions(-) create mode 100644 pkg/build/image/build_context_archive.go create mode 100644 pkg/container_backend/archive_interface.go delete mode 100644 pkg/container_backend/build_context/build_context.go diff --git a/pkg/build/base_phase.go b/pkg/build/base_phase.go index 4f7a080984..5b84bbc669 100644 --- a/pkg/build/base_phase.go +++ b/pkg/build/base_phase.go @@ -19,8 +19,8 @@ func (phase *BasePhase) AfterImages(_ context.Context) error { return nil } -func (phase *BasePhase) BeforeImageStages(_ context.Context, _ *image.Image) error { - return nil +func (phase *BasePhase) BeforeImageStages(ctx context.Context, img *image.Image) (deferFn func(), err error) { + return nil, nil } func (phase *BasePhase) OnImageStage(_ context.Context, _ *image.Image, _ stage.Interface) error { diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index bb0997ac63..2148f0cbcc 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -84,6 +84,8 @@ type BuildPhase struct { ShouldAddManagedImageRecord bool ImagesReport *ImagesReport + + buildContextArchive container_backend.BuildContextArchiver } const ( @@ -220,12 +222,27 @@ func (phase *BuildPhase) ImageProcessingShouldBeStopped(_ context.Context, _ *im return false } -func (phase *BuildPhase) BeforeImageStages(_ context.Context, img *image.Image) error { +func (phase *BuildPhase) BeforeImageStages(ctx context.Context, img *image.Image) (deferFn func(), err error) { phase.StagesIterator = NewStagesIterator(phase.Conveyor) img.SetupBaseImage() - return nil + if img.UsesBuildContext() { + phase.buildContextArchive = image.NewBuildContextArchive(phase.Conveyor.giterminismManager, img.TmpDir) + if err := phase.buildContextArchive.Create(ctx, container_backend.BuildContextArchiveCreateOptions{ + DockerfileRelToContextPath: img.DockerfileImageConfig.Dockerfile, + ContextGitSubDir: img.DockerfileImageConfig.Context, + ContextAddFiles: img.DockerfileImageConfig.ContextAddFiles, + }); err != nil { + return nil, fmt.Errorf("unable to create build context archive: %w", err) + } + + deferFn = func() { + phase.buildContextArchive.CleanupExtractedDir(ctx) + } + } + + return deferFn, nil } func (phase *BuildPhase) AfterImageStages(ctx context.Context, img *image.Image) error { @@ -561,7 +578,7 @@ func (phase *BuildPhase) fetchBaseImageForStage(ctx context.Context, img *image. func (phase *BuildPhase) calculateStage(ctx context.Context, img *image.Image, stg stage.Interface) (bool, func(), error) { // FIXME(stapel-to-buildah): store StageImage-s everywhere in stage and build pkgs - stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg)) + stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg), phase.buildContextArchive) if err != nil { return false, nil, err } @@ -681,7 +698,7 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *imag stageImage.Builder.DockerfileStageBuilder().AppendPostInstruction(backend_instruction.NewLabel(*dockerfile_instruction.NewLabel(serviceLabels))) } - err := stg.PrepareImage(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevBuiltImage(img, stg), stageImage) + err := stg.PrepareImage(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevBuiltImage(img, stg), stageImage, phase.buildContextArchive) if err != nil { return fmt.Errorf("error preparing stage %s: %w", stg.Name(), err) } diff --git a/pkg/build/conveyor.go b/pkg/build/conveyor.go index 40be142b9c..b70fedfab5 100644 --- a/pkg/build/conveyor.go +++ b/pkg/build/conveyor.go @@ -569,7 +569,11 @@ func (c *Conveyor) doImage(ctx context.Context, img *image.Image, phases []Phase for _, phase := range phases { logProcess := logboek.Context(ctx).Debug().LogProcess("Phase %s -- BeforeImageStages()", phase.Name()) logProcess.Start() - if err := phase.BeforeImageStages(ctx, img); err != nil { + deferFn, err := phase.BeforeImageStages(ctx, img) + if deferFn != nil { + defer deferFn() + } + if err != nil { logProcess.Fail() return fmt.Errorf("phase %s before image %s stages handler failed: %w", phase.Name(), img.GetLogName(), err) } diff --git a/pkg/build/image/build_context_archive.go b/pkg/build/image/build_context_archive.go new file mode 100644 index 0000000000..d47b35419f --- /dev/null +++ b/pkg/build/image/build_context_archive.go @@ -0,0 +1,163 @@ +package image + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "os" + "path/filepath" + + "github.com/docker/docker/builder/dockerignore" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/container_backend" + "github.com/werf/werf/pkg/context_manager" + "github.com/werf/werf/pkg/git_repo" + "github.com/werf/werf/pkg/giterminism_manager" + "github.com/werf/werf/pkg/path_matcher" + "github.com/werf/werf/pkg/util" +) + +func NewBuildContextArchive(giterminismMgr giterminism_manager.Interface, extractionRootTmpDir string) *BuildContextArchive { + return &BuildContextArchive{ + giterminismMgr: giterminismMgr, + extractionRootTmpDir: extractionRootTmpDir, + } +} + +type BuildContextArchive struct { + giterminismMgr giterminism_manager.Interface + path string + extractionRootTmpDir string + extractionDir string +} + +func (a *BuildContextArchive) Create(ctx context.Context, opts container_backend.BuildContextArchiveCreateOptions) error { + contextPathRelativeToGitWorkTree := filepath.Join(a.giterminismMgr.RelativeToGitProjectDir(), opts.ContextGitSubDir) + + pathMatcher := path_matcher.NewPathMatcher(path_matcher.PathMatcherOptions{BasePath: contextPathRelativeToGitWorkTree}) + if dockerIgnorePathMatcher, err := createDockerIgnorePathMatcher(ctx, a.giterminismMgr, opts.ContextGitSubDir, opts.DockerfileRelToContextPath); err != nil { + return fmt.Errorf("unable to create dockerignore path matcher: %w", err) + } else if dockerIgnorePathMatcher != nil { + pathMatcher = path_matcher.NewMultiPathMatcher(pathMatcher, dockerIgnorePathMatcher) + } + + archive, err := a.giterminismMgr.LocalGitRepo().GetOrCreateArchive(ctx, git_repo.ArchiveOptions{ + PathScope: contextPathRelativeToGitWorkTree, + PathMatcher: pathMatcher, + Commit: a.giterminismMgr.HeadCommit(), + }) + if err != nil { + return fmt.Errorf("unable to get or create archive: %w", err) + } + + a.path = archive.GetFilePath() + + if len(opts.ContextAddFiles) > 0 { + if err := logboek.Context(ctx).Debug().LogProcess("Add contextAddFiles to build context archive %s", a.path).DoError(func() error { + a.path, err = context_manager.AddContextAddFilesToContextArchive(ctx, a.path, a.giterminismMgr.ProjectDir(), opts.ContextGitSubDir, opts.ContextAddFiles) + return err + }); err != nil { + return fmt.Errorf("unable to add contextAddFiles to build context archive %s: %w", a.path, err) + } + } + + return nil +} + +func (a *BuildContextArchive) Path() string { + return a.path +} + +func (a *BuildContextArchive) ExtractOrGetExtractedDir(ctx context.Context) (string, error) { + if a.extractionDir != "" { + return a.extractionDir, nil + } + + if err := os.MkdirAll(a.extractionRootTmpDir, os.ModePerm); err != nil { + return "", fmt.Errorf("unable to create extraction root tmp dir %q: %w", a.extractionRootTmpDir, err) + } + + var err error + a.extractionDir, err = ioutil.TempDir(a.extractionRootTmpDir, "context") + if err != nil { + return "", fmt.Errorf("unable to create context tmp dir: %w", err) + } + + archiveReader, err := os.Open(a.path) + if err != nil { + return "", fmt.Errorf("unable to open context archive %q: %w", a.path, err) + } + defer archiveReader.Close() + + if err := util.ExtractTar(archiveReader, a.extractionDir, util.ExtractTarOptions{}); err != nil { + return "", fmt.Errorf("unable to extract context tar to tmp context dir %q: %w", a.extractionDir, err) + } + + return a.extractionDir, nil +} + +func (a *BuildContextArchive) CleanupExtractedDir(ctx context.Context) { + if a.extractionDir == "" { + return + } + + if err := os.RemoveAll(a.extractionDir); err != nil { + logboek.Context(ctx).Warn().LogF("WARNING: unable to remove extracted context dir %q: %s", a.extractionDir, err) + } +} + +// Might return nil. +func createDockerIgnorePathMatcher(ctx context.Context, giterminismMgr giterminism_manager.Interface, contextGitSubDir, dockerfileRelToContextPath string) (path_matcher.PathMatcher, error) { + dockerfileRelToGitPath := filepath.Join(contextGitSubDir, dockerfileRelToContextPath) + + var dockerIgnorePatterns []string + for _, dockerIgnoreRelToContextPath := range []string{ + dockerfileRelToContextPath + ".dockerignore", + ".dockerignore", + } { + dockerIgnoreRelToGitPath := filepath.Join(contextGitSubDir, dockerIgnoreRelToContextPath) + if exist, err := giterminismMgr.FileReader().IsDockerignoreExistAnywhere(ctx, dockerIgnoreRelToGitPath); err != nil { + return nil, err + } else if !exist { + continue + } + + dockerIgnore, err := giterminismMgr.FileReader().ReadDockerignore(ctx, dockerIgnoreRelToGitPath) + if err != nil { + return nil, err + } + + r := bytes.NewReader(dockerIgnore) + dockerIgnorePatterns, err = dockerignore.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("unable to read %q file: %w", dockerIgnoreRelToContextPath, err) + } + + break + } + + if dockerIgnorePatterns == nil { + return nil, nil + } + + dockerIgnorePathMatcher := path_matcher.NewPathMatcher(path_matcher.PathMatcherOptions{ + BasePath: filepath.Join(giterminismMgr.RelativeToGitProjectDir(), contextGitSubDir), + DockerignorePatterns: dockerIgnorePatterns, + }) + + if !dockerIgnorePathMatcher.IsPathMatched(dockerfileRelToGitPath) { + logboek.Context(ctx).Warn().LogLn("WARNING: There is no way to ignore the Dockerfile due to docker limitation when building an image for a compressed context that reads from STDIN.") + logboek.Context(ctx).Warn().LogF("WARNING: To hide this message, remove the Dockerfile ignore rule or add an exception rule.\n") + + exceptionRule := "!" + dockerfileRelToContextPath + dockerIgnorePatterns = append(dockerIgnorePatterns, exceptionRule) + dockerIgnorePathMatcher = path_matcher.NewPathMatcher(path_matcher.PathMatcherOptions{ + BasePath: filepath.Join(giterminismMgr.RelativeToGitProjectDir(), contextGitSubDir), + DockerignorePatterns: dockerIgnorePatterns, + }) + } + + return dockerIgnorePathMatcher, nil +} diff --git a/pkg/build/image/image.go b/pkg/build/image/image.go index bfd594a695..9fda4a742a 100644 --- a/pkg/build/image/image.go +++ b/pkg/build/image/image.go @@ -171,6 +171,16 @@ func (i *Image) GetStageID() string { return i.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.Tag } +func (i *Image) UsesBuildContext() bool { + for _, stg := range i.GetStages() { + if stg.UsesBuildContext() { + return true + } + } + + return false +} + func (i *Image) GetName() string { return i.Name } diff --git a/pkg/build/phase.go b/pkg/build/phase.go index 30dcc161be..1b924215f8 100644 --- a/pkg/build/phase.go +++ b/pkg/build/phase.go @@ -11,7 +11,7 @@ type Phase interface { Name() string BeforeImages(ctx context.Context) error AfterImages(ctx context.Context) error - BeforeImageStages(ctx context.Context, img *image.Image) error + BeforeImageStages(ctx context.Context, img *image.Image) (deferFn func(), err error) OnImageStage(ctx context.Context, img *image.Image, stg stage.Interface) error AfterImageStages(ctx context.Context, img *image.Image) error ImageProcessingShouldBeStopped(ctx context.Context, img *image.Image) bool diff --git a/pkg/build/stage/base.go b/pkg/build/stage/base.go index dd1504e8d5..f9c2eb17ce 100644 --- a/pkg/build/stage/base.go +++ b/pkg/build/stage/base.go @@ -134,7 +134,7 @@ func (s *BaseStage) FetchDependencies(_ context.Context, _ Conveyor, _ container return nil } -func (s *BaseStage) GetDependencies(_ context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *BaseStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { panic("method must be implemented!") } @@ -246,28 +246,28 @@ func (s *BaseStage) SelectSuitableStage(_ context.Context, c Conveyor, stages [] return s.selectStageByOldestCreationTimestamp(stages) } -func (s *BaseStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { +func (s *BaseStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { /* * NOTE: BaseStage.PrepareImage does not called in From.PrepareImage. * NOTE: Take into account when adding new base PrepareImage steps. */ addLabels := map[string]string{imagePkg.WerfProjectRepoCommitLabel: c.GiterminismManager().HeadCommit()} - if c.UseLegacyStapelBuilder(cr) { + if c.UseLegacyStapelBuilder(cb) { stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(addLabels) } else { stageImage.Builder.StapelStageBuilder().AddLabels(addLabels) } serviceMounts := s.getServiceMounts(prevBuiltImage) - s.addServiceMountsLabels(serviceMounts, c, cr, stageImage) - if err := s.addServiceMountsVolumes(serviceMounts, c, cr, stageImage, false); err != nil { + s.addServiceMountsLabels(serviceMounts, c, cb, stageImage) + if err := s.addServiceMountsVolumes(serviceMounts, c, cb, stageImage, false); err != nil { return fmt.Errorf("error adding mounts volumes: %w", err) } customMounts := s.getCustomMounts(prevBuiltImage) - s.addCustomMountLabels(customMounts, c, cr, stageImage) - if err := s.addCustomMountVolumes(customMounts, c, cr, stageImage, false); err != nil { + s.addCustomMountLabels(customMounts, c, cb, stageImage) + if err := s.addCustomMountVolumes(customMounts, c, cb, stageImage, false); err != nil { return fmt.Errorf("error adding mounts volumes: %w", err) } @@ -502,6 +502,10 @@ func (s *BaseStage) GetGitMappings() []*GitMapping { return s.gitMappings } +func (s *BaseStage) UsesBuildContext() bool { + return false +} + func mergeMounts(a, b map[string][]string) map[string][]string { res := map[string][]string{} diff --git a/pkg/build/stage/before_install.go b/pkg/build/stage/before_install.go index 34ced233dd..31e09f1589 100644 --- a/pkg/build/stage/before_install.go +++ b/pkg/build/stage/before_install.go @@ -27,16 +27,16 @@ type BeforeInstallStage struct { *UserStage } -func (s *BeforeInstallStage) GetDependencies(ctx context.Context, _ Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *BeforeInstallStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { return s.builder.BeforeInstallChecksum(ctx), nil } -func (s *BeforeInstallStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if err := s.BaseStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { +func (s *BeforeInstallStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.BaseStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } - if err := s.builder.BeforeInstall(ctx, cr, stageImage.Builder, c.UseLegacyStapelBuilder(cr)); err != nil { + if err := s.builder.BeforeInstall(ctx, cb, stageImage.Builder, c.UseLegacyStapelBuilder(cb)); err != nil { return err } diff --git a/pkg/build/stage/before_setup.go b/pkg/build/stage/before_setup.go index 8739e21077..4fadc42be4 100644 --- a/pkg/build/stage/before_setup.go +++ b/pkg/build/stage/before_setup.go @@ -28,7 +28,7 @@ type BeforeSetupStage struct { *UserWithGitPatchStage } -func (s *BeforeSetupStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *BeforeSetupStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, BeforeSetup) if err != nil { return "", err @@ -37,12 +37,12 @@ func (s *BeforeSetupStage) GetDependencies(ctx context.Context, c Conveyor, _ co return util.Sha256Hash(s.builder.BeforeSetupChecksum(ctx), stageDependenciesChecksum), nil } -func (s *BeforeSetupStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if err := s.UserWithGitPatchStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { +func (s *BeforeSetupStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.UserWithGitPatchStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } - if err := s.builder.BeforeSetup(ctx, cr, stageImage.Builder, c.UseLegacyStapelBuilder(cr)); err != nil { + if err := s.builder.BeforeSetup(ctx, cb, stageImage.Builder, c.UseLegacyStapelBuilder(cb)); err != nil { return err } diff --git a/pkg/build/stage/dependencies.go b/pkg/build/stage/dependencies.go index 49dbff76a8..a4439406ed 100644 --- a/pkg/build/stage/dependencies.go +++ b/pkg/build/stage/dependencies.go @@ -67,7 +67,7 @@ type DependenciesStage struct { dependencies []*config.Dependency } -func (s *DependenciesStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *DependenciesStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { var args []string for ind, elm := range s.imports { @@ -209,11 +209,11 @@ func (s *DependenciesStage) prepareImage(ctx context.Context, c Conveyor, cr con return nil } -func (s *DependenciesStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevImage, stageImage *StageImage) error { - if c.UseLegacyStapelBuilder(cr) { - return s.prepareImageWithLegacyStapelBuilder(ctx, c, cr, prevImage, stageImage) +func (s *DependenciesStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if c.UseLegacyStapelBuilder(cb) { + return s.prepareImageWithLegacyStapelBuilder(ctx, c, cb, prevBuiltImage, stageImage) } else { - return s.prepareImage(ctx, c, cr, prevImage, stageImage) + return s.prepareImage(ctx, c, cb, prevBuiltImage, stageImage) } } diff --git a/pkg/build/stage/dependencies_test.go b/pkg/build/stage/dependencies_test.go index 45bc821683..8d11821979 100644 --- a/pkg/build/stage/dependencies_test.go +++ b/pkg/build/stage/dependencies_test.go @@ -30,11 +30,11 @@ var _ = Describe("DependenciesStage", func() { Builder: stageBuilder, } - digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage) + digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage, nil) Expect(err).To(Succeed()) Expect(digest).To(Equal(data.ExpectedDigest)) - err = stage.PrepareImage(ctx, conveyor, containerBackend, nil, stageImage) + err = stage.PrepareImage(ctx, conveyor, containerBackend, nil, stageImage, nil) Expect(err).To(Succeed()) CheckImageDependenciesAfterPrepare(img, stageBuilder, data.Dependencies) }, diff --git a/pkg/build/stage/from.go b/pkg/build/stage/from.go index affba7dc4e..aca38fec50 100644 --- a/pkg/build/stage/from.go +++ b/pkg/build/stage/from.go @@ -51,7 +51,7 @@ func (s *FromStage) HasPrevStage() bool { return false } -func (s *FromStage) GetDependencies(_ context.Context, c Conveyor, _ container_backend.ContainerBackend, prevImage, _ *StageImage) (string, error) { +func (s *FromStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { var args []string if s.cacheVersion != "" { @@ -75,26 +75,26 @@ func (s *FromStage) GetDependencies(_ context.Context, c Conveyor, _ container_b return util.Sha256Hash(args...), nil } -func (s *FromStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { +func (s *FromStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { addLabels := map[string]string{imagePkg.WerfProjectRepoCommitLabel: c.GiterminismManager().HeadCommit()} - if c.UseLegacyStapelBuilder(cr) { + if c.UseLegacyStapelBuilder(cb) { stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(addLabels) } else { stageImage.Builder.StapelStageBuilder().AddLabels(addLabels) } serviceMounts := s.getServiceMounts(prevBuiltImage) - s.addServiceMountsLabels(serviceMounts, c, cr, stageImage) - if !c.UseLegacyStapelBuilder(cr) { - if err := s.addServiceMountsVolumes(serviceMounts, c, cr, stageImage, true); err != nil { + s.addServiceMountsLabels(serviceMounts, c, cb, stageImage) + if !c.UseLegacyStapelBuilder(cb) { + if err := s.addServiceMountsVolumes(serviceMounts, c, cb, stageImage, true); err != nil { return fmt.Errorf("error adding mounts volumes: %w", err) } } customMounts := s.getCustomMounts(prevBuiltImage) - s.addCustomMountLabels(customMounts, c, cr, stageImage) - if !c.UseLegacyStapelBuilder(cr) { - if err := s.addCustomMountVolumes(customMounts, c, cr, stageImage, true); err != nil { + s.addCustomMountLabels(customMounts, c, cb, stageImage) + if !c.UseLegacyStapelBuilder(cb) { + if err := s.addCustomMountVolumes(customMounts, c, cb, stageImage, true); err != nil { return fmt.Errorf("error adding mounts volumes: %w", err) } } @@ -104,7 +104,7 @@ func (s *FromStage) PrepareImage(ctx context.Context, c Conveyor, cr container_b mountpoints = append(mountpoints, mountCfg.To) } if len(mountpoints) > 0 { - if c.UseLegacyStapelBuilder(cr) { + if c.UseLegacyStapelBuilder(cb) { mountpointsStr := strings.Join(mountpoints, " ") stageImage.Builder.LegacyStapelStageBuilder().Container().AddServiceRunCommands(fmt.Sprintf("%s -rf %s", stapel.RmBinPath(), mountpointsStr)) } diff --git a/pkg/build/stage/full_dockerfile.go b/pkg/build/stage/full_dockerfile.go index 619fbbe8a4..4dfad22933 100644 --- a/pkg/build/stage/full_dockerfile.go +++ b/pkg/build/stage/full_dockerfile.go @@ -411,7 +411,7 @@ func isUnsupportedMediaTypeError(err error) bool { var errImageNotExistLocally = errors.New("IMAGE_NOT_EXIST_LOCALLY") -func (s *FullDockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *FullDockerfileStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c) resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash) @@ -509,6 +509,10 @@ func (s *FullDockerfileStage) IsStapelStage() bool { return false } +func (s *FullDockerfileStage) UsesBuildContext() bool { + return true +} + func (s *FullDockerfileStage) dockerfileInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, cmd interface{}, isOnbuildInstruction, isBaseImageOnbuildInstruction bool) ([]string, []string, error) { var dependencies []string var onBuildDependencies []string @@ -698,17 +702,12 @@ func (s *FullDockerfileStage) dockerfileOnBuildInstructionDependencies(ctx conte return []string{expression}, onBuildDependencies, nil } -func (s *FullDockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, _, stageImage *StageImage) error { - archivePath, err := s.prepareContextArchive(ctx, c.GiterminismManager()) - if err != nil { - return err - } - +func (s *FullDockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { if err := s.SetupDockerImageBuilder(stageImage.Builder.DockerfileBuilder(), c); err != nil { return err } - stageImage.Builder.DockerfileBuilder().SetContextArchivePath(archivePath) + stageImage.Builder.DockerfileBuilder().SetBuildContextArchive(buildContextArchive) stageImage.Builder.DockerfileBuilder().AppendLabels(fmt.Sprintf("%s=%s", image.WerfProjectRepoCommitLabel, c.GiterminismManager().HeadCommit())) @@ -719,41 +718,6 @@ func (s *FullDockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cr c return nil } -func (s *FullDockerfileStage) prepareContextArchive(ctx context.Context, giterminismManager giterminism_manager.Interface) (string, error) { - contextPathRelativeToGitWorkTree := s.contextRelativeToGitWorkTree(giterminismManager) - contextPathMatcher := path_matcher.NewPathMatcher(path_matcher.PathMatcherOptions{BasePath: contextPathRelativeToGitWorkTree}) - - archive, err := giterminismManager.LocalGitRepo().GetOrCreateArchive(ctx, git_repo.ArchiveOptions{ - PathScope: contextPathRelativeToGitWorkTree, - PathMatcher: path_matcher.NewMultiPathMatcher( - contextPathMatcher, - s.dockerignorePathMatcher, - ), - Commit: giterminismManager.HeadCommit(), - }) - if err != nil { - return "", fmt.Errorf("unable to create archive: %w", err) - } - - archivePath := archive.GetFilePath() - if len(s.contextAddFiles) != 0 { - if err := logboek.Context(ctx).Debug().LogProcess("Add contextAddFiles to build context archive %s", archivePath).DoError(func() error { - sourceArchivePath := archivePath - destinationArchivePath, err := context_manager.AddContextAddFilesToContextArchive(ctx, sourceArchivePath, giterminismManager.ProjectDir(), s.context, s.contextAddFiles) - if err != nil { - return err - } - - archivePath = destinationArchivePath - return nil - }); err != nil { - return "", err - } - } - - return archivePath, nil -} - func (s *FullDockerfileStage) SetupDockerImageBuilder(b stage_builder.DockerfileBuilderInterface, c Conveyor) error { b.SetDockerfile(s.dockerfile) b.SetDockerfileCtxRelPath(s.dockerfilePath) diff --git a/pkg/build/stage/full_dockerfile_test.go b/pkg/build/stage/full_dockerfile_test.go index 3f134edc1b..164a680c1e 100644 --- a/pkg/build/stage/full_dockerfile_test.go +++ b/pkg/build/stage/full_dockerfile_test.go @@ -78,11 +78,11 @@ var _ = Describe("FullDockerfileStage", func() { Builder: stageBuilder, } - digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage) + digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage, nil) Expect(err).To(Succeed()) Expect(digest).To(Equal(data.TestDependencies.ExpectedDigest)) - err = stage.PrepareImage(ctx, conveyor, containerBackend, nil, stageImage) + err = stage.PrepareImage(ctx, conveyor, containerBackend, nil, stageImage, nil) Expect(err).To(Succeed()) CheckImageDependenciesAfterPrepare(img, stageBuilder, data.TestDependencies.Dependencies) }, @@ -338,14 +338,14 @@ RUN --mount=type=bind,from=build,source=/usr/local/test_project/dist,target=/usr } { - digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage) + digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage, nil) Expect(err).To(Succeed()) Expect(digest).To(Equal("65d219096bc3718c101995b00584d700de791027f2e2ca00635e428932478a1c")) } gitRepoStub.headCommitHash = "23a0884072c0d31b7c42dfaa7f0772cbfa33ec75" { - digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage) + digest, err := stage.GetDependencies(ctx, conveyor, containerBackend, nil, stageImage, nil) Expect(err).To(Succeed()) Expect(digest).To(Equal("beb818f2c49f6501194c72449aff59e80be61b405ef39581b01dbf68da927609")) } diff --git a/pkg/build/stage/git.go b/pkg/build/stage/git.go index ebf0e54be8..82cd567d7f 100644 --- a/pkg/build/stage/git.go +++ b/pkg/build/stage/git.go @@ -25,14 +25,14 @@ func (s *GitStage) isEmpty(_ context.Context) bool { return len(s.gitMappings) == 0 } -func (s *GitStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if err := s.BaseStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { +func (s *GitStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.BaseStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } if c.GiterminismManager().Dev() { addLabels := map[string]string{imagePkg.WerfDevLabel: "true"} - if c.UseLegacyStapelBuilder(cr) { + if c.UseLegacyStapelBuilder(cb) { stageImage.Builder.LegacyStapelStageBuilder().BuilderContainer().AddLabel(addLabels) } else { stageImage.Builder.StapelStageBuilder().AddLabels(addLabels) diff --git a/pkg/build/stage/git_archive.go b/pkg/build/stage/git_archive.go index 079d3f5a2e..4145e521cd 100644 --- a/pkg/build/stage/git_archive.go +++ b/pkg/build/stage/git_archive.go @@ -47,7 +47,7 @@ func (s *GitArchiveStage) SelectSuitableStage(ctx context.Context, c Conveyor, s } // TODO: 1.3 add git mapping type (dir, file, ...) to gitArchive stage digest -func (s *GitArchiveStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *GitArchiveStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { var args []string for _, gitMapping := range s.gitMappings { if gitMapping.IsLocal() { @@ -68,18 +68,18 @@ func (s *GitArchiveStage) GetNextStageDependencies(ctx context.Context, c Convey return s.BaseStage.getNextStageGitDependencies(ctx, c) } -func (s *GitArchiveStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if err := s.GitStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { +func (s *GitArchiveStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.GitStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } for _, gitMapping := range s.gitMappings { - if err := gitMapping.PrepareArchiveForImage(ctx, c, cr, stageImage); err != nil { + if err := gitMapping.PrepareArchiveForImage(ctx, c, cb, stageImage); err != nil { return fmt.Errorf("unable to prepare git mapping %s for image stage: %w", gitMapping.Name, err) } } - if c.UseLegacyStapelBuilder(cr) { + if c.UseLegacyStapelBuilder(cb) { stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", git_repo.CommonGitDataManager.GetArchivesCacheDir(), s.ContainerArchivesDir)) stageImage.Builder.LegacyStapelStageBuilder().Container().RunOptions().AddVolume(fmt.Sprintf("%s:%s:ro", s.ScriptsDir, s.ContainerScriptsDir)) } diff --git a/pkg/build/stage/git_cache.go b/pkg/build/stage/git_cache.go index b62557be0a..7766c29fa5 100644 --- a/pkg/build/stage/git_cache.go +++ b/pkg/build/stage/git_cache.go @@ -46,7 +46,7 @@ func (s *GitCacheStage) IsEmpty(ctx context.Context, c Conveyor, prevBuiltImage return isEmpty, nil } -func (s *GitCacheStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, prevBuiltImage *StageImage) (string, error) { +func (s *GitCacheStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { patchSize, err := s.gitMappingsPatchSize(ctx, c, prevBuiltImage) if err != nil { return "", err diff --git a/pkg/build/stage/git_latest_patch.go b/pkg/build/stage/git_latest_patch.go index 16c57add92..e4ca222c52 100644 --- a/pkg/build/stage/git_latest_patch.go +++ b/pkg/build/stage/git_latest_patch.go @@ -50,7 +50,7 @@ func (s *GitLatestPatchStage) IsEmpty(ctx context.Context, c Conveyor, prevBuilt return isEmpty, nil } -func (s *GitLatestPatchStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, prevBuiltImage *StageImage) (string, error) { +func (s *GitLatestPatchStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { var args []string for _, gitMapping := range s.gitMappings { diff --git a/pkg/build/stage/git_patch.go b/pkg/build/stage/git_patch.go index b70ee42f86..e823ca4819 100644 --- a/pkg/build/stage/git_patch.go +++ b/pkg/build/stage/git_patch.go @@ -65,12 +65,12 @@ func (s *GitPatchStage) hasPrevBuiltStageHadActualGitMappings(ctx context.Contex return true, nil } -func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if err := s.GitStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { +func (s *GitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.GitStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } - if err := s.prepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { + if err := s.prepareImage(ctx, c, cb, prevBuiltImage, stageImage); err != nil { return err } diff --git a/pkg/build/stage/install.go b/pkg/build/stage/install.go index 3bcaf27798..dced378bbd 100644 --- a/pkg/build/stage/install.go +++ b/pkg/build/stage/install.go @@ -28,7 +28,7 @@ type InstallStage struct { *UserWithGitPatchStage } -func (s *InstallStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *InstallStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, Install) if err != nil { return "", err @@ -37,12 +37,12 @@ func (s *InstallStage) GetDependencies(ctx context.Context, c Conveyor, _ contai return util.Sha256Hash(s.builder.InstallChecksum(ctx), stageDependenciesChecksum), nil } -func (s *InstallStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if err := s.UserWithGitPatchStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { +func (s *InstallStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.UserWithGitPatchStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } - if err := s.builder.Install(ctx, cr, stageImage.Builder, c.UseLegacyStapelBuilder(cr)); err != nil { + if err := s.builder.Install(ctx, cb, stageImage.Builder, c.UseLegacyStapelBuilder(cb)); err != nil { return err } diff --git a/pkg/build/stage/instruction/run.go b/pkg/build/stage/instruction/run.go index d964b85a24..ea4c7fd1a4 100644 --- a/pkg/build/stage/instruction/run.go +++ b/pkg/build/stage/instruction/run.go @@ -22,11 +22,12 @@ func NewRun(i *backend_instruction.Run, dependencies []*config.Dependency, hasPr } } -func (stage *Run) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage) (string, error) { +func (stage *Run) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { return util.Sha256Hash(append([]string{string(InstructionRun)}, stage.instruction.Command...)...), nil } -func (stage *Run) PrepareImage(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage) error { +func (stage *Run) PrepareImage(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + stageImage.Builder.DockerfileStageBuilder().SetBuildContextArchive(buildContextArchive) stageImage.Builder.DockerfileStageBuilder().AppendInstruction(stage.instruction) return nil } diff --git a/pkg/build/stage/interface.go b/pkg/build/stage/interface.go index 3e34691d04..bf7a4b4700 100644 --- a/pkg/build/stage/interface.go +++ b/pkg/build/stage/interface.go @@ -15,10 +15,10 @@ type Interface interface { IsEmpty(ctx context.Context, c Conveyor, prevBuiltImage *StageImage) (bool, error) FetchDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error - GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage) (string, error) + GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) GetNextStageDependencies(ctx context.Context, c Conveyor) (string, error) - PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error + PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error PreRunHook(context.Context, Conveyor) error @@ -38,4 +38,6 @@ type Interface interface { HasPrevStage() bool IsStapelStage() bool + + UsesBuildContext() bool } diff --git a/pkg/build/stage/setup.go b/pkg/build/stage/setup.go index 91dbd08b92..60f668cdff 100644 --- a/pkg/build/stage/setup.go +++ b/pkg/build/stage/setup.go @@ -28,7 +28,7 @@ type SetupStage struct { *UserWithGitPatchStage } -func (s *SetupStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *SetupStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { stageDependenciesChecksum, err := s.getStageDependenciesChecksum(ctx, c, Setup) if err != nil { return "", err @@ -37,12 +37,12 @@ func (s *SetupStage) GetDependencies(ctx context.Context, c Conveyor, _ containe return util.Sha256Hash(s.builder.SetupChecksum(ctx), stageDependenciesChecksum), nil } -func (s *SetupStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if err := s.UserWithGitPatchStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { +func (s *SetupStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.UserWithGitPatchStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } - if err := s.builder.Setup(ctx, cr, stageImage.Builder, c.UseLegacyStapelBuilder(cr)); err != nil { + if err := s.builder.Setup(ctx, cb, stageImage.Builder, c.UseLegacyStapelBuilder(cb)); err != nil { return err } diff --git a/pkg/build/stage/stapel_docker_instructions.go b/pkg/build/stage/stapel_docker_instructions.go index ee9d8760f0..825cf84706 100644 --- a/pkg/build/stage/stapel_docker_instructions.go +++ b/pkg/build/stage/stapel_docker_instructions.go @@ -32,10 +32,10 @@ type StapelDockerInstructionsStage struct { instructions *config.Docker } -func (s *StapelDockerInstructionsStage) GetDependencies(_ context.Context, c Conveyor, backend container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *StapelDockerInstructionsStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { var args []string - if c.UseLegacyStapelBuilder(backend) && s.instructions.ExactValues { + if c.UseLegacyStapelBuilder(cb) && s.instructions.ExactValues { args = append(args, "exact-values:::") } @@ -66,16 +66,16 @@ func mapToSortedArgs(h map[string]string) (result []string) { return } -func (s *StapelDockerInstructionsStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if c.UseLegacyStapelBuilder(cr) { +func (s *StapelDockerInstructionsStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if c.UseLegacyStapelBuilder(cb) { stageImage.Image.SetCommitChangeOptions(container_backend.LegacyCommitChangeOptions{ExactValues: s.instructions.ExactValues}) } - if err := s.BaseStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { + if err := s.BaseStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } - if c.UseLegacyStapelBuilder(cr) { + if c.UseLegacyStapelBuilder(cb) { imageCommitChangeOptions := stageImage.Builder.LegacyStapelStageBuilder().Container().CommitChangeOptions() imageCommitChangeOptions.AddVolume(s.instructions.Volume...) imageCommitChangeOptions.AddExpose(s.instructions.Expose...) diff --git a/pkg/build/stage/user_with_git_patch.go b/pkg/build/stage/user_with_git_patch.go index 2f9c2c6da4..4eb76e85b8 100644 --- a/pkg/build/stage/user_with_git_patch.go +++ b/pkg/build/stage/user_with_git_patch.go @@ -35,15 +35,15 @@ func (s *UserWithGitPatchStage) GetNextStageDependencies(ctx context.Context, c return s.BaseStage.getNextStageGitDependencies(ctx, c) } -func (s *UserWithGitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { - if err := s.BaseStage.PrepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { +func (s *UserWithGitPatchStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := s.BaseStage.PrepareImage(ctx, c, cb, prevBuiltImage, stageImage, nil); err != nil { return err } if isPatchEmpty, err := s.GitPatchStage.IsEmpty(ctx, c, prevBuiltImage); err != nil { return err } else if !isPatchEmpty { - if err := s.GitPatchStage.prepareImage(ctx, c, cr, prevBuiltImage, stageImage); err != nil { + if err := s.GitPatchStage.prepareImage(ctx, c, cb, prevBuiltImage, stageImage); err != nil { return err } } diff --git a/pkg/container_backend/archive_interface.go b/pkg/container_backend/archive_interface.go new file mode 100644 index 0000000000..7e1cf9ae6d --- /dev/null +++ b/pkg/container_backend/archive_interface.go @@ -0,0 +1,16 @@ +package container_backend + +import "context" + +type BuildContextArchiver interface { + Create(ctx context.Context, opts BuildContextArchiveCreateOptions) error + Path() string + ExtractOrGetExtractedDir(ctx context.Context) (string, error) + CleanupExtractedDir(ctx context.Context) +} + +type BuildContextArchiveCreateOptions struct { + DockerfileRelToContextPath string + ContextGitSubDir string + ContextAddFiles []string +} diff --git a/pkg/container_backend/build_context/build_context.go b/pkg/container_backend/build_context/build_context.go deleted file mode 100644 index 556a6882e4..0000000000 --- a/pkg/container_backend/build_context/build_context.go +++ /dev/null @@ -1,54 +0,0 @@ -package build_context - -import ( - "context" - "fmt" - "io" - "io/ioutil" - "os" - - "github.com/werf/werf/pkg/util" -) - -type BuildContext struct { - ContextTarReader io.ReadCloser - TmpDir string - - contextTmpDir string -} - -func NewBuildContext(tmpDir string, contextTarReader io.ReadCloser) *BuildContext { - return &BuildContext{TmpDir: tmpDir, ContextTarReader: contextTarReader} -} - -func (c *BuildContext) GetContextDir(ctx context.Context) (string, error) { - if c.contextTmpDir != "" { - return c.contextTmpDir, nil - } - - contextTmpDir, err := ioutil.TempDir(c.TmpDir, "context") - if err != nil { - return "", fmt.Errorf("unable to create context tmp dir: %w", err) - } - - if err := util.ExtractTar(c.ContextTarReader, contextTmpDir, util.ExtractTarOptions{}); err != nil { - return "", fmt.Errorf("unable to extract context tar to tmp context dir: %w", err) - } - if err := c.ContextTarReader.Close(); err != nil { - return "", fmt.Errorf("error closing context tar: %w", err) - } - - c.contextTmpDir = contextTmpDir - - return c.contextTmpDir, nil -} - -func (c *BuildContext) Terminate() error { - if c.contextTmpDir == "" { - return nil - } - if err := os.RemoveAll(c.contextTmpDir); err != nil { - return fmt.Errorf("unable to remove dir %q: %w", c.contextTmpDir, err) - } - return nil -} diff --git a/pkg/container_backend/buildah_backend.go b/pkg/container_backend/buildah_backend.go index 81e5f61cef..3e59c4a5ac 100644 --- a/pkg/container_backend/buildah_backend.go +++ b/pkg/container_backend/buildah_backend.go @@ -20,7 +20,6 @@ import ( copyrec "github.com/werf/copy-recurse" "github.com/werf/logboek" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" "github.com/werf/werf/pkg/image" "github.com/werf/werf/pkg/path_matcher" "github.com/werf/werf/pkg/util" @@ -423,22 +422,10 @@ func (runtime *BuildahBackend) applyDependenciesImports(ctx context.Context, con return nil } -func (runtime *BuildahBackend) BuildDockerfileStage(ctx context.Context, baseImage string, opts BuildDockerfileStageOptions, instructions ...InstructionInterface) (string, *build_context.BuildContext, error) { - buildContext := opts.BuildContext - if buildContext == nil { - for _, instruction := range instructions { - if instruction.UsesBuildContext() { - if opts.ContextTarReader == nil { - panic(fmt.Sprintf("opts.ContextTarReader needed for %q instruction", instruction.Name())) - } - buildContext = build_context.NewBuildContext(runtime.TmpDir, opts.ContextTarReader) - } - } - } - +func (runtime *BuildahBackend) BuildDockerfileStage(ctx context.Context, baseImage string, opts BuildDockerfileStageOptions, instructions ...InstructionInterface) (string, error) { var container *containerDesc if c, err := runtime.createContainers(ctx, []string{baseImage}); err != nil { - return "", nil, err + return "", err } else { container = c[0] } @@ -452,7 +439,7 @@ func (runtime *BuildahBackend) BuildDockerfileStage(ctx context.Context, baseIma logboek.Context(ctx).Debug().LogF("Mounting build container %s\n", container.Name) if err := runtime.mountContainers(ctx, []*containerDesc{container}); err != nil { - return "", buildContext, fmt.Errorf("unable to mount build container %s: %w", container.Name, err) + return "", fmt.Errorf("unable to mount build container %s: %w", container.Name, err) } defer func() { logboek.Context(ctx).Debug().LogF("Unmounting build container %s\n", container.Name) @@ -464,8 +451,8 @@ func (runtime *BuildahBackend) BuildDockerfileStage(ctx context.Context, baseIma logboek.Context(ctx).Debug().LogF("Executing commands for build container %s: %#v\n", container.Name, instructions) for _, instruction := range instructions { - if err := instruction.Apply(ctx, container.Name, runtime.buildah, runtime.getBuildahCommonOpts(ctx, false), buildContext); err != nil { - return "", buildContext, fmt.Errorf("unable to apply instruction %s: %w", instruction.Name(), err) + if err := instruction.Apply(ctx, container.Name, runtime.buildah, runtime.getBuildahCommonOpts(ctx, false), opts.BuildContextArchive); err != nil { + return "", fmt.Errorf("unable to apply instruction %s: %w", instruction.Name(), err) } } @@ -474,10 +461,10 @@ func (runtime *BuildahBackend) BuildDockerfileStage(ctx context.Context, baseIma CommonOpts: runtime.getBuildahCommonOpts(ctx, true), }) if err != nil { - return "", buildContext, fmt.Errorf("error committing container %s: %w", container.Name, err) + return "", fmt.Errorf("error committing container %s: %w", container.Name, err) } - return imageID, buildContext, nil + return imageID, nil } func (runtime *BuildahBackend) BuildStapelStage(ctx context.Context, baseImage string, opts BuildStapelStageOptions) (string, error) { @@ -625,16 +612,10 @@ func (runtime *BuildahBackend) BuildDockerfile(ctx context.Context, dockerfileCo buildArgs[argParts[0]] = argParts[1] } - buildContext := build_context.NewBuildContext(runtime.TmpDir, opts.ContextTar) - contextTmpDir, err := buildContext.GetContextDir(ctx) + buildContextTmpDir, err := opts.BuildContextArchive.ExtractOrGetExtractedDir(ctx) if err != nil { - return "", fmt.Errorf("unable to get context dir: %w", err) + return "", fmt.Errorf("unable to extract build context: %w", err) } - defer func() { - if err := buildContext.Terminate(); err != nil { - logboek.Context(ctx).Error().LogF("ERROR: unable to terminate dockerfile building context: %s\n", err) - } - }() dockerfile, err := ioutil.TempFile(runtime.TmpDir, "*.Dockerfile") if err != nil { @@ -654,7 +635,7 @@ func (runtime *BuildahBackend) BuildDockerfile(ctx context.Context, dockerfileCo CommonOpts: buildah.CommonOpts{ LogWriter: logboek.Context(ctx).OutStream(), }, - ContextDir: contextTmpDir, + ContextDir: buildContextTmpDir, BuildArgs: buildArgs, Target: opts.Target, }) diff --git a/pkg/container_backend/docker_server_backend.go b/pkg/container_backend/docker_server_backend.go index 3051934203..5cd80b2de9 100644 --- a/pkg/container_backend/docker_server_backend.go +++ b/pkg/container_backend/docker_server_backend.go @@ -3,6 +3,7 @@ package container_backend import ( "context" "fmt" + "os" "strings" "github.com/docker/docker/api/types" @@ -10,7 +11,6 @@ import ( "github.com/google/uuid" "github.com/werf/logboek" - "github.com/werf/werf/pkg/container_backend/build_context" "github.com/werf/werf/pkg/docker" "github.com/werf/werf/pkg/image" "github.com/werf/werf/pkg/util" @@ -36,8 +36,8 @@ func (runtime *DockerServerBackend) CalculateDependencyImportChecksum(ctx contex func (runtime *DockerServerBackend) BuildDockerfile(ctx context.Context, _ []byte, opts BuildDockerfileOpts) (string, error) { switch { - case opts.ContextTar == nil: - panic(fmt.Sprintf("ContextTar can't be nil: %+v", opts)) + case opts.BuildContextArchive == nil: + panic(fmt.Sprintf("BuildContextArchive can't be nil: %+v", opts)) case opts.DockerfileCtxRelPath == "": panic(fmt.Sprintf("DockerfileCtxRelPath can't be empty: %+v", opts)) } @@ -77,10 +77,16 @@ func (runtime *DockerServerBackend) BuildDockerfile(ctx context.Context, _ []byt fmt.Printf("[DOCKER BUILD] docker build %s\n", strings.Join(cliArgs, " ")) } - return tempID, docker.CliBuild_LiveOutputWithCustomIn(ctx, opts.ContextTar, cliArgs...) + contextReader, err := os.Open(opts.BuildContextArchive.Path()) + if err != nil { + return "", fmt.Errorf("unable to open context archive %q: %w", opts.BuildContextArchive.Path(), err) + } + defer contextReader.Close() + + return tempID, docker.CliBuild_LiveOutputWithCustomIn(ctx, contextReader, cliArgs...) } -func (runtime *DockerServerBackend) BuildDockerfileStage(ctx context.Context, baseImage string, opts BuildDockerfileStageOptions, instructions ...InstructionInterface) (string, *build_context.BuildContext, error) { +func (runtime *DockerServerBackend) BuildDockerfileStage(ctx context.Context, baseImage string, opts BuildDockerfileStageOptions, instructions ...InstructionInterface) (string, error) { panic("not implemented") } diff --git a/pkg/container_backend/instruction.go b/pkg/container_backend/instruction.go index a5242102cf..e1269e02f2 100644 --- a/pkg/container_backend/instruction.go +++ b/pkg/container_backend/instruction.go @@ -4,11 +4,10 @@ import ( "context" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" ) type InstructionInterface interface { Name() string - Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error + Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive BuildContextArchiver) error UsesBuildContext() bool } diff --git a/pkg/container_backend/instruction/add.go b/pkg/container_backend/instruction/add.go index 14eb431ffd..1ab8390739 100644 --- a/pkg/container_backend/instruction/add.go +++ b/pkg/container_backend/instruction/add.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,13 +21,13 @@ func (i *Add) UsesBuildContext() bool { return true } -func (i *Add) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { - contextDir, err := buildContext.GetContextDir(ctx) +func (i *Add) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { + buildContextTmpDir, err := buildContextArchive.ExtractOrGetExtractedDir(ctx) if err != nil { - return fmt.Errorf("unable to get build context dir: %w", err) + return fmt.Errorf("unable to extract build context: %w", err) } - if err := drv.Add(ctx, containerName, i.Src, i.Dst, buildah.AddOpts{CommonOpts: drvOpts, ContextDir: contextDir}); err != nil { + if err := drv.Add(ctx, containerName, i.Src, i.Dst, buildah.AddOpts{CommonOpts: drvOpts, ContextDir: buildContextTmpDir}); err != nil { return fmt.Errorf("error adding %v to %s for container %s: %w", i.Src, i.Dst, containerName, err) } return nil diff --git a/pkg/container_backend/instruction/cmd.go b/pkg/container_backend/instruction/cmd.go index bfe096d873..517f1dc6f1 100644 --- a/pkg/container_backend/instruction/cmd.go +++ b/pkg/container_backend/instruction/cmd.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *Cmd) UsesBuildContext() bool { return false } -func (i *Cmd) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Cmd) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, Cmd: i.Cmd.Cmd}); err != nil { return fmt.Errorf("error setting cmd %v for container %s: %w", i.Cmd, containerName, err) } diff --git a/pkg/container_backend/instruction/copy.go b/pkg/container_backend/instruction/copy.go index b6f6eabc9e..83e3dc8dee 100644 --- a/pkg/container_backend/instruction/copy.go +++ b/pkg/container_backend/instruction/copy.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,13 +21,13 @@ func (i *Copy) UsesBuildContext() bool { return true } -func (i *Copy) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { - contextDir, err := buildContext.GetContextDir(ctx) +func (i *Copy) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { + buildContextTmpDir, err := buildContextArchive.ExtractOrGetExtractedDir(ctx) if err != nil { - return fmt.Errorf("unable to get build context dir: %w", err) + return fmt.Errorf("unable to extract build context: %w", err) } - if err := drv.Copy(ctx, containerName, contextDir, i.Src, i.Dst, buildah.CopyOpts{CommonOpts: drvOpts, From: i.From}); err != nil { + if err := drv.Copy(ctx, containerName, buildContextTmpDir, i.Src, i.Dst, buildah.CopyOpts{CommonOpts: drvOpts, From: i.From}); err != nil { return fmt.Errorf("error copying %v to %s for container %s: %w", i.Src, i.Dst, containerName, err) } return nil diff --git a/pkg/container_backend/instruction/entrypoint.go b/pkg/container_backend/instruction/entrypoint.go index 0e5e238534..13f673fa1e 100644 --- a/pkg/container_backend/instruction/entrypoint.go +++ b/pkg/container_backend/instruction/entrypoint.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *Entrypoint) UsesBuildContext() bool { return false } -func (i *Entrypoint) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Entrypoint) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, Entrypoint: i.Entrypoint.Entrypoint}); err != nil { return fmt.Errorf("error setting entrypoint %v for container %s: %w", i.Entrypoint, containerName, err) } diff --git a/pkg/container_backend/instruction/env.go b/pkg/container_backend/instruction/env.go index ced5fda77f..9aeba1d3a7 100644 --- a/pkg/container_backend/instruction/env.go +++ b/pkg/container_backend/instruction/env.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *Env) UsesBuildContext() bool { return false } -func (i *Env) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Env) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, Envs: i.Envs}); err != nil { return fmt.Errorf("error setting envs %v for container %s: %w", i.Envs, containerName, err) } diff --git a/pkg/container_backend/instruction/expose.go b/pkg/container_backend/instruction/expose.go index f35a8afdf3..dc712e37f5 100644 --- a/pkg/container_backend/instruction/expose.go +++ b/pkg/container_backend/instruction/expose.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *Expose) UsesBuildContext() bool { return false } -func (i *Expose) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Expose) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, Expose: i.Ports}); err != nil { return fmt.Errorf("error setting exposed ports %v for container %s: %w", i.Ports, containerName, err) } diff --git a/pkg/container_backend/instruction/label.go b/pkg/container_backend/instruction/label.go index 697116a0f1..78609be54e 100644 --- a/pkg/container_backend/instruction/label.go +++ b/pkg/container_backend/instruction/label.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -29,7 +29,7 @@ func (i *Label) LabelsAsList() []string { return labels } -func (i *Label) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Label) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, Labels: i.LabelsAsList()}); err != nil { return fmt.Errorf("error setting labels %v for container %s: %w", i.LabelsAsList(), containerName, err) } diff --git a/pkg/container_backend/instruction/on_build.go b/pkg/container_backend/instruction/on_build.go index b10d7a732c..864d55d784 100644 --- a/pkg/container_backend/instruction/on_build.go +++ b/pkg/container_backend/instruction/on_build.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *OnBuild) UsesBuildContext() bool { return false } -func (i *OnBuild) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *OnBuild) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, OnBuild: i.Instruction}); err != nil { return fmt.Errorf("error setting onbuild %v for container %s: %w", i.Instruction, containerName, err) } diff --git a/pkg/container_backend/instruction/run.go b/pkg/container_backend/instruction/run.go index 59832a1774..16ea2f9887 100644 --- a/pkg/container_backend/instruction/run.go +++ b/pkg/container_backend/instruction/run.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,9 +21,8 @@ func (i *Run) UsesBuildContext() bool { return false } -func (i *Run) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Run) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.RunCommand(ctx, containerName, i.Command, buildah.RunCommandOpts{ - // FIXME(ilya-lesikov): should we suppress or not? CommonOpts: drvOpts, }); err != nil { return fmt.Errorf("error running command %v for container %s: %w", i.Command, containerName, err) diff --git a/pkg/container_backend/instruction/shell.go b/pkg/container_backend/instruction/shell.go index 839ee09d3d..01c82584af 100644 --- a/pkg/container_backend/instruction/shell.go +++ b/pkg/container_backend/instruction/shell.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *Shell) UsesBuildContext() bool { return false } -func (i *Shell) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Shell) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, Shell: i.Shell.Shell}); err != nil { return fmt.Errorf("error setting shell %v for container %s: %w", i.Shell, containerName, err) } diff --git a/pkg/container_backend/instruction/stop_signal.go b/pkg/container_backend/instruction/stop_signal.go index 3e10f4bb59..a165cb7e85 100644 --- a/pkg/container_backend/instruction/stop_signal.go +++ b/pkg/container_backend/instruction/stop_signal.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *StopSignal) UsesBuildContext() bool { return false } -func (i *StopSignal) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *StopSignal) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, StopSignal: i.Signal}); err != nil { return fmt.Errorf("error setting stop signal %v for container %s: %w", i.Signal, containerName, err) } diff --git a/pkg/container_backend/instruction/user.go b/pkg/container_backend/instruction/user.go index e5a3dcaeed..a189e7fb1c 100644 --- a/pkg/container_backend/instruction/user.go +++ b/pkg/container_backend/instruction/user.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *User) UsesBuildContext() bool { return false } -func (i *User) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *User) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, User: i.User.User}); err != nil { return fmt.Errorf("error setting user %s for container %s: %w", i.User, containerName, err) } diff --git a/pkg/container_backend/instruction/volume.go b/pkg/container_backend/instruction/volume.go index 7ee8b924f4..412dc2bb94 100644 --- a/pkg/container_backend/instruction/volume.go +++ b/pkg/container_backend/instruction/volume.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *Volume) UsesBuildContext() bool { return false } -func (i *Volume) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Volume) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, Volumes: i.Volumes}); err != nil { return fmt.Errorf("error setting volumes %v for container %s: %w", i.Volumes, containerName, err) } diff --git a/pkg/container_backend/instruction/workdir.go b/pkg/container_backend/instruction/workdir.go index 580ae6f6bc..cb3a85d804 100644 --- a/pkg/container_backend/instruction/workdir.go +++ b/pkg/container_backend/instruction/workdir.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/werf/werf/pkg/buildah" - "github.com/werf/werf/pkg/container_backend/build_context" + "github.com/werf/werf/pkg/container_backend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) @@ -21,7 +21,7 @@ func (i *Workdir) UsesBuildContext() bool { return false } -func (i *Workdir) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContext *build_context.BuildContext) error { +func (i *Workdir) Apply(ctx context.Context, containerName string, drv buildah.Buildah, drvOpts buildah.CommonOpts, buildContextArchive container_backend.BuildContextArchiver) error { if err := drv.Config(ctx, containerName, buildah.ConfigOpts{CommonOpts: drvOpts, Workdir: i.Workdir.Workdir}); err != nil { return fmt.Errorf("error setting workdir %s for container %s: %w", i.Workdir, containerName, err) } diff --git a/pkg/container_backend/interface.go b/pkg/container_backend/interface.go index 4dfe81f84f..9c04a27883 100644 --- a/pkg/container_backend/interface.go +++ b/pkg/container_backend/interface.go @@ -2,9 +2,7 @@ package container_backend import ( "context" - "io" - "github.com/werf/werf/pkg/container_backend/build_context" "github.com/werf/werf/pkg/image" ) @@ -33,7 +31,7 @@ type GetImageInfoOpts struct { type BuildDockerfileOpts struct { CommonOpts - ContextTar io.ReadCloser + BuildContextArchive BuildContextArchiver DockerfileCtxRelPath string // TODO: remove this and instead write the []byte dockerfile to /Dockerfile in the ContextTar inDockerServerBackend.BuildDockerfile(). Target string BuildArgs []string // {"key1=value1", "key2=value2", ... } @@ -47,8 +45,7 @@ type BuildDockerfileOpts struct { type BuildDockerfileStageOptions struct { CommonOpts - BuildContext *build_context.BuildContext - ContextTarReader io.ReadCloser + BuildContextArchive BuildContextArchiver } type ContainerBackend interface { @@ -59,7 +56,7 @@ type ContainerBackend interface { GetImageInfo(ctx context.Context, ref string, opts GetImageInfoOpts) (*image.Info, error) BuildDockerfile(ctx context.Context, dockerfile []byte, opts BuildDockerfileOpts) (string, error) - BuildDockerfileStage(ctx context.Context, baseImage string, opts BuildDockerfileStageOptions, instructions ...InstructionInterface) (string, *build_context.BuildContext, error) + BuildDockerfileStage(ctx context.Context, baseImage string, opts BuildDockerfileStageOptions, instructions ...InstructionInterface) (string, error) BuildStapelStage(ctx context.Context, baseImage string, opts BuildStapelStageOptions) (string, error) CalculateDependencyImportChecksum(ctx context.Context, dependencyImport DependencyImportSpec) (string, error) diff --git a/pkg/container_backend/perf_check_container_backend.go b/pkg/container_backend/perf_check_container_backend.go index a8a1d3a507..b039827883 100644 --- a/pkg/container_backend/perf_check_container_backend.go +++ b/pkg/container_backend/perf_check_container_backend.go @@ -4,7 +4,6 @@ import ( "context" "github.com/werf/logboek" - "github.com/werf/werf/pkg/container_backend/build_context" "github.com/werf/werf/pkg/image" ) @@ -72,10 +71,10 @@ func (runtime *PerfCheckContainerBackend) BuildDockerfile(ctx context.Context, d return } -func (runtime *PerfCheckContainerBackend) BuildDockerfileStage(ctx context.Context, baseImage string, opts BuildDockerfileStageOptions, instructions ...InstructionInterface) (resID string, resBuildContext *build_context.BuildContext, resErr error) { +func (runtime *PerfCheckContainerBackend) BuildDockerfileStage(ctx context.Context, baseImage string, opts BuildDockerfileStageOptions, instructions ...InstructionInterface) (resID string, resErr error) { logboek.Context(ctx).Default().LogProcess("ContainerBackend.BuildDockerfile"). Do(func() { - resID, resBuildContext, resErr = runtime.ContainerBackend.BuildDockerfileStage(ctx, baseImage, opts, instructions...) + resID, resErr = runtime.ContainerBackend.BuildDockerfileStage(ctx, baseImage, opts, instructions...) }) return } diff --git a/pkg/container_backend/stage_builder/dockerfile_builder.go b/pkg/container_backend/stage_builder/dockerfile_builder.go index d59d163d5f..547fdda654 100644 --- a/pkg/container_backend/stage_builder/dockerfile_builder.go +++ b/pkg/container_backend/stage_builder/dockerfile_builder.go @@ -3,7 +3,6 @@ package stage_builder import ( "context" "fmt" - "os" "github.com/werf/logboek" "github.com/werf/werf/pkg/container_backend" @@ -20,14 +19,14 @@ type DockerfileBuilderInterface interface { SetNetwork(network string) SetSSH(ssh string) AppendLabels(labels ...string) - SetContextArchivePath(contextArchivePath string) + SetBuildContextArchive(buildContextArchive container_backend.BuildContextArchiver) } type DockerfileBuilder struct { ContainerBackend container_backend.ContainerBackend Dockerfile []byte BuildDockerfileOptions container_backend.BuildDockerfileOpts - ContextArchivePath string + BuildContextArchive container_backend.BuildContextArchiver Image container_backend.ImageInterface } @@ -40,20 +39,14 @@ func (b *DockerfileBuilder) Build(ctx context.Context) error { // filePathToStdin != "" ?? if container_backend.Debug() { - fmt.Printf("[DOCKER BUILD] context archive path: %s\n", b.ContextArchivePath) + fmt.Printf("[DOCKER BUILD] context archive path: %s\n", b.BuildContextArchive.Path()) } - contextReader, err := os.Open(b.ContextArchivePath) - if err != nil { - return fmt.Errorf("unable to open context archive %q: %w", b.ContextArchivePath, err) - } - defer contextReader.Close() - opts := b.BuildDockerfileOptions - opts.ContextTar = contextReader + opts.BuildContextArchive = b.BuildContextArchive if container_backend.Debug() { - fmt.Printf("ContextArchivePath=%q\n", b.ContextArchivePath) + fmt.Printf("BuildContextArchive=%q\n", b.BuildContextArchive) fmt.Printf("BiuldDockerfileOptions: %#v\n", opts) } @@ -110,6 +103,6 @@ func (b *DockerfileBuilder) AppendLabels(labels ...string) { b.BuildDockerfileOptions.Labels = append(b.BuildDockerfileOptions.Labels, labels...) } -func (b *DockerfileBuilder) SetContextArchivePath(contextArchivePath string) { - b.ContextArchivePath = contextArchivePath +func (b *DockerfileBuilder) SetBuildContextArchive(buildContextArchive container_backend.BuildContextArchiver) { + b.BuildContextArchive = buildContextArchive } diff --git a/pkg/container_backend/stage_builder/dockerfile_stage_builder.go b/pkg/container_backend/stage_builder/dockerfile_stage_builder.go index cf3e60ba39..ccd08f1f5f 100644 --- a/pkg/container_backend/stage_builder/dockerfile_stage_builder.go +++ b/pkg/container_backend/stage_builder/dockerfile_stage_builder.go @@ -5,12 +5,11 @@ import ( "fmt" "github.com/werf/werf/pkg/container_backend" - "github.com/werf/werf/pkg/container_backend/build_context" ) type DockerfileStageBuilderInterface interface { DockerfileStageIntructionBuilderInterface - SetBuildContext(buildContext *build_context.BuildContext) DockerfileStageBuilderInterface + SetBuildContextArchive(buildContextArchive container_backend.BuildContextArchiver) DockerfileStageBuilderInterface Build(ctx context.Context, opts container_backend.BuildOptions) error } @@ -21,10 +20,10 @@ type DockerfileStageIntructionBuilderInterface interface { } type DockerfileStageBuilder struct { - preInstructions []container_backend.InstructionInterface - instructions []container_backend.InstructionInterface - postInstructions []container_backend.InstructionInterface - buildContext *build_context.BuildContext + preInstructions []container_backend.InstructionInterface + instructions []container_backend.InstructionInterface + postInstructions []container_backend.InstructionInterface + buildContextArchive container_backend.BuildContextArchiver baseImage string resultImage container_backend.ImageInterface @@ -54,19 +53,18 @@ func (b *DockerfileStageBuilder) AppendPostInstruction(i container_backend.Instr return b } -func (b *DockerfileStageBuilder) SetBuildContext(buildContext *build_context.BuildContext) DockerfileStageBuilderInterface { - b.buildContext = buildContext +func (b *DockerfileStageBuilder) SetBuildContextArchive(buildContextArchive container_backend.BuildContextArchiver) DockerfileStageBuilderInterface { + b.buildContextArchive = buildContextArchive return b } func (b *DockerfileStageBuilder) Build(ctx context.Context, opts container_backend.BuildOptions) error { instructions := append(append(b.preInstructions, b.instructions...), b.postInstructions...) - backendOpts := container_backend.BuildDockerfileStageOptions{BuildContext: b.buildContext} + backendOpts := container_backend.BuildDockerfileStageOptions{BuildContextArchive: b.buildContextArchive} - if builtID, buildContext, err := b.containerBackend.BuildDockerfileStage(ctx, b.baseImage, backendOpts, instructions...); err != nil { + if builtID, err := b.containerBackend.BuildDockerfileStage(ctx, b.baseImage, backendOpts, instructions...); err != nil { return fmt.Errorf("error building dockerfile stage: %w", err) } else { - b.buildContext = buildContext b.resultImage.SetBuiltID(builtID) }