From db8d3373588a0c19a9415def4245c997574776f9 Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Tue, 4 Oct 2022 17:22:01 +0300 Subject: [PATCH] feat(staged-dockerfile): prepare conveyor, stage and dockerfile parser for new impl Signed-off-by: Timofey Kirillov --- pkg/build/build_phase.go | 2 +- pkg/build/image/dockerfile.go | 32 +++++-- pkg/build/image/stapel.go | 2 +- pkg/build/stage/dockerfile_instruction.go | 40 ++++++++ .../{dockerfile.go => full_dockerfile.go} | 28 +++--- ...erfile_test.go => full_dockerfile_test.go} | 48 +++++----- ...tions.go => stapel_docker_instructions.go} | 14 +-- pkg/config/image_from_dockerfile.go | 1 + pkg/config/raw_image_from_dockerfile.go | 2 + pkg/container_backend/buildah_backend.go | 95 ++++++++++--------- pkg/dockerfile/dockerfile.go | 55 ++++++++++- pkg/dockerfile/dockerfile_stage.go | 5 +- .../helpers.go} | 2 +- .../instructions.go} | 34 +++---- 14 files changed, 233 insertions(+), 127 deletions(-) create mode 100644 pkg/build/stage/dockerfile_instruction.go rename pkg/build/stage/{dockerfile.go => full_dockerfile.go} (91%) rename pkg/build/stage/{dockerfile_test.go => full_dockerfile_test.go} (88%) rename pkg/build/stage/{docker_instructions.go => stapel_docker_instructions.go} (81%) rename pkg/{build/dockerfile_helpers/dockerfile_helpers.go => dockerfile/helpers.go} (97%) rename pkg/{container_backend/dockerfile_commands.go => dockerfile/instructions.go} (56%) diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 713b80aa6b..9e27285c72 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -631,7 +631,7 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *imag } switch stg.(type) { - case *stage.DockerfileStage: + case *stage.FullDockerfileStage: var labels []string for key, value := range serviceLabels { labels = append(labels, fmt.Sprintf("%s=%v", key, value)) diff --git a/pkg/build/image/dockerfile.go b/pkg/build/image/dockerfile.go index d902ac5eda..470d664bd5 100644 --- a/pkg/build/image/dockerfile.go +++ b/pkg/build/image/dockerfile.go @@ -11,7 +11,6 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/werf/logboek" - "github.com/werf/werf/pkg/build/dockerfile_helpers" "github.com/werf/werf/pkg/build/stage" "github.com/werf/werf/pkg/config" "github.com/werf/werf/pkg/dockerfile" @@ -20,12 +19,25 @@ import ( ) func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig *config.ImageFromDockerfile, opts CommonImageOptions) (ImagesSets, error) { - // TODO: check dockerfile-config mode: use legacy (default) dockerfile mapper or new staged-dockerfile mapper - useLegacyMapper := true + if dockerfileImageConfig.Staged { + relDockerfilePath := filepath.Join(dockerfileImageConfig.Context, dockerfileImageConfig.Dockerfile) + dockerfileData, err := opts.GiterminismManager.FileReader().ReadDockerfile(ctx, relDockerfilePath) + if err != nil { + return nil, fmt.Errorf("unable to read dockerfile %s: %w", relDockerfilePath, err) + } + + d, err := dockerfile.ParseDockerfile(dockerfileData, dockerfile.DockerfileOptions{ + Target: dockerfileImageConfig.Target, + BuildArgs: util.MapStringInterfaceToMapStringString(dockerfileImageConfig.Args), + AddHost: dockerfileImageConfig.AddHost, + Network: dockerfileImageConfig.Network, + SSH: dockerfileImageConfig.SSH, + }) + if err != nil { + return nil, fmt.Errorf("unable to parse dockerfile %s: %w", relDockerfilePath, err) + } - if !useLegacyMapper { - // TODO: dockerfileImageConfig to dockerfile.Dockerfile obj - return mapDockerfileToImagesSets(ctx, dockerfile.Dockerfile{}) + return mapDockerfileToImagesSets(ctx, d) } img, err := mapLegacyDockerfileToImage(ctx, dockerfileImageConfig, opts) @@ -40,7 +52,7 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig return ret, nil } -func mapDockerfileToImagesSets(ctx context.Context, cfg dockerfile.Dockerfile) (ImagesSets, error) { +func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile) (ImagesSets, error) { var ret ImagesSets stagesSets, err := cfg.GroupStagesByIndependentSets(ctx) @@ -136,9 +148,9 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf return nil, err } - dockerfile_helpers.ResolveDockerStagesFromValue(dockerStages) + dockerfile.ResolveDockerStagesFromValue(dockerStages) - dockerTargetIndex, err := dockerfile_helpers.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target) + dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target) if err != nil { return nil, err } @@ -155,7 +167,7 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf ProjectName: opts.ProjectName, } - dockerfileStage := stage.GenerateDockerfileStage( + dockerfileStage := stage.GenerateFullDockerfileStage( stage.NewDockerRunArgs( dockerfileData, dockerfileImageConfig.Dockerfile, diff --git a/pkg/build/image/stapel.go b/pkg/build/image/stapel.go index 7eaa7f39c7..a796ee4237 100644 --- a/pkg/build/image/stapel.go +++ b/pkg/build/image/stapel.go @@ -108,7 +108,7 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap stages = append(stages, stage.NewGitLatestPatchStage(gitPatchStageOptions, baseStageOptions)) } - stages = appendIfExist(ctx, stages, stage.GenerateDockerInstructionsStage(stapelImageConfig.(*config.StapelImage), baseStageOptions)) + stages = appendIfExist(ctx, stages, stage.GenerateStapelDockerInstructionsStage(stapelImageConfig.(*config.StapelImage), baseStageOptions)) } if len(gitMappings) != 0 { diff --git a/pkg/build/stage/dockerfile_instruction.go b/pkg/build/stage/dockerfile_instruction.go new file mode 100644 index 0000000000..f9d626d304 --- /dev/null +++ b/pkg/build/stage/dockerfile_instruction.go @@ -0,0 +1,40 @@ +package stage + +import ( + "context" + + "github.com/werf/werf/pkg/config" + "github.com/werf/werf/pkg/container_backend" + "github.com/werf/werf/pkg/dockerfile" +) + +// TODO(staged-dockerfile): common implementation and possible implementation for each separate dockerfile instruction + +type DockerfileInstruction interface { + Name() string +} + +type DockerfileInstructionStage struct { + *BaseStage + + instruction DockerfileInstruction + dependencies []*config.Dependency +} + +func NewDockerfileInstructionStage(instruction DockerfileInstruction, dependencies []*config.Dependency, dockerfileStage *dockerfile.DockerfileStage, opts *NewBaseStageOptions) *DockerfileInstructionStage { + return &DockerfileInstructionStage{ + instruction: instruction, + dependencies: dependencies, + BaseStage: newBaseStage(StageName(instruction.Name()), opts), + } +} + +func (stage *DockerfileInstructionStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage) (string, error) { + // TODO: digest + return "", nil +} + +func (stage *DockerfileInstructionStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { + // TODO: setup builder + return nil +} diff --git a/pkg/build/stage/dockerfile.go b/pkg/build/stage/full_dockerfile.go similarity index 91% rename from pkg/build/stage/dockerfile.go rename to pkg/build/stage/full_dockerfile.go index 28f95ee301..7d2c93c7fd 100644 --- a/pkg/build/stage/dockerfile.go +++ b/pkg/build/stage/full_dockerfile.go @@ -34,12 +34,12 @@ func IsErrInvalidBaseImage(err error) bool { return err != nil && errors.Is(err, ErrInvalidBaseImage) } -func GenerateDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages, contextChecksum *ContextChecksum, baseStageOptions *NewBaseStageOptions, dependencies []*config.Dependency) *DockerfileStage { - return newDockerfileStage(dockerRunArgs, dockerStages, contextChecksum, baseStageOptions, dependencies) +func GenerateFullDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages, contextChecksum *ContextChecksum, baseStageOptions *NewBaseStageOptions, dependencies []*config.Dependency) *FullDockerfileStage { + return newFullDockerfileStage(dockerRunArgs, dockerStages, contextChecksum, baseStageOptions, dependencies) } -func newDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages, contextChecksum *ContextChecksum, baseStageOptions *NewBaseStageOptions, dependencies []*config.Dependency) *DockerfileStage { - s := &DockerfileStage{} +func newFullDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages, contextChecksum *ContextChecksum, baseStageOptions *NewBaseStageOptions, dependencies []*config.Dependency) *FullDockerfileStage { + s := &FullDockerfileStage{} s.DockerRunArgs = dockerRunArgs s.DockerStages = dockerStages s.ContextChecksum = contextChecksum @@ -49,7 +49,7 @@ func newDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages return s } -type DockerfileStage struct { +type FullDockerfileStage struct { dependencies []*config.Dependency *DockerRunArgs @@ -319,7 +319,7 @@ type dockerfileInstructionInterface interface { Name() string } -func (s *DockerfileStage) FetchDependencies(ctx context.Context, c Conveyor, containerBackend container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error { +func (s *FullDockerfileStage) FetchDependencies(ctx context.Context, c Conveyor, containerBackend container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error { resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c) resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash) @@ -411,7 +411,7 @@ func isUnsupportedMediaTypeError(err error) bool { var errImageNotExistLocally = errors.New("IMAGE_NOT_EXIST_LOCALLY") -func (s *DockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *FullDockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) { resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c) resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash) @@ -501,7 +501,7 @@ func (s *DockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ con return util.Sha256Hash(dockerfileStageDependencies...), nil } -func (s *DockerfileStage) dockerfileInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, cmd interface{}, isOnbuildInstruction, isBaseImageOnbuildInstruction bool) ([]string, []string, error) { +func (s *FullDockerfileStage) dockerfileInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, cmd interface{}, isOnbuildInstruction, isBaseImageOnbuildInstruction bool) ([]string, []string, error) { var dependencies []string var onBuildDependencies []string @@ -666,7 +666,7 @@ func (s *DockerfileStage) dockerfileInstructionDependencies(ctx context.Context, return dependencies, onBuildDependencies, nil } -func (s *DockerfileStage) dockerfileOnBuildInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, expression string, isBaseImageOnbuildInstruction bool) ([]string, []string, error) { +func (s *FullDockerfileStage) dockerfileOnBuildInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, expression string, isBaseImageOnbuildInstruction bool) ([]string, []string, error) { p, err := parser.Parse(bytes.NewReader([]byte(expression))) if err != nil { return nil, nil, err @@ -690,7 +690,7 @@ func (s *DockerfileStage) dockerfileOnBuildInstructionDependencies(ctx context.C return []string{expression}, onBuildDependencies, nil } -func (s *DockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, _, stageImage *StageImage) error { +func (s *FullDockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, _, stageImage *StageImage) error { archivePath, err := s.prepareContextArchive(ctx, c.GiterminismManager()) if err != nil { return err @@ -711,7 +711,7 @@ func (s *DockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cr conta return nil } -func (s *DockerfileStage) prepareContextArchive(ctx context.Context, giterminismManager giterminism_manager.Interface) (string, error) { +func (s *FullDockerfileStage) prepareContextArchive(ctx context.Context, giterminismManager giterminism_manager.Interface) (string, error) { contextPathRelativeToGitWorkTree := s.contextRelativeToGitWorkTree(giterminismManager) contextPathMatcher := path_matcher.NewPathMatcher(path_matcher.PathMatcherOptions{BasePath: contextPathRelativeToGitWorkTree}) @@ -746,7 +746,7 @@ func (s *DockerfileStage) prepareContextArchive(ctx context.Context, giterminism return archivePath, nil } -func (s *DockerfileStage) SetupDockerImageBuilder(b stage_builder.DockerfileBuilderInterface, c Conveyor) error { +func (s *FullDockerfileStage) SetupDockerImageBuilder(b stage_builder.DockerfileBuilderInterface, c Conveyor) error { b.SetDockerfile(s.dockerfile) b.SetDockerfileCtxRelPath(s.dockerfilePath) @@ -782,7 +782,7 @@ func (s *DockerfileStage) SetupDockerImageBuilder(b stage_builder.DockerfileBuil return nil } -func (s *DockerfileStage) calculateFilesChecksum(ctx context.Context, giterminismManager giterminism_manager.Interface, wildcards []string, dockerfileLine string) (string, error) { +func (s *FullDockerfileStage) calculateFilesChecksum(ctx context.Context, giterminismManager giterminism_manager.Interface, wildcards []string, dockerfileLine string) (string, error) { var checksum string var err error @@ -827,7 +827,7 @@ func (s *DockerfileStage) calculateFilesChecksum(ctx context.Context, giterminis return checksum, nil } -func (s *DockerfileStage) calculateFilesChecksumWithGit(ctx context.Context, giterminismManager giterminism_manager.Interface, wildcards []string, dockerfileLine string) (string, error) { +func (s *FullDockerfileStage) calculateFilesChecksumWithGit(ctx context.Context, giterminismManager giterminism_manager.Interface, wildcards []string, dockerfileLine string) (string, error) { contextPathRelativeToGitWorkTree := s.contextRelativeToGitWorkTree(giterminismManager) wildcardsPathMatcher := path_matcher.NewPathMatcher(path_matcher.PathMatcherOptions{ BasePath: contextPathRelativeToGitWorkTree, diff --git a/pkg/build/stage/dockerfile_test.go b/pkg/build/stage/full_dockerfile_test.go similarity index 88% rename from pkg/build/stage/dockerfile_test.go rename to pkg/build/stage/full_dockerfile_test.go index 8526e8b46f..a97be7783b 100644 --- a/pkg/build/stage/dockerfile_test.go +++ b/pkg/build/stage/full_dockerfile_test.go @@ -9,25 +9,25 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/werf/werf/pkg/build/dockerfile_helpers" "github.com/werf/werf/pkg/container_backend/stage_builder" + "github.com/werf/werf/pkg/dockerfile" "github.com/werf/werf/pkg/util" ) -func testDockerfileToDockerStages(dockerfile []byte) ([]instructions.Stage, []instructions.ArgCommand) { - p, err := parser.Parse(bytes.NewReader(dockerfile)) +func testDockerfileToDockerStages(dockerfileData []byte) ([]instructions.Stage, []instructions.ArgCommand) { + p, err := parser.Parse(bytes.NewReader(dockerfileData)) Expect(err).To(Succeed()) dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST) Expect(err).To(Succeed()) - dockerfile_helpers.ResolveDockerStagesFromValue(dockerStages) + dockerfile.ResolveDockerStagesFromValue(dockerStages) return dockerStages, dockerMetaArgs } -func newTestDockerfileStage(dockerfile []byte, target string, buildArgs map[string]interface{}, dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dependencies []*TestDependency) *DockerfileStage { - dockerTargetIndex, err := dockerfile_helpers.GetDockerTargetStageIndex(dockerStages, target) +func newTestFullDockerfileStage(dockerfileData []byte, target string, buildArgs map[string]interface{}, dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dependencies []*TestDependency) *FullDockerfileStage { + dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, target) Expect(err).To(Succeed()) ds := NewDockerStages( @@ -37,9 +37,9 @@ func newTestDockerfileStage(dockerfile []byte, target string, buildArgs map[stri dockerTargetIndex, ) - return newDockerfileStage( + return newFullDockerfileStage( NewDockerRunArgs( - dockerfile, + dockerfileData, "no-such-path", target, "", @@ -59,7 +59,7 @@ func newTestDockerfileStage(dockerfile []byte, target string, buildArgs map[stri ) } -var _ = Describe("DockerfileStage", func() { +var _ = Describe("FullDockerfileStage", func() { DescribeTable("configuring images dependencies for dockerfile stage", func(data TestDockerfileDependencies) { ctx := context.Background() @@ -67,9 +67,9 @@ var _ = Describe("DockerfileStage", func() { conveyor := NewConveyorStubForDependencies(NewGiterminismManagerStub(NewLocalGitRepoStub("9d8059842b6fde712c58315ca0ab4713d90761c0"), NewGiterminismInspectorStub()), data.TestDependencies.Dependencies) containerBackend := NewContainerBackendMock() - dockerStages, dockerMetaArgs := testDockerfileToDockerStages(data.Dockerfile) + dockerStages, dockerMetaArgs := testDockerfileToDockerStages(data.DockerfileData) - stage := newTestDockerfileStage(data.Dockerfile, data.Target, data.BuildArgs, dockerStages, dockerMetaArgs, data.TestDependencies.Dependencies) + stage := newTestFullDockerfileStage(data.DockerfileData, data.Target, data.BuildArgs, dockerStages, dockerMetaArgs, data.TestDependencies.Dependencies) img := NewLegacyImageStub() stageBuilder := stage_builder.NewStageBuilder(containerBackend, nil, img) @@ -89,7 +89,7 @@ var _ = Describe("DockerfileStage", func() { Entry("should calculate dockerfile stage digest when no dependencies are set", TestDockerfileDependencies{ - Dockerfile: []byte(` + DockerfileData: []byte(` FROM alpine:latest RUN echo hello `), @@ -100,7 +100,7 @@ RUN echo hello Entry("should not change dockerfile stage digest when dependencies are defined, but build args not used", TestDockerfileDependencies{ - Dockerfile: []byte(` + DockerfileData: []byte(` FROM alpine:latest RUN echo hello `), @@ -122,7 +122,7 @@ RUN echo hello Entry("should change dockerfile stage digest when dependant image build args used in the Dockerfile", TestDockerfileDependencies{ - Dockerfile: []byte(` + DockerfileData: []byte(` FROM alpine:latest ARG IMAGE_ONE_NAME @@ -154,7 +154,7 @@ RUN echo {"name": "${IMAGE_ONE_NAME}", "repo": "${IMAGE_ONE_REPO}", "tag": "${IM Entry("should change dockerfile stage digest when dependant image name changed", TestDockerfileDependencies{ - Dockerfile: []byte(` + DockerfileData: []byte(` FROM alpine:latest ARG IMAGE_ONE_NAME @@ -186,7 +186,7 @@ RUN echo {"name": "${IMAGE_ONE_NAME}", "repo": "${IMAGE_ONE_REPO}", "tag": "${IM Entry("should change dockerfile stage digest when dependant image id changed", TestDockerfileDependencies{ - Dockerfile: []byte(` + DockerfileData: []byte(` FROM alpine:latest ARG IMAGE_ONE_NAME @@ -218,7 +218,7 @@ RUN echo {"name": "${IMAGE_ONE_NAME}", "repo": "${IMAGE_ONE_REPO}", "tag": "${IM Entry("should calculate dockerfile stage digest when no dependencies are set", TestDockerfileDependencies{ - Dockerfile: []byte(` + DockerfileData: []byte(` ARG BASE_IMAGE=alpine:latest FROM ${BASE_IMAGE} @@ -231,7 +231,7 @@ RUN echo hello Entry("should allow usage of dependency image as a base image in the dockerfile", TestDockerfileDependencies{ - Dockerfile: []byte(` + DockerfileData: []byte(` ARG BASE_IMAGE=alpine:latest FROM ${BASE_IMAGE} @@ -255,7 +255,7 @@ RUN echo hello Entry("should change dockerfile stage digest when base dependency image has changed", TestDockerfileDependencies{ - Dockerfile: []byte(` + DockerfileData: []byte(` ARG BASE_IMAGE=alpine:latest FROM ${BASE_IMAGE} @@ -293,7 +293,7 @@ RUN echo hello dockerStages, dockerMetaArgs := testDockerfileToDockerStages(dockerfile) - stage := newTestDockerfileStage(dockerfile, "", nil, dockerStages, dockerMetaArgs, nil) + stage := newTestFullDockerfileStage(dockerfile, "", nil, dockerStages, dockerMetaArgs, nil) containerBackend := NewContainerBackendMock() @@ -326,7 +326,7 @@ RUN --mount=type=bind,from=build,source=/usr/local/test_project/dist,target=/usr dockerStages, dockerMetaArgs := testDockerfileToDockerStages(dockerfile) - stage := newTestDockerfileStage(dockerfile, "", nil, dockerStages, dockerMetaArgs, nil) + stage := newTestFullDockerfileStage(dockerfile, "", nil, dockerStages, dockerMetaArgs, nil) containerBackend := NewContainerBackendMock() @@ -354,9 +354,9 @@ RUN --mount=type=bind,from=build,source=/usr/local/test_project/dist,target=/usr }) type TestDockerfileDependencies struct { - Dockerfile []byte - Target string - BuildArgs map[string]interface{} + DockerfileData []byte + Target string + BuildArgs map[string]interface{} TestDependencies *TestDependencies } diff --git a/pkg/build/stage/docker_instructions.go b/pkg/build/stage/stapel_docker_instructions.go similarity index 81% rename from pkg/build/stage/docker_instructions.go rename to pkg/build/stage/stapel_docker_instructions.go index 6ce8cfe2ab..7c4cf0606c 100644 --- a/pkg/build/stage/docker_instructions.go +++ b/pkg/build/stage/stapel_docker_instructions.go @@ -11,28 +11,28 @@ import ( "github.com/werf/werf/pkg/util" ) -func GenerateDockerInstructionsStage(imageConfig *config.StapelImage, baseStageOptions *NewBaseStageOptions) *DockerInstructionsStage { +func GenerateStapelDockerInstructionsStage(imageConfig *config.StapelImage, baseStageOptions *NewBaseStageOptions) *StapelDockerInstructionsStage { if imageConfig.Docker != nil { - return newDockerInstructionsStage(imageConfig.Docker, baseStageOptions) + return newStapelDockerInstructionsStage(imageConfig.Docker, baseStageOptions) } return nil } -func newDockerInstructionsStage(instructions *config.Docker, baseStageOptions *NewBaseStageOptions) *DockerInstructionsStage { - s := &DockerInstructionsStage{} +func newStapelDockerInstructionsStage(instructions *config.Docker, baseStageOptions *NewBaseStageOptions) *StapelDockerInstructionsStage { + s := &StapelDockerInstructionsStage{} s.instructions = instructions s.BaseStage = newBaseStage(DockerInstructions, baseStageOptions) return s } -type DockerInstructionsStage struct { +type StapelDockerInstructionsStage struct { *BaseStage instructions *config.Docker } -func (s *DockerInstructionsStage) GetDependencies(_ context.Context, c Conveyor, backend container_backend.ContainerBackend, _, _ *StageImage) (string, error) { +func (s *StapelDockerInstructionsStage) GetDependencies(_ context.Context, c Conveyor, backend container_backend.ContainerBackend, _, _ *StageImage) (string, error) { var args []string if c.UseLegacyStapelBuilder(backend) && s.instructions.ExactValues { @@ -66,7 +66,7 @@ func mapToSortedArgs(h map[string]string) (result []string) { return } -func (s *DockerInstructionsStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { +func (s *StapelDockerInstructionsStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error { if c.UseLegacyStapelBuilder(cr) { stageImage.Image.SetCommitChangeOptions(container_backend.LegacyCommitChangeOptions{ExactValues: s.instructions.ExactValues}) } diff --git a/pkg/config/image_from_dockerfile.go b/pkg/config/image_from_dockerfile.go index bddc85cbe3..3221a0571d 100644 --- a/pkg/config/image_from_dockerfile.go +++ b/pkg/config/image_from_dockerfile.go @@ -18,6 +18,7 @@ type ImageFromDockerfile struct { Network string SSH string Dependencies []*Dependency + Staged bool raw *rawImageFromDockerfile } diff --git a/pkg/config/raw_image_from_dockerfile.go b/pkg/config/raw_image_from_dockerfile.go index d6efe60a2c..ebce9985d6 100644 --- a/pkg/config/raw_image_from_dockerfile.go +++ b/pkg/config/raw_image_from_dockerfile.go @@ -18,6 +18,7 @@ type rawImageFromDockerfile struct { Network string `yaml:"network,omitempty"` SSH string `yaml:"ssh,omitempty"` RawDependencies []*rawDependency `yaml:"dependencies,omitempty"` + Staged bool `yaml:"staged,omitempty"` doc *doc `yaml:"-"` // parent @@ -125,6 +126,7 @@ func (c *rawImageFromDockerfile) toImageFromDockerfileDirective(giterminismManag image.Dependencies = append(image.Dependencies, dependencyDirective) } + image.Staged = c.Staged image.raw = c if err := image.validate(giterminismManager); err != nil { diff --git a/pkg/container_backend/buildah_backend.go b/pkg/container_backend/buildah_backend.go index f0164f5f2b..3c54060e7c 100644 --- a/pkg/container_backend/buildah_backend.go +++ b/pkg/container_backend/buildah_backend.go @@ -20,6 +20,7 @@ import ( copyrec "github.com/werf/copy-recurse" "github.com/werf/logboek" "github.com/werf/werf/pkg/buildah" + "github.com/werf/werf/pkg/dockerfile" "github.com/werf/werf/pkg/image" "github.com/werf/werf/pkg/path_matcher" "github.com/werf/werf/pkg/util" @@ -417,7 +418,7 @@ func (runtime *BuildahBackend) applyDependenciesImports(ctx context.Context, con return nil } -func (runtime *BuildahBackend) BuildDockerfileStage(ctx context.Context, image ImageInterface, contextTar io.ReadCloser, opts BuildDockerfileStageOptions, commands ...any) (string, error) { +func (runtime *BuildahBackend) BuildDockerfileStage(ctx context.Context, image ImageInterface, contextTar io.ReadCloser, opts BuildDockerfileStageOptions, instructions ...any) (string, error) { var container *containerDesc if c, err := runtime.createContainers(ctx, []string{image.Name()}); err != nil { return "", err @@ -453,108 +454,108 @@ func (runtime *BuildahBackend) BuildDockerfileStage(ctx context.Context, image I }() logboek.Context(ctx).Debug().LogF("Executing commands for build container %s\n", container.Name) - for _, command := range commands { - switch cmd := command.(type) { - case CommandLabel: + for _, i := range instructions { + switch instruction := i.(type) { + case dockerfile.InstructionLabel: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - Labels: cmd.LabelsAsList(), + Labels: instruction.LabelsAsList(), }); err != nil { - return "", fmt.Errorf("error setting labels %v for container %s: %w", cmd.LabelsAsList(), container.Name, err) + return "", fmt.Errorf("error setting labels %v for container %s: %w", instruction.LabelsAsList(), container.Name, err) } - case CommandExpose: + case dockerfile.InstructionExpose: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - Expose: cmd.Ports, + Expose: instruction.Ports, }); err != nil { - return "", fmt.Errorf("error setting exposed ports %v for container %s: %w", cmd.Ports, container.Name, err) + return "", fmt.Errorf("error setting exposed ports %v for container %s: %w", instruction.Ports, container.Name, err) } - case CommandVolume: + case dockerfile.InstructionVolume: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - Volumes: cmd.Volumes, + Volumes: instruction.Volumes, }); err != nil { - return "", fmt.Errorf("error setting volumes %v for container %s: %w", cmd.Volumes, container.Name, err) + return "", fmt.Errorf("error setting volumes %v for container %s: %w", instruction.Volumes, container.Name, err) } - case CommandOnBuild: + case dockerfile.InstructionOnBuild: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - OnBuild: cmd.Instruction, + OnBuild: instruction.Instruction, }); err != nil { - return "", fmt.Errorf("error setting onbuild %v for container %s: %w", cmd.Instruction, container.Name, err) + return "", fmt.Errorf("error setting onbuild %v for container %s: %w", instruction.Instruction, container.Name, err) } - case CommandStopSignal: + case dockerfile.InstructionStopSignal: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - StopSignal: cmd.Signal, + StopSignal: instruction.Signal, }); err != nil { - return "", fmt.Errorf("error setting stop signal %v for container %s: %w", cmd.Signal, container.Name, err) + return "", fmt.Errorf("error setting stop signal %v for container %s: %w", instruction.Signal, container.Name, err) } - case CommandShell: + case dockerfile.InstructionShell: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - Shell: cmd.Shell, + Shell: instruction.Shell, }); err != nil { - return "", fmt.Errorf("error setting shell %v for container %s: %w", cmd.Shell, container.Name, err) + return "", fmt.Errorf("error setting shell %v for container %s: %w", instruction.Shell, container.Name, err) } - case CommandEnv: + case dockerfile.InstructionEnv: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - Envs: cmd.Envs, + Envs: instruction.Envs, }); err != nil { - return "", fmt.Errorf("error setting envs %v for container %s: %w", cmd.Envs, container.Name, err) + return "", fmt.Errorf("error setting envs %v for container %s: %w", instruction.Envs, container.Name, err) } - case CommandRun: - if err := runtime.buildah.RunCommand(ctx, container.Name, cmd.Command, buildah.RunCommandOpts{ + case dockerfile.InstructionRun: + if err := runtime.buildah.RunCommand(ctx, container.Name, instruction.Command, buildah.RunCommandOpts{ // FIXME(ilya-lesikov): should we suppress or not? CommonOpts: runtime.getBuildahCommonOpts(ctx, true), }); err != nil { - return "", fmt.Errorf("error running command %v for container %s: %w", cmd.Command, container.Name, err) + return "", fmt.Errorf("error running command %v for container %s: %w", instruction.Command, container.Name, err) } - case CommandCopy: - if err := runtime.buildah.Copy(ctx, container.Name, contextTmpDir, cmd.Src, cmd.Dst, buildah.CopyOpts{ + case dockerfile.InstructionCopy: + if err := runtime.buildah.Copy(ctx, container.Name, contextTmpDir, instruction.Src, instruction.Dst, buildah.CopyOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - From: cmd.From, + From: instruction.From, }); err != nil { - return "", fmt.Errorf("error copying %v to %s for container %s: %w", cmd.Src, cmd.Dst, container.Name, err) + return "", fmt.Errorf("error copying %v to %s for container %s: %w", instruction.Src, instruction.Dst, container.Name, err) } - case CommandAdd: - if err := runtime.buildah.Add(ctx, container.Name, cmd.Src, cmd.Dst, buildah.AddOpts{ + case dockerfile.InstructionAdd: + if err := runtime.buildah.Add(ctx, container.Name, instruction.Src, instruction.Dst, buildah.AddOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), ContextDir: contextTmpDir, }); err != nil { - return "", fmt.Errorf("error adding %v to %s for container %s: %w", cmd.Src, cmd.Dst, container.Name, err) + return "", fmt.Errorf("error adding %v to %s for container %s: %w", instruction.Src, instruction.Dst, container.Name, err) } - case CommandUser: + case dockerfile.InstructionUser: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - User: cmd.User, + User: instruction.User, }); err != nil { - return "", fmt.Errorf("error setting user %s for container %s: %w", cmd.User, container.Name, err) + return "", fmt.Errorf("error setting user %s for container %s: %w", instruction.User, container.Name, err) } - case CommandWorkdir: + case dockerfile.InstructionWorkdir: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - Workdir: cmd.Workdir, + Workdir: instruction.Workdir, }); err != nil { - return "", fmt.Errorf("error setting workdir %s for container %s: %w", cmd.Workdir, container.Name, err) + return "", fmt.Errorf("error setting workdir %s for container %s: %w", instruction.Workdir, container.Name, err) } - case CommandEntrypoint: + case dockerfile.InstructionEntrypoint: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - Entrypoint: cmd.Entrypoint, + Entrypoint: instruction.Entrypoint, }); err != nil { - return "", fmt.Errorf("error setting entrypoint %v for container %s: %w", cmd.Entrypoint, container.Name, err) + return "", fmt.Errorf("error setting entrypoint %v for container %s: %w", instruction.Entrypoint, container.Name, err) } - case CommandCmd: + case dockerfile.InstructionCmd: if err := runtime.buildah.Config(ctx, container.Name, buildah.ConfigOpts{ CommonOpts: runtime.getBuildahCommonOpts(ctx, true), - Cmd: cmd.Cmd, + Cmd: instruction.Cmd, }); err != nil { - return "", fmt.Errorf("error setting cmd %v for container %s: %w", cmd.Cmd, container.Name, err) + return "", fmt.Errorf("error setting cmd %v for container %s: %w", instruction.Cmd, container.Name, err) } default: - return "", fmt.Errorf("invalid command type: %T", command) + return "", fmt.Errorf("invalid command type: %T", i) } } diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 70c3590273..a99afa93e6 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -1,13 +1,60 @@ package dockerfile -import "context" +import ( + "bytes" + "context" + "fmt" -func NewDockerfile(dockerfile []byte) *Dockerfile { - return &Dockerfile{dockerfile: dockerfile} + "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/frontend/dockerfile/parser" +) + +func ParseDockerfile(dockerfile []byte, opts DockerfileOptions) (*Dockerfile, error) { + p, err := parser.Parse(bytes.NewReader(dockerfile)) + if err != nil { + return nil, fmt.Errorf("parsing dockerfile data: %w", err) + } + + dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST) + if err != nil { + return nil, fmt.Errorf("parsing instructions tree: %w", err) + } + + // FIXME(staged-dockerfile): is this needed? + ResolveDockerStagesFromValue(dockerStages) + + dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target) + if err != nil { + return nil, fmt.Errorf("determine target stage: %w", err) + } + + return newDockerfile(dockerStages, dockerMetaArgs, dockerTargetIndex, opts), nil +} + +type DockerfileOptions struct { + Target string + BuildArgs map[string]string + AddHost []string + Network string + SSH string +} + +func newDockerfile(dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dockerTargetStageIndex int, opts DockerfileOptions) *Dockerfile { + return &Dockerfile{ + DockerfileOptions: opts, + + dockerStages: dockerStages, + dockerMetaArgs: dockerMetaArgs, + dockerTargetStageIndex: dockerTargetStageIndex, + } } type Dockerfile struct { - dockerfile []byte + DockerfileOptions + + dockerStages []instructions.Stage + dockerMetaArgs []instructions.ArgCommand + dockerTargetStageIndex int } func (dockerfile *Dockerfile) GroupStagesByIndependentSets(ctx context.Context) ([][]*DockerfileStage, error) { diff --git a/pkg/dockerfile/dockerfile_stage.go b/pkg/dockerfile/dockerfile_stage.go index 56a98d8c1f..edaedcc818 100644 --- a/pkg/dockerfile/dockerfile_stage.go +++ b/pkg/dockerfile/dockerfile_stage.go @@ -4,4 +4,7 @@ func NewDockerfileStage() *DockerfileStage { return &DockerfileStage{} } -type DockerfileStage struct{} +type DockerfileStage struct { + Dockerfile *Dockerfile + DependenciesStages []*DockerfileStage +} diff --git a/pkg/build/dockerfile_helpers/dockerfile_helpers.go b/pkg/dockerfile/helpers.go similarity index 97% rename from pkg/build/dockerfile_helpers/dockerfile_helpers.go rename to pkg/dockerfile/helpers.go index 740811aa2f..7cdfcf9242 100644 --- a/pkg/build/dockerfile_helpers/dockerfile_helpers.go +++ b/pkg/dockerfile/helpers.go @@ -1,4 +1,4 @@ -package dockerfile_helpers +package dockerfile import ( "fmt" diff --git a/pkg/container_backend/dockerfile_commands.go b/pkg/dockerfile/instructions.go similarity index 56% rename from pkg/container_backend/dockerfile_commands.go rename to pkg/dockerfile/instructions.go index 09f71aaf4b..0cdcc37583 100644 --- a/pkg/container_backend/dockerfile_commands.go +++ b/pkg/dockerfile/instructions.go @@ -1,63 +1,63 @@ -package container_backend +package dockerfile import "fmt" -type CommandEnv struct { +type InstructionEnv struct { Envs map[string]string } -type CommandCopy struct { +type InstructionCopy struct { From string Src []string Dst string } -type CommandAdd struct { +type InstructionAdd struct { Src []string Dst string } -type CommandRun struct { +type InstructionRun struct { Command []string } -type CommandEntrypoint struct { +type InstructionEntrypoint struct { Entrypoint []string } -type CommandCmd struct { +type InstructionCmd struct { Cmd []string } -type CommandUser struct { +type InstructionUser struct { User string } -type CommandWorkdir struct { +type InstructionWorkdir struct { Workdir string } -type CommandExpose struct { +type InstructionExpose struct { Ports []string } -type CommandVolume struct { +type InstructionVolume struct { Volumes []string } -type CommandOnBuild struct { +type InstructionOnBuild struct { Instruction string } -type CommandStopSignal struct { +type InstructionStopSignal struct { Signal string } -type CommandShell struct { +type InstructionShell struct { Shell []string } -type CommandHealthcheck struct { +type InstructionHealthcheck struct { Type HealthcheckType Command string } @@ -70,11 +70,11 @@ var ( HealthcheckTypeCmdShell HealthcheckType = "CMD-SHELL" ) -type CommandLabel struct { +type InstructionLabel struct { Labels map[string]string } -func (c *CommandLabel) LabelsAsList() []string { +func (c *InstructionLabel) LabelsAsList() []string { var labels []string for k, v := range c.Labels { labels = append(labels, fmt.Sprintf("%s=%s", k, v))