diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 372a9dad05..47592064f2 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -225,7 +225,9 @@ func (phase *BuildPhase) ImageProcessingShouldBeStopped(_ context.Context, _ *im func (phase *BuildPhase) BeforeImageStages(ctx context.Context, img *image.Image) (deferFn func(), err error) { phase.StagesIterator = NewStagesIterator(phase.Conveyor) - img.SetupBaseImage() + if err := img.SetupBaseImage(); err != nil { + return nil, err + } if img.UsesBuildContext() { phase.buildContextArchive = image.NewBuildContextArchive(phase.Conveyor.giterminismManager, img.TmpDir) diff --git a/pkg/build/image/dockerfile.go b/pkg/build/image/dockerfile.go index 430bb5f062..442a1f29e1 100644 --- a/pkg/build/image/dockerfile.go +++ b/pkg/build/image/dockerfile.go @@ -30,11 +30,12 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig } d, err := frontend.ParseDockerfileWithBuildkit(dockerfileData, dockerfile.DockerfileOptions{ - Target: dockerfileImageConfig.Target, - BuildArgs: util.MapStringInterfaceToMapStringString(dockerfileImageConfig.Args), - AddHost: dockerfileImageConfig.AddHost, - Network: dockerfileImageConfig.Network, - SSH: dockerfileImageConfig.SSH, + Target: dockerfileImageConfig.Target, + BuildArgs: util.MapStringInterfaceToMapStringString(dockerfileImageConfig.Args), + AddHost: dockerfileImageConfig.AddHost, + Network: dockerfileImageConfig.Network, + SSH: dockerfileImageConfig.SSH, + DependenciesArgsKeys: stage.GetDependenciesArgsKeys(dockerfileImageConfig.Dependencies), }) if err != nil { return nil, fmt.Errorf("unable to parse dockerfile %s: %w", relDockerfilePath, err) @@ -101,10 +102,11 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile, var err error if baseStg := cfg.FindStage(stg.BaseName); baseStg != nil { img, err = NewImage(ctx, item.WerfImageName, StageAsBaseImage, ImageOptions{ - IsDockerfileImage: true, - DockerfileImageConfig: dockerfileImageConfig, - CommonImageOptions: opts, - BaseImageName: baseStg.WerfImageName(), + IsDockerfileImage: true, + DockerfileImageConfig: dockerfileImageConfig, + CommonImageOptions: opts, + BaseImageName: baseStg.WerfImageName(), + DockerfileExpanderFactory: stg.ExpanderFactory, }) if err != nil { return nil, fmt.Errorf("unable to map stage %s to werf image %q: %w", stg.LogName(), dockerfileImageConfig.Name, err) @@ -113,13 +115,14 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile, appendQueue(baseStg.WerfImageName(), baseStg, item.Level+1) } else { img, err = NewImage(ctx, item.WerfImageName, ImageFromRegistryAsBaseImage, ImageOptions{ - IsDockerfileImage: true, - DockerfileImageConfig: dockerfileImageConfig, - CommonImageOptions: opts, - BaseImageReference: targetStage.BaseName, + IsDockerfileImage: true, + DockerfileImageConfig: dockerfileImageConfig, + CommonImageOptions: opts, + BaseImageReference: stg.BaseName, + DockerfileExpanderFactory: stg.ExpanderFactory, }) if err != nil { - return nil, fmt.Errorf("unable to map stage %s to werf image %q: %w", targetStage.LogName(), dockerfileImageConfig.Name, err) + return nil, fmt.Errorf("unable to map stage %s to werf image %q: %w", stg.LogName(), dockerfileImageConfig.Name, err) } } @@ -136,7 +139,6 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile, var stg stage.Interface switch typedInstr := any(instr).(type) { case *dockerfile.DockerfileStageInstruction[*instructions.ArgCommand]: - // TODO(staged-dockerfile): support build-args at this level or dockerfile pkg level (?) continue case *dockerfile.DockerfileStageInstruction[*instructions.AddCommand]: stg = stage_instruction.NewAdd(stageName, typedInstr, dockerfileImageConfig.Dependencies, !isFirstStage, baseStageOptions) diff --git a/pkg/build/image/image.go b/pkg/build/image/image.go index 2b5962c58a..7e58ad2eec 100644 --- a/pkg/build/image/image.go +++ b/pkg/build/image/image.go @@ -13,6 +13,7 @@ import ( "github.com/werf/werf/pkg/config" "github.com/werf/werf/pkg/container_backend" "github.com/werf/werf/pkg/docker_registry" + "github.com/werf/werf/pkg/dockerfile" "github.com/werf/werf/pkg/giterminism_manager" "github.com/werf/werf/pkg/image" "github.com/werf/werf/pkg/logging" @@ -43,9 +44,10 @@ type ImageOptions struct { IsArtifact, IsDockerfileImage bool DockerfileImageConfig *config.ImageFromDockerfile - BaseImageReference string - BaseImageName string - FetchLatestBaseImage bool + BaseImageReference string + BaseImageName string + FetchLatestBaseImage bool + DockerfileExpanderFactory dockerfile.ExpanderFactory } func NewImage(ctx context.Context, name string, baseImageType BaseImageType, opts ImageOptions) (*Image, error) { @@ -62,9 +64,10 @@ func NewImage(ctx context.Context, name string, baseImageType BaseImageType, opt IsDockerfileImage: opts.IsDockerfileImage, DockerfileImageConfig: opts.DockerfileImageConfig, - baseImageType: baseImageType, - baseImageReference: opts.BaseImageReference, - baseImageName: opts.BaseImageName, + baseImageType: baseImageType, + baseImageReference: opts.BaseImageReference, + baseImageName: opts.BaseImageName, + dockerfileExpanderFactory: opts.DockerfileExpanderFactory, } if opts.FetchLatestBaseImage { @@ -89,9 +92,10 @@ type Image struct { contentDigest string rebuilt bool - baseImageType BaseImageType - baseImageReference string - baseImageName string + baseImageType BaseImageType + baseImageReference string + baseImageName string + dockerfileExpanderFactory dockerfile.ExpanderFactory baseImageRepoId string baseStageImage *stage.StageImage @@ -197,18 +201,28 @@ func (i *Image) GetRebuilt() bool { return i.rebuilt } -func (i *Image) SetupBaseImage() { +func (i *Image) SetupBaseImage() error { switch i.baseImageType { case StageAsBaseImage: i.stageAsBaseImage = i.Conveyor.GetImage(i.baseImageName).GetLastNonEmptyStage() i.baseImageReference = i.stageAsBaseImage.GetStageImage().Image.Name() i.baseStageImage = i.stageAsBaseImage.GetStageImage() case ImageFromRegistryAsBaseImage: + if i.IsDockerfileImage && i.dockerfileExpanderFactory != nil { + dependenciesArgs := stage.ResolveDependenciesArgs(i.DockerfileImageConfig.Dependencies, i.Conveyor) + ref, err := i.dockerfileExpanderFactory.GetExpander(dockerfile.ExpandOptions{SkipUnsetEnv: false}).ProcessWordWithMap(i.baseImageReference, dependenciesArgs) + if err != nil { + return fmt.Errorf("unable to expand dockerfile base image reference %q: %w", i.baseImageReference, err) + } + i.baseImageReference = ref + } i.baseStageImage = i.Conveyor.GetOrCreateStageImage(i.baseImageReference, nil, nil, i) case NoBaseImage: default: panic(fmt.Sprintf("unknown base image type %q", i.baseImageType)) } + + return nil } // TODO(staged-dockerfile): this is only for compatibility with stapel-builder logic, and this should be unified with new staged-dockerfile logic diff --git a/pkg/build/stage/base.go b/pkg/build/stage/base.go index f9c2eb17ce..9c845a9922 100644 --- a/pkg/build/stage/base.go +++ b/pkg/build/stage/base.go @@ -122,6 +122,10 @@ func (s *BaseStage) LogDetailedName() string { return fmt.Sprintf("%s/%s", imageName, s.Name()) } +func (s *BaseStage) ImageName() string { + return s.imageName +} + func (s *BaseStage) Name() StageName { if s.name != "" { return s.name diff --git a/pkg/build/stage/dependencies_utils.go b/pkg/build/stage/dependencies_utils.go new file mode 100644 index 0000000000..22c51f6057 --- /dev/null +++ b/pkg/build/stage/dependencies_utils.go @@ -0,0 +1,40 @@ +package stage + +import ( + "github.com/werf/werf/pkg/config" + "github.com/werf/werf/pkg/image" +) + +func GetDependenciesArgsKeys(dependencies []*config.Dependency) (res []string) { + for _, dep := range dependencies { + for _, imp := range dep.Imports { + res = append(res, imp.TargetBuildArg) + } + } + return +} + +func ResolveDependenciesArgs(dependencies []*config.Dependency, c Conveyor) map[string]string { + resolved := make(map[string]string) + + for _, dep := range dependencies { + depImageName := c.GetImageNameForLastImageStage(dep.ImageName) + depImageID := c.GetImageIDForLastImageStage(dep.ImageName) + depImageRepo, depImageTag := image.ParseRepositoryAndTag(depImageName) + + for _, imp := range dep.Imports { + switch imp.Type { + case config.ImageRepoImport: + resolved[imp.TargetBuildArg] = depImageRepo + case config.ImageTagImport: + resolved[imp.TargetBuildArg] = depImageTag + case config.ImageNameImport: + resolved[imp.TargetBuildArg] = depImageName + case config.ImageIDImport: + resolved[imp.TargetBuildArg] = depImageID + } + } + } + + return resolved +} diff --git a/pkg/build/stage/full_dockerfile.go b/pkg/build/stage/full_dockerfile.go index 84ca1a8a2d..4b6f770de7 100644 --- a/pkg/build/stage/full_dockerfile.go +++ b/pkg/build/stage/full_dockerfile.go @@ -174,31 +174,6 @@ func (ds *DockerStages) resolveDockerMetaArg(key, value string, resolvedDockerMe return resolvedKey, resolvedValue, err } -func resolveDependenciesArgsHash(dependencies []*config.Dependency, c Conveyor) map[string]string { - resolved := make(map[string]string) - - for _, dep := range dependencies { - depImageName := c.GetImageNameForLastImageStage(dep.ImageName) - depImageID := c.GetImageIDForLastImageStage(dep.ImageName) - depImageRepo, depImageTag := image.ParseRepositoryAndTag(depImageName) - - for _, img := range dep.Imports { - switch img.Type { - case config.ImageRepoImport: - resolved[img.TargetBuildArg] = depImageRepo - case config.ImageTagImport: - resolved[img.TargetBuildArg] = depImageTag - case config.ImageNameImport: - resolved[img.TargetBuildArg] = depImageName - case config.ImageIDImport: - resolved[img.TargetBuildArg] = depImageID - } - } - } - - return resolved -} - // resolveDockerStageArg function sets dependency arg value, or --build-arg value, or resolved dockerfile stage ARG value, or resolved meta ARG value (if stage ARG value is empty) func (ds *DockerStages) resolveDockerStageArg(dockerStageID int, key, value string, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string) (string, string, error) { resolvedKey, err := ds.ShlexProcessWordWithStageArgsAndEnvs(dockerStageID, key) @@ -320,7 +295,7 @@ type dockerfileInstructionInterface interface { } func (s *FullDockerfileStage) FetchDependencies(ctx context.Context, c Conveyor, containerBackend container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error { - resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c) + resolvedDependenciesArgsHash := ResolveDependenciesArgs(s.dependencies, c) resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash) if err != nil { @@ -412,7 +387,7 @@ func isUnsupportedMediaTypeError(err error) bool { var errImageNotExistLocally = errors.New("IMAGE_NOT_EXIST_LOCALLY") func (s *FullDockerfileStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) { - resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c) + resolvedDependenciesArgsHash := ResolveDependenciesArgs(s.dependencies, c) resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash) if err != nil { @@ -732,7 +707,7 @@ func (s *FullDockerfileStage) SetupDockerImageBuilder(b stage_builder.Dockerfile } } - resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c) + resolvedDependenciesArgsHash := ResolveDependenciesArgs(s.dependencies, c) if len(resolvedDependenciesArgsHash) > 0 { for key, value := range resolvedDependenciesArgsHash { b.AppendBuildArgs(fmt.Sprintf("%s=%v", key, value)) diff --git a/pkg/build/stage/instruction/base.go b/pkg/build/stage/instruction/base.go index a4666c96e3..e6fbdd993e 100644 --- a/pkg/build/stage/instruction/base.go +++ b/pkg/build/stage/instruction/base.go @@ -54,7 +54,9 @@ func (stg *Base[T, BT]) PrepareImage(ctx context.Context, c stage.Conveyor, cb c } func (stg *Base[T, BT]) ExpandInstruction(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error { - return nil + dependenciesArgs := stage.ResolveDependenciesArgs(stg.dependencies, c) + // NOTE: do not skip unset envs during second stage expansion + return stg.instruction.Expand(dependenciesArgs, dockerfile.ExpandOptions{SkipUnsetEnv: false}) } type InstructionExpander interface { diff --git a/pkg/build/stage/instruction/copy.go b/pkg/build/stage/instruction/copy.go index f47cb796d2..559370591d 100644 --- a/pkg/build/stage/instruction/copy.go +++ b/pkg/build/stage/instruction/copy.go @@ -23,6 +23,10 @@ func NewCopy(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*ins } func (stg *Copy) ExpandInstruction(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error { + if err := stg.Base.ExpandInstruction(ctx, c, cb, prevBuiltImage, stageImage, buildContextArchive); err != nil { + return err + } + if stg.instruction.Data.From != "" { if ds := stg.instruction.GetDependencyByStageRef(stg.instruction.Data.From); ds != nil { depStageImageName := c.GetImageNameForLastImageStage(ds.WerfImageName()) diff --git a/pkg/container_backend/instruction/run.go b/pkg/container_backend/instruction/run.go index 8d2266b5cc..3a9b68bd01 100644 --- a/pkg/container_backend/instruction/run.go +++ b/pkg/container_backend/instruction/run.go @@ -3,9 +3,11 @@ package instruction import ( "context" "fmt" + "strings" "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/werf/logboek" "github.com/werf/werf/pkg/buildah" "github.com/werf/werf/pkg/container_backend" ) @@ -55,6 +57,8 @@ func (i *Run) Apply(ctx context.Context, containerName string, drv buildah.Build addCapabilities = []string{"all"} } + logboek.Context(ctx).Default().LogF("$ %s\n", strings.Join(i.CmdLine, " ")) + if err := drv.RunCommand(ctx, containerName, i.CmdLine, buildah.RunCommandOpts{ CommonOpts: drvOpts, ContextDir: contextDir, diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 1cf3fd81ef..f314b63c6b 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -5,11 +5,12 @@ import ( ) type DockerfileOptions struct { - Target string - BuildArgs map[string]string - AddHost []string - Network string - SSH string + Target string + BuildArgs map[string]string + DependenciesArgsKeys []string + AddHost []string + Network string + SSH string } func NewDockerfile(stages []*DockerfileStage, opts DockerfileOptions) *Dockerfile { diff --git a/pkg/dockerfile/dockerfile_stage.go b/pkg/dockerfile/dockerfile_stage.go index be9bdab739..3f0a7b739e 100644 --- a/pkg/dockerfile/dockerfile_stage.go +++ b/pkg/dockerfile/dockerfile_stage.go @@ -8,14 +8,21 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/instructions" ) -func NewDockerfileStage(index int, baseName, stageName string, instructions []DockerfileStageInstructionInterface, platform string) *DockerfileStage { - return &DockerfileStage{BaseName: baseName, StageName: stageName, Instructions: instructions, Platform: platform} +func NewDockerfileStage(index int, baseName, stageName string, instructions []DockerfileStageInstructionInterface, platform string, expanderFactory ExpanderFactory) *DockerfileStage { + return &DockerfileStage{ + ExpanderFactory: expanderFactory, + BaseName: baseName, + StageName: stageName, + Instructions: instructions, + Platform: platform, + } } type DockerfileStage struct { - Dockerfile *Dockerfile - Dependencies []*DockerfileStage - BaseStage *DockerfileStage + Dockerfile *Dockerfile + Dependencies []*DockerfileStage + BaseStage *DockerfileStage + ExpanderFactory ExpanderFactory BaseName string Index int diff --git a/pkg/dockerfile/frontend/buildkit_dockerfile.go b/pkg/dockerfile/frontend/buildkit_dockerfile.go index 5021a14703..856511c58d 100644 --- a/pkg/dockerfile/frontend/buildkit_dockerfile.go +++ b/pkg/dockerfile/frontend/buildkit_dockerfile.go @@ -8,7 +8,6 @@ import ( "github.com/moby/buildkit/frontend/dockerfile/instructions" "github.com/moby/buildkit/frontend/dockerfile/parser" - "github.com/moby/buildkit/frontend/dockerfile/shell" "github.com/werf/werf/pkg/dockerfile" ) @@ -24,16 +23,16 @@ func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.Dockerf return nil, fmt.Errorf("parsing instructions tree: %w", err) } - shlex := shell.NewLex(p.EscapeToken) + expanderFactory := NewShlexExpanderFactory(p.EscapeToken) - metaArgs, err := processMetaArgs(dockerMetaArgs, opts.BuildArgs, shlex) + metaArgs, err := resolveMetaArgs(dockerMetaArgs, opts.BuildArgs, opts.DependenciesArgsKeys, expanderFactory) if err != nil { return nil, fmt.Errorf("unable to process meta args: %w", err) } var stages []*dockerfile.DockerfileStage for i, dockerStage := range dockerStages { - name, err := shlex.ProcessWordWithMap(dockerStage.BaseName, metaArgs) + name, err := expanderFactory.GetExpander(dockerfile.ExpandOptions{SkipUnsetEnv: true}).ProcessWordWithMap(dockerStage.BaseName, metaArgs) if err != nil { return nil, fmt.Errorf("unable to expand docker stage base image name %q: %w", dockerStage.BaseName, err) } @@ -44,7 +43,7 @@ func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.Dockerf // TODO(staged-dockerfile): support meta-args expansion for dockerStage.Platform - if stage, err := NewDockerfileStageFromBuildkitStage(i, dockerStage, shlex, metaArgs, opts.BuildArgs); err != nil { + if stage, err := NewDockerfileStageFromBuildkitStage(i, dockerStage, expanderFactory, metaArgs, opts.BuildArgs, opts.DependenciesArgsKeys); err != nil { return nil, fmt.Errorf("error converting buildkit stage to dockerfile stage: %w", err) } else { stages = append(stages, stage) @@ -60,11 +59,11 @@ func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.Dockerf return d, nil } -func NewDockerfileStageFromBuildkitStage(index int, stage instructions.Stage, shlex *shell.Lex, metaArgs, buildArgs map[string]string) (*dockerfile.DockerfileStage, error) { +func NewDockerfileStageFromBuildkitStage(index int, stage instructions.Stage, expanderFactory *ShlexExpanderFactory, metaArgs, buildArgs map[string]string, dependenciesArgsKeys []string) (*dockerfile.DockerfileStage, error) { var stageInstructions []dockerfile.DockerfileStageInstructionInterface env := map[string]string{} - opts := dockerfile.DockerfileStageInstructionOptions{Expander: shlex} + opts := dockerfile.DockerfileStageInstructionOptions{ExpanderFactory: expanderFactory} for _, cmd := range stage.Commands { var i dockerfile.DockerfileStageInstructionInterface @@ -77,6 +76,8 @@ func NewDockerfileStageFromBuildkitStage(index int, stage instructions.Stage, sh i = instr } case *instructions.ArgCommand: + instrData.Args = removeDependenciesArgs(instrData.Args, dependenciesArgsKeys) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { return nil, err } else { @@ -199,18 +200,40 @@ func NewDockerfileStageFromBuildkitStage(index int, stage instructions.Stage, sh stageInstructions = append(stageInstructions, i) } - return dockerfile.NewDockerfileStage(index, stage.BaseName, stage.Name, stageInstructions, stage.Platform), nil + return dockerfile.NewDockerfileStage(index, stage.BaseName, stage.Name, stageInstructions, stage.Platform, expanderFactory), nil } func createAndExpandInstruction[T dockerfile.InstructionDataInterface](data T, env map[string]string, opts dockerfile.DockerfileStageInstructionOptions) (*dockerfile.DockerfileStageInstruction[T], error) { i := dockerfile.NewDockerfileStageInstruction(data, opts) - if err := i.Expand(env); err != nil { + + // NOTE: skip unset envs during first stage expansion + if err := i.Expand(env, dockerfile.ExpandOptions{SkipUnsetEnv: true}); err != nil { return nil, fmt.Errorf("unable to expand instruction %q: %w", i.GetInstructionData().Name(), err) } + return i, nil } -func processMetaArgs(metaArgs []instructions.ArgCommand, buildArgs map[string]string, shlex *shell.Lex) (map[string]string, error) { +func isDependencyArg(argKey string, dependenciesArgsKeys []string) bool { + for _, dep := range dependenciesArgsKeys { + if dep == argKey { + return true + } + } + return false +} + +func removeDependenciesArgs(args []instructions.KeyValuePairOptional, dependenciesArgsKeys []string) (res []instructions.KeyValuePairOptional) { + // NOTE: dependencies will be expanded on the second stage expansion + for _, arg := range args { + if !isDependencyArg(arg.Key, dependenciesArgsKeys) { + res = append(res, arg) + } + } + return +} + +func resolveMetaArgs(metaArgs []instructions.ArgCommand, buildArgs map[string]string, dependenciesArgsKeys []string, expanderFactory *ShlexExpanderFactory) (map[string]string, error) { var optMetaArgs []instructions.KeyValuePairOptional // TODO(staged-dockerfile): need to support builtin BUILD* and TARGET* args @@ -223,8 +246,12 @@ func processMetaArgs(metaArgs []instructions.ArgCommand, buildArgs map[string]st for _, cmd := range metaArgs { for _, metaArg := range cmd.Args { + if isDependencyArg(metaArg.Key, dependenciesArgsKeys) { + continue + } + if metaArg.Value != nil { - *metaArg.Value, _ = shlex.ProcessWordWithMap(*metaArg.Value, metaArgsToMap(optMetaArgs)) + *metaArg.Value, _ = expanderFactory.GetExpander(dockerfile.ExpandOptions{SkipUnsetEnv: true}).ProcessWordWithMap(*metaArg.Value, metaArgsToMap(optMetaArgs)) } optMetaArgs = append(optMetaArgs, setKVValue(metaArg, buildArgs)) } diff --git a/pkg/dockerfile/frontend/shlex_expander.go b/pkg/dockerfile/frontend/shlex_expander.go new file mode 100644 index 0000000000..0caafa8e62 --- /dev/null +++ b/pkg/dockerfile/frontend/shlex_expander.go @@ -0,0 +1,21 @@ +package frontend + +import ( + "github.com/moby/buildkit/frontend/dockerfile/shell" + + "github.com/werf/werf/pkg/dockerfile" +) + +type ShlexExpanderFactory struct { + EscapeToken rune +} + +func NewShlexExpanderFactory(escapeToken rune) *ShlexExpanderFactory { + return &ShlexExpanderFactory{EscapeToken: escapeToken} +} + +func (factory *ShlexExpanderFactory) GetExpander(opts dockerfile.ExpandOptions) dockerfile.Expander { + shlex := shell.NewLex(factory.EscapeToken) + shlex.SkipUnsetEnv = opts.SkipUnsetEnv + return shlex +} diff --git a/pkg/dockerfile/instruction.go b/pkg/dockerfile/instruction.go index 34e5228656..706815dfff 100644 --- a/pkg/dockerfile/instruction.go +++ b/pkg/dockerfile/instruction.go @@ -10,18 +10,21 @@ type InstructionDataInterface interface { Name() string } +type ExpandOptions struct { + SkipUnsetEnv bool +} + type DockerfileStageInstructionInterface interface { SetDependencyByStageRef(ref string, dep *DockerfileStage) GetDependencyByStageRef(ref string) *DockerfileStage GetDependenciesByStageRef() map[string]*DockerfileStage GetInstructionData() InstructionDataInterface - Expand(env map[string]string) error + Expand(env map[string]string, opts ExpandOptions) error } -type ( - ExpandWordFunc func(word string, env map[string]string) (string, error) - ExpandWordsFunc func(word string, env map[string]string) (string, error) -) +type ExpanderFactory interface { + GetExpander(opts ExpandOptions) Expander +} type Expander interface { ProcessWordWithMap(word string, env map[string]string) (string, error) @@ -29,20 +32,20 @@ type Expander interface { } type DockerfileStageInstructionOptions struct { - Expander Expander + ExpanderFactory ExpanderFactory } type DockerfileStageInstruction[T InstructionDataInterface] struct { Data T DependenciesByStageRef map[string]*DockerfileStage - Expander Expander + ExpanderFactory ExpanderFactory } func NewDockerfileStageInstruction[T InstructionDataInterface](data T, opts DockerfileStageInstructionOptions) *DockerfileStageInstruction[T] { return &DockerfileStageInstruction[T]{ Data: data, DependenciesByStageRef: make(map[string]*DockerfileStage), - Expander: opts.Expander, + ExpanderFactory: opts.ExpanderFactory, } } @@ -68,11 +71,16 @@ func (i *DockerfileStageInstruction[T]) GetInstructionData() InstructionDataInte return i.Data } -func (i *DockerfileStageInstruction[T]) Expand(env map[string]string) error { +func (i *DockerfileStageInstruction[T]) Expand(env map[string]string, opts ExpandOptions) error { + if i.ExpanderFactory == nil { + return nil + } + expander := i.ExpanderFactory.GetExpander(opts) + switch instr := any(i.Data).(type) { case instructions.SupportsSingleWordExpansion: return instr.Expand(func(word string) (string, error) { - return i.Expander.ProcessWordWithMap(word, env) + return expander.ProcessWordWithMap(word, env) }) case *instructions.ExposeCommand: @@ -80,13 +88,27 @@ func (i *DockerfileStageInstruction[T]) Expand(env map[string]string) error { ports := []string{} for _, p := range instr.Ports { - ps, err := i.Expander.ProcessWordsWithMap(p, env) + ps, err := expander.ProcessWordsWithMap(p, env) if err != nil { return fmt.Errorf("unable to expand expose instruction port %q: %w", p, err) } ports = append(ports, ps...) } instr.Ports = ports + + case *instructions.RunCommand: + var newCmdLine []string + for _, line := range instr.CmdLine { + exline, err := expander.ProcessWordWithMap(line, env) + if err != nil { + return fmt.Errorf("unable to expand cmd line %q: %w", line, err) + } + newCmdLine = append(newCmdLine, exline) + } + instr.ShellDependantCmdLine = instructions.ShellDependantCmdLine{ + CmdLine: newCmdLine, + PrependShell: instr.PrependShell, + } } return nil diff --git a/test/e2e/build/_fixtures/complex/state0/Dockerfile b/test/e2e/build/_fixtures/complex/state0/Dockerfile index 280851a425..324e5fce4d 100644 --- a/test/e2e/build/_fixtures/complex/state0/Dockerfile +++ b/test/e2e/build/_fixtures/complex/state0/Dockerfile @@ -1,4 +1,7 @@ -FROM ubuntu:22.04 AS builder +ARG STAPEL_SHELL_IMAGE_NAME="no_such_image" + + +FROM ${STAPEL_SHELL_IMAGE_NAME} AS builder ADD src/file* /app/added/ ADD "https://github.com/octocat/Hello-World/tarball/7fd1a60b01f91b314f59955a4e4d4e80d8edf11d" /helloworld.tgz @@ -8,6 +11,11 @@ COPY src/file* /app/copied/ FROM ubuntu:22.04 AS result ARG CHANGED_ARG="should_be_changed" +ARG BASE_STAPEL_SHELL_IMAGE_NAME="no_such_image" +ARG BASE_STAPEL_SHELL_IMAGE_ID="no_such_image" +ARG BASE_STAPEL_SHELL_IMAGE_REPO="no_such_image" +ARG BASE_STAPEL_SHELL_IMAGE_TAG="no_such_image" + ENV COMPOSED_ENV="env-${CHANGED_ARG}" LABEL COMPOSED_LABEL="label-${CHANGED_ARG}" MAINTAINER "maintainer" @@ -22,6 +30,11 @@ COPY --from=builder /helloworld.tgz /helloworld.tgz RUN touch /created-by-run-state0 RUN mkdir -p /volume10/should-exist-in-volume +RUN echo ${BASE_STAPEL_SHELL_IMAGE_NAME} >> base_image_data.txt +RUN echo ${BASE_STAPEL_SHELL_IMAGE_ID} >> base_image_data.txt +RUN echo ${BASE_STAPEL_SHELL_IMAGE_REPO} >> base_image_data.txt +RUN echo ${BASE_STAPEL_SHELL_IMAGE_TAG} >> base_image_data.txt + ENTRYPOINT ["sh", "-ec"] CMD ["tail -f /dev/null"] VOLUME /volume10 diff --git a/test/e2e/build/_fixtures/complex/state0/werf.yaml b/test/e2e/build/_fixtures/complex/state0/werf.yaml index ba1f3ab9d1..d663364aae 100644 --- a/test/e2e/build/_fixtures/complex/state0/werf.yaml +++ b/test/e2e/build/_fixtures/complex/state0/werf.yaml @@ -9,6 +9,21 @@ target: result network: default args: CHANGED_ARG: "was_changed" +dependencies: +- image: stapel-shell + imports: + - type: ImageName + targetBuildArg: STAPEL_SHELL_IMAGE_NAME +- image: base-stapel-shell + imports: + - type: ImageName + targetBuildArg: BASE_STAPEL_SHELL_IMAGE_NAME + - type: ImageID + targetBuildArg: BASE_STAPEL_SHELL_IMAGE_ID + - type: ImageRepo + targetBuildArg: BASE_STAPEL_SHELL_IMAGE_REPO + - type: ImageTag + targetBuildArg: BASE_STAPEL_SHELL_IMAGE_TAG --- image: base-stapel-shell diff --git a/test/e2e/build/_fixtures/complex/state1/Dockerfile b/test/e2e/build/_fixtures/complex/state1/Dockerfile index 763bd81dd7..78dc7aee45 100644 --- a/test/e2e/build/_fixtures/complex/state1/Dockerfile +++ b/test/e2e/build/_fixtures/complex/state1/Dockerfile @@ -1,4 +1,7 @@ -FROM ubuntu:22.04 AS builder +ARG STAPEL_SHELL_IMAGE_NAME="no_such_image" + + +FROM ${STAPEL_SHELL_IMAGE_NAME} AS builder ADD src/file* /app/added/ COPY src/file* /app/copied/ @@ -7,6 +10,11 @@ COPY src/file* /app/copied/ FROM ubuntu:22.04 AS result ARG CHANGED_ARG="should_be_changed" +ARG BASE_STAPEL_SHELL_IMAGE_NAME="no_such_image" +ARG BASE_STAPEL_SHELL_IMAGE_ID="no_such_image" +ARG BASE_STAPEL_SHELL_IMAGE_REPO="no_such_image" +ARG BASE_STAPEL_SHELL_IMAGE_TAG="no_such_image" + ENV COMPOSED_ENV="env-${CHANGED_ARG}" LABEL COMPOSED_LABEL="label-${CHANGED_ARG}" MAINTAINER "maintainer-${CHANGED_ARG}" @@ -19,6 +27,11 @@ COPY --from=builder /app /app RUN touch /created-by-run-state1 +RUN echo ${BASE_STAPEL_SHELL_IMAGE_NAME} >> base_image_data.txt +RUN echo ${BASE_STAPEL_SHELL_IMAGE_ID} >> base_image_data.txt +RUN echo ${BASE_STAPEL_SHELL_IMAGE_REPO} >> base_image_data.txt +RUN echo ${BASE_STAPEL_SHELL_IMAGE_TAG} >> base_image_data.txt + ENTRYPOINT ["sh", "-ec"] CMD ["tail -f /dev/null"] ONBUILD RUN echo onbuild diff --git a/test/e2e/build/_fixtures/complex/state1/werf.yaml b/test/e2e/build/_fixtures/complex/state1/werf.yaml index 601fd225cd..3ad0397e8c 100644 --- a/test/e2e/build/_fixtures/complex/state1/werf.yaml +++ b/test/e2e/build/_fixtures/complex/state1/werf.yaml @@ -9,6 +9,21 @@ target: result network: default args: CHANGED_ARG: "was_changed-state1" +dependencies: +- image: stapel-shell + imports: + - type: ImageName + targetBuildArg: STAPEL_SHELL_IMAGE_NAME +- image: base-stapel-shell + imports: + - type: ImageName + targetBuildArg: BASE_STAPEL_SHELL_IMAGE_NAME + - type: ImageID + targetBuildArg: BASE_STAPEL_SHELL_IMAGE_ID + - type: ImageRepo + targetBuildArg: BASE_STAPEL_SHELL_IMAGE_REPO + - type: ImageTag + targetBuildArg: BASE_STAPEL_SHELL_IMAGE_TAG --- image: base-stapel-shell diff --git a/test/e2e/build/complex_test.go b/test/e2e/build/complex_test.go index 5b98d359fc..9a21377b90 100644 --- a/test/e2e/build/complex_test.go +++ b/test/e2e/build/complex_test.go @@ -259,19 +259,17 @@ var _ = Describe("Complex build", Label("e2e", "build", "complex"), func() { WithLocalRepo: true, WithStagedDockerfileBuilder: false, }), - // TODO(ilya-lesikov): uncomment after Staged Dockerfile builder finished - // // TODO(1.3): after Full Dockerfile Builder removed and Staged Dockerfile Builder enabled by default this test no longer needed - // Entry("with local repo using Native Buildah and Staged Dockerfile builder with rootless isolation", complexTestOptions{ - // BuildahMode: "native-rootless", - // WithLocalRepo: true, - // WithStagedDockerfileBuilder: true, - // }), - // TODO(ilya-lesikov): uncomment after Staged Dockerfile builder finished - // // TODO(1.3): after Full Dockerfile Builder removed and Staged Dockerfile Builder enabled by default this test no longer needed - // Entry("with local repo using Native Buildah and Staged Dockerfile builder with chroot isolation", complexTestOptions{ - // BuildahMode: "native-chroot", - // WithLocalRepo: true, - // WithStagedDockerfileBuilder: true, - // }), + // TODO(1.3): after Full Dockerfile Builder removed and Staged Dockerfile Builder enabled by default this test no longer needed + Entry("with local repo using Native Buildah and Staged Dockerfile builder with rootless isolation", complexTestOptions{ + BuildahMode: "native-rootless", + WithLocalRepo: true, + WithStagedDockerfileBuilder: true, + }), + // TODO(1.3): after Full Dockerfile Builder removed and Staged Dockerfile Builder enabled by default this test no longer needed + Entry("with local repo using Native Buildah and Staged Dockerfile builder with chroot isolation", complexTestOptions{ + BuildahMode: "native-chroot", + WithLocalRepo: true, + WithStagedDockerfileBuilder: true, + }), ) })