diff --git a/pkg/build/stage/instruction/add_test.go b/pkg/build/stage/instruction/add_test.go index e02656fbf2..2cff5b5138 100644 --- a/pkg/build/stage/instruction/add_test.go +++ b/pkg/build/stage/instruction/add_test.go @@ -28,6 +28,7 @@ var _ = DescribeTable("ADD digest", NewAdd("ADD", dockerfile.NewDockerfileStageInstruction( &instructions.AddCommand{SourcesAndDest: []string{"src", "/app"}, Chown: "1000:1000", Chmod: ""}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ @@ -48,6 +49,7 @@ var _ = DescribeTable("ADD digest", NewAdd("ADD", dockerfile.NewDockerfileStageInstruction( &instructions.AddCommand{SourcesAndDest: []string{"src", "/app"}, Chown: "1000:1001", Chmod: ""}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ @@ -69,6 +71,7 @@ var _ = DescribeTable("ADD digest", NewAdd("ADD", dockerfile.NewDockerfileStageInstruction( &instructions.AddCommand{SourcesAndDest: []string{"src", "/app"}, Chown: "1000:1001", Chmod: "0777"}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ @@ -90,6 +93,7 @@ var _ = DescribeTable("ADD digest", NewAdd("ADD", dockerfile.NewDockerfileStageInstruction( &instructions.AddCommand{SourcesAndDest: []string{"src", "pom.xml", "/app"}, Chown: "1000:1001", Chmod: "0777"}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ @@ -111,6 +115,7 @@ var _ = DescribeTable("ADD digest", NewAdd("ADD", dockerfile.NewDockerfileStageInstruction( &instructions.AddCommand{SourcesAndDest: []string{"src", "pom.xml", "/app"}, Chown: "1000:1001", Chmod: "0777"}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ @@ -132,6 +137,7 @@ var _ = DescribeTable("ADD digest", NewAdd("ADD", dockerfile.NewDockerfileStageInstruction( &instructions.AddCommand{SourcesAndDest: []string{"src", "pom.xml", "/app2"}, Chown: "1000:1001", Chmod: "0777"}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ diff --git a/pkg/build/stage/instruction/cmd_test.go b/pkg/build/stage/instruction/cmd_test.go index 6280ec7bbf..20bcc2f863 100644 --- a/pkg/build/stage/instruction/cmd_test.go +++ b/pkg/build/stage/instruction/cmd_test.go @@ -28,6 +28,7 @@ var _ = DescribeTable("CMD digest", NewCmd("CMD", dockerfile.NewDockerfileStageInstruction( &instructions.CmdCommand{ShellDependantCmdLine: instructions.ShellDependantCmdLine{CmdLine: []string{"/bin/bash", "-lec", "while true ; do date ; sleep 1 ; done"}, PrependShell: false}}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ @@ -48,6 +49,7 @@ var _ = DescribeTable("CMD digest", NewCmd("CMD", dockerfile.NewDockerfileStageInstruction( &instructions.CmdCommand{ShellDependantCmdLine: instructions.ShellDependantCmdLine{CmdLine: []string{"/bin/bash", "-lec", "while true ; do date ; sleep 1 ; done"}, PrependShell: true}}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ @@ -68,6 +70,7 @@ var _ = DescribeTable("CMD digest", NewCmd("CMD", dockerfile.NewDockerfileStageInstruction( &instructions.CmdCommand{ShellDependantCmdLine: instructions.ShellDependantCmdLine{CmdLine: []string{"/bin/bash", "-lec", "while true ; do date ; sleep 1 ; done"}, PrependShell: true}}, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ diff --git a/pkg/build/stage/instruction/copy_test.go b/pkg/build/stage/instruction/copy_test.go index 9d52d42484..86c390bfdf 100644 --- a/pkg/build/stage/instruction/copy_test.go +++ b/pkg/build/stage/instruction/copy_test.go @@ -30,6 +30,7 @@ var _ = DescribeTable("COPY digest", &instructions.CopyCommand{ SourcesAndDest: []string{"src/", "doc/", "/app"}, }, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ @@ -53,6 +54,7 @@ var _ = DescribeTable("COPY digest", &instructions.CopyCommand{ SourcesAndDest: []string{"src/", "doc/", "/app"}, }, + dockerfile.DockerfileStageInstructionOptions{}, ), nil, false, &stage.BaseStageOptions{ diff --git a/pkg/build/stage/instruction/stubs_test.go b/pkg/build/stage/instruction/stubs_test.go index 93a702243d..c4c73776d3 100644 --- a/pkg/build/stage/instruction/stubs_test.go +++ b/pkg/build/stage/instruction/stubs_test.go @@ -12,7 +12,7 @@ import ( ) func NewDockerfileStageInstructionWithDependencyStages[T dockerfile.InstructionDataInterface](data T, dependencyStages []string) *dockerfile.DockerfileStageInstruction[T] { - i := dockerfile.NewDockerfileStageInstruction(data) + i := dockerfile.NewDockerfileStageInstruction(data, dockerfile.DockerfileStageInstructionOptions{}) for _, stageName := range dependencyStages { i.SetDependencyByStageRef(stageName, &dockerfile.DockerfileStage{StageName: stageName}) } diff --git a/pkg/dockerfile/frontend/buildkit_dockerfile.go b/pkg/dockerfile/frontend/buildkit_dockerfile.go index 54ec9ebc81..5021a14703 100644 --- a/pkg/dockerfile/frontend/buildkit_dockerfile.go +++ b/pkg/dockerfile/frontend/buildkit_dockerfile.go @@ -24,26 +24,33 @@ func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.Dockerf return nil, fmt.Errorf("parsing instructions tree: %w", err) } - dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target) + shlex := shell.NewLex(p.EscapeToken) + + metaArgs, err := processMetaArgs(dockerMetaArgs, opts.BuildArgs, shlex) if err != nil { - return nil, fmt.Errorf("determine target stage: %w", err) + return nil, fmt.Errorf("unable to process meta args: %w", err) } - shlex := shell.NewLex(p.EscapeToken) - var stages []*dockerfile.DockerfileStage for i, dockerStage := range dockerStages { - if stage, err := NewDockerfileStageFromBuildkitStage(i, dockerStage, shlex); err != nil { + name, err := shlex.ProcessWordWithMap(dockerStage.BaseName, metaArgs) + if err != nil { + return nil, fmt.Errorf("unable to expand docker stage base image name %q: %w", dockerStage.BaseName, err) + } + if name == "" { + return nil, fmt.Errorf("expanded docker stage base image name %q to empty string: expected image name", dockerStage.BaseName) + } + dockerStage.BaseName = name + + // TODO(staged-dockerfile): support meta-args expansion for dockerStage.Platform + + if stage, err := NewDockerfileStageFromBuildkitStage(i, dockerStage, shlex, metaArgs, opts.BuildArgs); err != nil { return nil, fmt.Errorf("error converting buildkit stage to dockerfile stage: %w", err) } else { stages = append(stages, stage) } } - // TODO(staged-dockerfile): convert meta-args and initialize into Dockerfile obj - _ = dockerMetaArgs - _ = dockerTargetIndex - dockerfile.SetupDockerfileStagesDependencies(stages) d := dockerfile.NewDockerfile(stages, opts) @@ -53,62 +60,217 @@ func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.Dockerf return d, nil } -func NewDockerfileStageFromBuildkitStage(index int, stage instructions.Stage, shlex *shell.Lex) (*dockerfile.DockerfileStage, error) { +func NewDockerfileStageFromBuildkitStage(index int, stage instructions.Stage, shlex *shell.Lex, metaArgs, buildArgs map[string]string) (*dockerfile.DockerfileStage, error) { var stageInstructions []dockerfile.DockerfileStageInstructionInterface - for _, cmd := range stage.Commands { - if expandable, ok := cmd.(instructions.SupportsSingleWordExpansion); ok { - if err := expandable.Expand(func(word string) (string, error) { - // FIXME(ilya-lesikov): add envs/buildargs here - return shlex.ProcessWord(word, []string{}) - }); err != nil { - return nil, fmt.Errorf("error expanding command %q: %w", cmd.Name(), err) - } - } + env := map[string]string{} + opts := dockerfile.DockerfileStageInstructionOptions{Expander: shlex} + for _, cmd := range stage.Commands { var i dockerfile.DockerfileStageInstructionInterface - switch typedCmd := cmd.(type) { + + switch instrData := cmd.(type) { case *instructions.AddCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.ArgCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + + for _, arg := range instr.Data.Args { + if inputValue, hasKey := buildArgs[arg.Key]; hasKey { + arg.Value = new(string) + *arg.Value = inputValue + } + + if arg.Value == nil { + if mvalue, hasKey := metaArgs[arg.Key]; hasKey { + arg.Value = new(string) + *arg.Value = mvalue + } + } + + if arg.Value != nil { + env[arg.Key] = *arg.Value + } + } + } case *instructions.CmdCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.CopyCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.EntrypointCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.EnvCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + + for _, envKV := range instr.Data.Env { + env[envKV.Key] = envKV.Value + } + } case *instructions.ExposeCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.HealthCheckCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.LabelCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.MaintainerCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.OnbuildCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.RunCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.ShellCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.StopSignalCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.UserCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.VolumeCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } case *instructions.WorkdirCommand: - i = dockerfile.NewDockerfileStageInstruction(typedCmd) + if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil { + return nil, err + } else { + i = instr + } } + stageInstructions = append(stageInstructions, i) } return dockerfile.NewDockerfileStage(index, stage.BaseName, stage.Name, stageInstructions, stage.Platform), nil } +func createAndExpandInstruction[T dockerfile.InstructionDataInterface](data T, env map[string]string, opts dockerfile.DockerfileStageInstructionOptions) (*dockerfile.DockerfileStageInstruction[T], error) { + i := dockerfile.NewDockerfileStageInstruction(data, opts) + if err := i.Expand(env); err != nil { + return nil, fmt.Errorf("unable to expand instruction %q: %w", i.GetInstructionData().Name(), err) + } + return i, nil +} + +func processMetaArgs(metaArgs []instructions.ArgCommand, buildArgs map[string]string, shlex *shell.Lex) (map[string]string, error) { + var optMetaArgs []instructions.KeyValuePairOptional + + // TODO(staged-dockerfile): need to support builtin BUILD* and TARGET* args + + // platformOpt := buildPlatformOpt(&opt) + // optMetaArgs := getPlatformArgs(platformOpt) + // for i, arg := range optMetaArgs { + // optMetaArgs[i] = setKVValue(arg, opt.BuildArgs) + // } + + for _, cmd := range metaArgs { + for _, metaArg := range cmd.Args { + if metaArg.Value != nil { + *metaArg.Value, _ = shlex.ProcessWordWithMap(*metaArg.Value, metaArgsToMap(optMetaArgs)) + } + optMetaArgs = append(optMetaArgs, setKVValue(metaArg, buildArgs)) + } + } + + return nil, nil +} + +func metaArgsToMap(metaArgs []instructions.KeyValuePairOptional) map[string]string { + m := map[string]string{} + for _, arg := range metaArgs { + m[arg.Key] = arg.ValueString() + } + return m +} + +func setKVValue(kvpo instructions.KeyValuePairOptional, values map[string]string) instructions.KeyValuePairOptional { + if v, ok := values[kvpo.Key]; ok { + kvpo.Value = &v + } + return kvpo +} + +// TODO(staged-dockerfile) +// +// func getPlatformArgs(po *platformOpt) []instructions.KeyValuePairOptional { +// bp := po.buildPlatforms[0] +// tp := po.targetPlatform +// m := map[string]string{ +// "BUILDPLATFORM": platforms.Format(bp), +// "BUILDOS": bp.OS, +// "BUILDARCH": bp.Architecture, +// "BUILDVARIANT": bp.Variant, +// "TARGETPLATFORM": platforms.Format(tp), +// "TARGETOS": tp.OS, +// "TARGETARCH": tp.Architecture, +// "TARGETVARIANT": tp.Variant, +// } +// opts := make([]instructions.KeyValuePairOptional, 0, len(m)) +// for k, v := range m { +// s := v +// opts = append(opts, instructions.KeyValuePairOptional{Key: k, Value: &s}) +// } +// return opts +// } + func GetDockerStagesNameToIndexMap(stages []instructions.Stage) map[string]int { nameToIndex := make(map[string]int) for i, s := range stages { diff --git a/pkg/dockerfile/instruction.go b/pkg/dockerfile/instruction.go index b6221d3803..34e5228656 100644 --- a/pkg/dockerfile/instruction.go +++ b/pkg/dockerfile/instruction.go @@ -2,6 +2,8 @@ package dockerfile import ( "fmt" + + "github.com/moby/buildkit/frontend/dockerfile/instructions" ) type InstructionDataInterface interface { @@ -13,18 +15,34 @@ type DockerfileStageInstructionInterface interface { GetDependencyByStageRef(ref string) *DockerfileStage GetDependenciesByStageRef() map[string]*DockerfileStage GetInstructionData() InstructionDataInterface - // TODO(staged-dockerfile): something like Expand(args, envs map[string]string) + Expand(env map[string]string) error +} + +type ( + ExpandWordFunc func(word string, env map[string]string) (string, error) + ExpandWordsFunc func(word string, env map[string]string) (string, error) +) + +type Expander interface { + ProcessWordWithMap(word string, env map[string]string) (string, error) + ProcessWordsWithMap(word string, env map[string]string) ([]string, error) +} + +type DockerfileStageInstructionOptions struct { + Expander Expander } type DockerfileStageInstruction[T InstructionDataInterface] struct { Data T DependenciesByStageRef map[string]*DockerfileStage + Expander Expander } -func NewDockerfileStageInstruction[T InstructionDataInterface](data T) *DockerfileStageInstruction[T] { +func NewDockerfileStageInstruction[T InstructionDataInterface](data T, opts DockerfileStageInstructionOptions) *DockerfileStageInstruction[T] { return &DockerfileStageInstruction[T]{ Data: data, DependenciesByStageRef: make(map[string]*DockerfileStage), + Expander: opts.Expander, } } @@ -49,3 +67,27 @@ func (i *DockerfileStageInstruction[T]) GetDependenciesByStageRef() map[string]* func (i *DockerfileStageInstruction[T]) GetInstructionData() InstructionDataInterface { return i.Data } + +func (i *DockerfileStageInstruction[T]) Expand(env map[string]string) error { + switch instr := any(i.Data).(type) { + case instructions.SupportsSingleWordExpansion: + return instr.Expand(func(word string) (string, error) { + return i.Expander.ProcessWordWithMap(word, env) + }) + + case *instructions.ExposeCommand: + // NOTE: ExposeCommand does not implement Expander interface, but actually needs expansion + + ports := []string{} + for _, p := range instr.Ports { + ps, err := i.Expander.ProcessWordsWithMap(p, env) + if err != nil { + return fmt.Errorf("unable to expand expose instruction port %q: %w", p, err) + } + ports = append(ports, ps...) + } + instr.Ports = ports + } + + return nil +}