Skip to content

Commit

Permalink
feat(staged-dockerfile): implement first stage of build-args expansion
Browse files Browse the repository at this point in the history 
First stage: standard expansion on dockerfile parsing stage.
Second stage: expand dependencies build-args.

* Added universal method for expansion any instruction, which would be needed for second-stage inspection, which mutates instruction data in place.
* Implemented meta-args expansion and per-stage args expansion.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Nov 1, 2022
1 parent e7d93b2 commit c0de754
Show file tree
Hide file tree
Showing 6 changed files with 255 additions and 40 deletions.
6 changes: 6 additions & 0 deletions pkg/build/stage/instruction/add_test.go
View file
Expand Up @@ -28,6 +28,7 @@ var _ = DescribeTable("ADD digest",
NewAdd("ADD",
dockerfile.NewDockerfileStageInstruction(
&instructions.AddCommand{SourcesAndDest: []string{"src", "/app"}, Chown: "1000:1000", Chmod: ""},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand All @@ -48,6 +49,7 @@ var _ = DescribeTable("ADD digest",
NewAdd("ADD",
dockerfile.NewDockerfileStageInstruction(
&instructions.AddCommand{SourcesAndDest: []string{"src", "/app"}, Chown: "1000:1001", Chmod: ""},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand All @@ -69,6 +71,7 @@ var _ = DescribeTable("ADD digest",
NewAdd("ADD",
dockerfile.NewDockerfileStageInstruction(
&instructions.AddCommand{SourcesAndDest: []string{"src", "/app"}, Chown: "1000:1001", Chmod: "0777"},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand All @@ -90,6 +93,7 @@ var _ = DescribeTable("ADD digest",
NewAdd("ADD",
dockerfile.NewDockerfileStageInstruction(
&instructions.AddCommand{SourcesAndDest: []string{"src", "pom.xml", "/app"}, Chown: "1000:1001", Chmod: "0777"},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand All @@ -111,6 +115,7 @@ var _ = DescribeTable("ADD digest",
NewAdd("ADD",
dockerfile.NewDockerfileStageInstruction(
&instructions.AddCommand{SourcesAndDest: []string{"src", "pom.xml", "/app"}, Chown: "1000:1001", Chmod: "0777"},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand All @@ -132,6 +137,7 @@ var _ = DescribeTable("ADD digest",
NewAdd("ADD",
dockerfile.NewDockerfileStageInstruction(
&instructions.AddCommand{SourcesAndDest: []string{"src", "pom.xml", "/app2"}, Chown: "1000:1001", Chmod: "0777"},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand Down
3 changes: 3 additions & 0 deletions pkg/build/stage/instruction/cmd_test.go
View file
Expand Up @@ -28,6 +28,7 @@ var _ = DescribeTable("CMD digest",
NewCmd("CMD",
dockerfile.NewDockerfileStageInstruction(
&instructions.CmdCommand{ShellDependantCmdLine: instructions.ShellDependantCmdLine{CmdLine: []string{"/bin/bash", "-lec", "while true ; do date ; sleep 1 ; done"}, PrependShell: false}},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand All @@ -48,6 +49,7 @@ var _ = DescribeTable("CMD digest",
NewCmd("CMD",
dockerfile.NewDockerfileStageInstruction(
&instructions.CmdCommand{ShellDependantCmdLine: instructions.ShellDependantCmdLine{CmdLine: []string{"/bin/bash", "-lec", "while true ; do date ; sleep 1 ; done"}, PrependShell: true}},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand All @@ -68,6 +70,7 @@ var _ = DescribeTable("CMD digest",
NewCmd("CMD",
dockerfile.NewDockerfileStageInstruction(
&instructions.CmdCommand{ShellDependantCmdLine: instructions.ShellDependantCmdLine{CmdLine: []string{"/bin/bash", "-lec", "while true ; do date ; sleep 1 ; done"}, PrependShell: true}},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand Down
2 changes: 2 additions & 0 deletions pkg/build/stage/instruction/copy_test.go
View file
Expand Up @@ -30,6 +30,7 @@ var _ = DescribeTable("COPY digest",
&instructions.CopyCommand{
SourcesAndDest: []string{"src/", "doc/", "/app"},
},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand All @@ -53,6 +54,7 @@ var _ = DescribeTable("COPY digest",
&instructions.CopyCommand{
SourcesAndDest: []string{"src/", "doc/", "/app"},
},
dockerfile.DockerfileStageInstructionOptions{},
),
nil, false,
&stage.BaseStageOptions{
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/stage/instruction/stubs_test.go
View file
Expand Up @@ -12,7 +12,7 @@ import (
)

func NewDockerfileStageInstructionWithDependencyStages[T dockerfile.InstructionDataInterface](data T, dependencyStages []string) *dockerfile.DockerfileStageInstruction[T] {
i := dockerfile.NewDockerfileStageInstruction(data)
i := dockerfile.NewDockerfileStageInstruction(data, dockerfile.DockerfileStageInstructionOptions{})
for _, stageName := range dependencyStages {
i.SetDependencyByStageRef(stageName, &dockerfile.DockerfileStage{StageName: stageName})
}
Expand Down
236 changes: 199 additions & 37 deletions pkg/dockerfile/frontend/buildkit_dockerfile.go
View file
Expand Up @@ -24,26 +24,33 @@ func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.Dockerf
return nil, fmt.Errorf("parsing instructions tree: %w", err)
}

dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target)
shlex := shell.NewLex(p.EscapeToken)

metaArgs, err := processMetaArgs(dockerMetaArgs, opts.BuildArgs, shlex)
if err != nil {
return nil, fmt.Errorf("determine target stage: %w", err)
return nil, fmt.Errorf("unable to process meta args: %w", err)
}

shlex := shell.NewLex(p.EscapeToken)

var stages []*dockerfile.DockerfileStage
for i, dockerStage := range dockerStages {
if stage, err := NewDockerfileStageFromBuildkitStage(i, dockerStage, shlex); err != nil {
name, err := shlex.ProcessWordWithMap(dockerStage.BaseName, metaArgs)
if err != nil {
return nil, fmt.Errorf("unable to expand docker stage base image name %q: %w", dockerStage.BaseName, err)
}
if name == "" {
return nil, fmt.Errorf("expanded docker stage base image name %q to empty string: expected image name", dockerStage.BaseName)
}
dockerStage.BaseName = name

// TODO(staged-dockerfile): support meta-args expansion for dockerStage.Platform

if stage, err := NewDockerfileStageFromBuildkitStage(i, dockerStage, shlex, metaArgs, opts.BuildArgs); err != nil {
return nil, fmt.Errorf("error converting buildkit stage to dockerfile stage: %w", err)
} else {
stages = append(stages, stage)
}
}

// TODO(staged-dockerfile): convert meta-args and initialize into Dockerfile obj
_ = dockerMetaArgs
_ = dockerTargetIndex

dockerfile.SetupDockerfileStagesDependencies(stages)

d := dockerfile.NewDockerfile(stages, opts)
Expand All @@ -53,62 +60,217 @@ func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.Dockerf
return d, nil
}

func NewDockerfileStageFromBuildkitStage(index int, stage instructions.Stage, shlex *shell.Lex) (*dockerfile.DockerfileStage, error) {
func NewDockerfileStageFromBuildkitStage(index int, stage instructions.Stage, shlex *shell.Lex, metaArgs, buildArgs map[string]string) (*dockerfile.DockerfileStage, error) {
var stageInstructions []dockerfile.DockerfileStageInstructionInterface

for _, cmd := range stage.Commands {
if expandable, ok := cmd.(instructions.SupportsSingleWordExpansion); ok {
if err := expandable.Expand(func(word string) (string, error) {
// FIXME(ilya-lesikov): add envs/buildargs here
return shlex.ProcessWord(word, []string{})
}); err != nil {
return nil, fmt.Errorf("error expanding command %q: %w", cmd.Name(), err)
}
}
env := map[string]string{}
opts := dockerfile.DockerfileStageInstructionOptions{Expander: shlex}

for _, cmd := range stage.Commands {
var i dockerfile.DockerfileStageInstructionInterface
switch typedCmd := cmd.(type) {

switch instrData := cmd.(type) {
case *instructions.AddCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.ArgCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr

for _, arg := range instr.Data.Args {
if inputValue, hasKey := buildArgs[arg.Key]; hasKey {
arg.Value = new(string)
*arg.Value = inputValue
}

if arg.Value == nil {
if mvalue, hasKey := metaArgs[arg.Key]; hasKey {
arg.Value = new(string)
*arg.Value = mvalue
}
}

if arg.Value != nil {
env[arg.Key] = *arg.Value
}
}
}
case *instructions.CmdCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.CopyCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.EntrypointCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.EnvCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr

for _, envKV := range instr.Data.Env {
env[envKV.Key] = envKV.Value
}
}
case *instructions.ExposeCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.HealthCheckCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.LabelCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.MaintainerCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.OnbuildCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.RunCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.ShellCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.StopSignalCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.UserCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.VolumeCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
case *instructions.WorkdirCommand:
i = dockerfile.NewDockerfileStageInstruction(typedCmd)
if instr, err := createAndExpandInstruction(instrData, env, opts); err != nil {
return nil, err
} else {
i = instr
}
}

stageInstructions = append(stageInstructions, i)
}

return dockerfile.NewDockerfileStage(index, stage.BaseName, stage.Name, stageInstructions, stage.Platform), nil
}

func createAndExpandInstruction[T dockerfile.InstructionDataInterface](data T, env map[string]string, opts dockerfile.DockerfileStageInstructionOptions) (*dockerfile.DockerfileStageInstruction[T], error) {
i := dockerfile.NewDockerfileStageInstruction(data, opts)
if err := i.Expand(env); err != nil {
return nil, fmt.Errorf("unable to expand instruction %q: %w", i.GetInstructionData().Name(), err)
}
return i, nil
}

func processMetaArgs(metaArgs []instructions.ArgCommand, buildArgs map[string]string, shlex *shell.Lex) (map[string]string, error) {
var optMetaArgs []instructions.KeyValuePairOptional

// TODO(staged-dockerfile): need to support builtin BUILD* and TARGET* args

// platformOpt := buildPlatformOpt(&opt)
// optMetaArgs := getPlatformArgs(platformOpt)
// for i, arg := range optMetaArgs {
// optMetaArgs[i] = setKVValue(arg, opt.BuildArgs)
// }

for _, cmd := range metaArgs {
for _, metaArg := range cmd.Args {
if metaArg.Value != nil {
*metaArg.Value, _ = shlex.ProcessWordWithMap(*metaArg.Value, metaArgsToMap(optMetaArgs))
}
optMetaArgs = append(optMetaArgs, setKVValue(metaArg, buildArgs))
}
}

return nil, nil
}

func metaArgsToMap(metaArgs []instructions.KeyValuePairOptional) map[string]string {
m := map[string]string{}
for _, arg := range metaArgs {
m[arg.Key] = arg.ValueString()
}
return m
}

func setKVValue(kvpo instructions.KeyValuePairOptional, values map[string]string) instructions.KeyValuePairOptional {
if v, ok := values[kvpo.Key]; ok {
kvpo.Value = &v
}
return kvpo
}

// TODO(staged-dockerfile)
//
// func getPlatformArgs(po *platformOpt) []instructions.KeyValuePairOptional {
// bp := po.buildPlatforms[0]
// tp := po.targetPlatform
// m := map[string]string{
// "BUILDPLATFORM": platforms.Format(bp),
// "BUILDOS": bp.OS,
// "BUILDARCH": bp.Architecture,
// "BUILDVARIANT": bp.Variant,
// "TARGETPLATFORM": platforms.Format(tp),
// "TARGETOS": tp.OS,
// "TARGETARCH": tp.Architecture,
// "TARGETVARIANT": tp.Variant,
// }
// opts := make([]instructions.KeyValuePairOptional, 0, len(m))
// for k, v := range m {
// s := v
// opts = append(opts, instructions.KeyValuePairOptional{Key: k, Value: &s})
// }
// return opts
// }

func GetDockerStagesNameToIndexMap(stages []instructions.Stage) map[string]int {
nameToIndex := make(map[string]int)
for i, s := range stages {
Expand Down

0 comments on commit c0de754

Please sign in to comment.