diff --git a/pkg/build/image/dockerfile.go b/pkg/build/image/dockerfile.go index 82e1f37a8f..b465188344 100644 --- a/pkg/build/image/dockerfile.go +++ b/pkg/build/image/dockerfile.go @@ -16,6 +16,7 @@ import ( "github.com/werf/werf/pkg/config" backend_instruction "github.com/werf/werf/pkg/container_backend/instruction" "github.com/werf/werf/pkg/dockerfile" + "github.com/werf/werf/pkg/dockerfile/frontend" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" "github.com/werf/werf/pkg/path_matcher" "github.com/werf/werf/pkg/util" @@ -29,7 +30,7 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig return nil, fmt.Errorf("unable to read dockerfile %s: %w", relDockerfilePath, err) } - d, err := dockerfile.ParseDockerfile(dockerfileData, dockerfile.DockerfileOptions{ + d, err := frontend.ParseDockerfileWithBuildkit(dockerfileData, dockerfile.DockerfileOptions{ Target: dockerfileImageConfig.Target, BuildArgs: util.MapStringInterfaceToMapStringString(dockerfileImageConfig.Args), AddHost: dockerfileImageConfig.AddHost, @@ -178,9 +179,9 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf return nil, err } - dockerfile.ResolveDockerStagesFromValue(dockerStages) + frontend.ResolveDockerStagesFromValue(dockerStages) - dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target) + dockerTargetIndex, err := frontend.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target) if err != nil { return nil, err } diff --git a/pkg/build/stage/full_dockerfile_test.go b/pkg/build/stage/full_dockerfile_test.go index 6b62e0ba13..3f134edc1b 100644 --- a/pkg/build/stage/full_dockerfile_test.go +++ b/pkg/build/stage/full_dockerfile_test.go @@ -10,7 +10,7 @@ import ( . "github.com/onsi/gomega" "github.com/werf/werf/pkg/container_backend/stage_builder" - "github.com/werf/werf/pkg/dockerfile" + "github.com/werf/werf/pkg/dockerfile/frontend" "github.com/werf/werf/pkg/util" ) @@ -21,13 +21,13 @@ func testDockerfileToDockerStages(dockerfileData []byte) ([]instructions.Stage, dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST) Expect(err).To(Succeed()) - dockerfile.ResolveDockerStagesFromValue(dockerStages) + frontend.ResolveDockerStagesFromValue(dockerStages) return dockerStages, dockerMetaArgs } func newTestFullDockerfileStage(dockerfileData []byte, target string, buildArgs map[string]interface{}, dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dependencies []*TestDependency) *FullDockerfileStage { - dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, target) + dockerTargetIndex, err := frontend.GetDockerTargetStageIndex(dockerStages, target) Expect(err).To(Succeed()) ds := NewDockerStages( diff --git a/pkg/dockerfile/dockerfile.go b/pkg/dockerfile/dockerfile.go index 9e7f228b74..3f59175625 100644 --- a/pkg/dockerfile/dockerfile.go +++ b/pkg/dockerfile/dockerfile.go @@ -1,36 +1,9 @@ package dockerfile import ( - "bytes" "context" - "fmt" - - "github.com/moby/buildkit/frontend/dockerfile/instructions" - "github.com/moby/buildkit/frontend/dockerfile/parser" ) -func ParseDockerfile(dockerfile []byte, opts DockerfileOptions) (*Dockerfile, error) { - p, err := parser.Parse(bytes.NewReader(dockerfile)) - if err != nil { - return nil, fmt.Errorf("parsing dockerfile data: %w", err) - } - - dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST) - if err != nil { - return nil, fmt.Errorf("parsing instructions tree: %w", err) - } - - // FIXME(staged-dockerfile): is this needed? - ResolveDockerStagesFromValue(dockerStages) - - dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target) - if err != nil { - return nil, fmt.Errorf("determine target stage: %w", err) - } - - return newDockerfile(dockerStages, dockerMetaArgs, dockerTargetIndex, opts), nil -} - type DockerfileOptions struct { Target string BuildArgs map[string]string @@ -39,26 +12,37 @@ type DockerfileOptions struct { SSH string } -func newDockerfile(dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dockerTargetStageIndex int, opts DockerfileOptions) *Dockerfile { +func NewDockerfile(stages []*DockerfileStage, opts DockerfileOptions) *Dockerfile { return &Dockerfile{ DockerfileOptions: opts, - - dockerStages: dockerStages, - dockerMetaArgs: dockerMetaArgs, - dockerTargetStageIndex: dockerTargetStageIndex, - nameToIndex: GetDockerStagesNameToIndexMap(dockerStages), + Stages: stages, } } type Dockerfile struct { DockerfileOptions - dockerStages []instructions.Stage - dockerMetaArgs []instructions.ArgCommand - dockerTargetStageIndex int - nameToIndex map[string]string + Stages []*DockerfileStage } -func (dockerfile *Dockerfile) GroupStagesByIndependentSets(ctx context.Context) ([][]*DockerfileStage, error) { +func (df *Dockerfile) GroupStagesByIndependentSets(ctx context.Context) ([][]*DockerfileStage, error) { + // FIXME(staged-dockerfile): build real dependencies tree + + // var res [][]*DockerfileStage + // var curLevel []*DockerfileStage + + // stagesQueue + + // res = append(res, curLevel) + + // for _, stg := range df.Stages { + // stg.Dependencies + // } + + // var res [][]*DockerfileStage + // for _, stg := range df.Stages { + // res = append(res, []*DockerfileStage{stg}) + // } + // return res, nil return nil, nil } diff --git a/pkg/dockerfile/dockerfile_stage.go b/pkg/dockerfile/dockerfile_stage.go index c3f315e67a..2166df89ad 100644 --- a/pkg/dockerfile/dockerfile_stage.go +++ b/pkg/dockerfile/dockerfile_stage.go @@ -1,10 +1,85 @@ package dockerfile -func NewDockerfileStage(dockerfile *Dockerfile, instructions []InstructionInterface) *DockerfileStage { - return &DockerfileStage{Dockerfile: dockerfile, Instructions: instructions} +import ( + "fmt" + "strconv" + "strings" + + dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" +) + +func NewDockerfileStage(index int, baseName, stageName string, instructions []InstructionInterface, platform string) *DockerfileStage { + return &DockerfileStage{BaseName: baseName, StageName: stageName, Instructions: instructions, Platform: platform} } type DockerfileStage struct { Dockerfile *Dockerfile + Dependencies []*DockerfileStage + + BaseName string + Index int + StageName string + Platform string Instructions []InstructionInterface } + +func (stage DockerfileStage) LogName() string { + if stage.HasStageName() { + return stage.StageName + } else { + return fmt.Sprintf("<%d>", stage.Index) + } +} + +func (stage DockerfileStage) HasStageName() bool { + return stage.StageName != "" +} + +func SetupDockerfileStagesDependencies(stages []*DockerfileStage) error { + stageByName := make(map[string]*DockerfileStage) + for _, stage := range stages { + if stage.HasStageName() { + stageByName[strings.ToLower(stage.StageName)] = stage + } + } + + for _, stage := range stages { + // Base image dependency + if dependency, hasKey := stageByName[strings.ToLower(stage.BaseName)]; hasKey { + stage.Dependencies = append(stage.Dependencies, dependency) + } + + for _, instr := range stage.Instructions { + switch typedInstr := instr.(type) { + case *dockerfile_instruction.Copy: + if dep := findStageByNameOrIndex(typedInstr.From, stages, stageByName); dep != nil { + stage.Dependencies = append(stage.Dependencies, dep) + } else { + return fmt.Errorf("unable to resolve stage %q instruction %s --from=%q: no such stage", stage.LogName(), instr.Name(), typedInstr.From) + } + + case *dockerfile_instruction.Run: + for _, mount := range typedInstr.Mounts { + if mount.From != "" { + if dep := findStageByNameOrIndex(mount.From, stages, stageByName); dep != nil { + stage.Dependencies = append(stage.Dependencies, dep) + } else { + return fmt.Errorf("unable to resolve stage %q instruction %s --mount=from=%s: no such stage", stage.LogName(), instr.Name(), mount.From) + } + } + } + } + } + } + + return nil +} + +func findStageByNameOrIndex(ref string, stages []*DockerfileStage, stageByName map[string]*DockerfileStage) *DockerfileStage { + if stg, found := stageByName[strings.ToLower(ref)]; found { + return stg + } else if ind, err := strconv.Atoi(ref); err == nil && ind >= 0 && ind < len(stages) { + return stages[ind] + } + return nil +} diff --git a/pkg/dockerfile/frontend/buildkit.go b/pkg/dockerfile/frontend/buildkit.go index c517cca1d2..c431a533a1 100644 --- a/pkg/dockerfile/frontend/buildkit.go +++ b/pkg/dockerfile/frontend/buildkit.go @@ -1,15 +1,53 @@ package frontend import ( + "bytes" "fmt" + "strconv" + "strings" "github.com/moby/buildkit/frontend/dockerfile/instructions" + "github.com/moby/buildkit/frontend/dockerfile/parser" "github.com/werf/werf/pkg/dockerfile" dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction" ) -func DockerfileStageFromBuildkitStage(d *dockerfile.Dockerfile, stage instructions.Stage) (*dockerfile.DockerfileStage, error) { +func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.DockerfileOptions) (*dockerfile.Dockerfile, error) { + p, err := parser.Parse(bytes.NewReader(dockerfileBytes)) + if err != nil { + return nil, fmt.Errorf("parsing dockerfile data: %w", err) + } + + dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST) + if err != nil { + return nil, fmt.Errorf("parsing instructions tree: %w", err) + } + + dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target) + if err != nil { + return nil, fmt.Errorf("determine target stage: %w", err) + } + + var stages []*dockerfile.DockerfileStage + for i, dockerStage := range dockerStages { + stages = append(stages, DockerfileStageFromBuildkitStage(i, dockerStage)) + } + + // TODO(staged-dockerfile): convert meta-args and initialize into Dockerfile obj + _ = dockerMetaArgs + _ = dockerTargetIndex + + dockerfile.SetupDockerfileStagesDependencies(stages) + + d := dockerfile.NewDockerfile(stages, opts) + for _, stage := range d.Stages { + stage.Dockerfile = d + } + return d, nil +} + +func DockerfileStageFromBuildkitStage(index int, stage instructions.Stage) *dockerfile.DockerfileStage { var i []dockerfile.InstructionInterface for _, cmd := range stage.Commands { @@ -56,7 +94,7 @@ func DockerfileStageFromBuildkitStage(d *dockerfile.Dockerfile, stage instructio } } - return dockerfile.NewDockerfileStage(d, i), nil + return dockerfile.NewDockerfileStage(index, stage.BaseName, stage.Name, i, stage.Platform) } func extractSrcAndDst(sourcesAndDest instructions.SourcesAndDest) ([]string, string) { @@ -75,3 +113,56 @@ func extractKeyValuePairsAsMap(pairs instructions.KeyValuePairs) (res map[string } return } + +func GetDockerStagesNameToIndexMap(stages []instructions.Stage) map[string]int { + nameToIndex := make(map[string]int) + for i, s := range stages { + name := strings.ToLower(s.Name) + if name != strconv.Itoa(i) { + nameToIndex[name] = i + } + } + return nameToIndex +} + +func ResolveDockerStagesFromValue(stages []instructions.Stage) { + nameToIndex := GetDockerStagesNameToIndexMap(stages) + + for _, s := range stages { + for _, cmd := range s.Commands { + switch typedCmd := cmd.(type) { + case *instructions.CopyCommand: + if typedCmd.From != "" { + from := strings.ToLower(typedCmd.From) + if val, ok := nameToIndex[from]; ok { + typedCmd.From = strconv.Itoa(val) + } + } + + case *instructions.RunCommand: + for _, mount := range instructions.GetMounts(typedCmd) { + if mount.From != "" { + from := strings.ToLower(mount.From) + if val, ok := nameToIndex[from]; ok { + mount.From = strconv.Itoa(val) + } + } + } + } + } + } +} + +func GetDockerTargetStageIndex(dockerStages []instructions.Stage, dockerTargetStage string) (int, error) { + if dockerTargetStage == "" { + return len(dockerStages) - 1, nil + } + + for i, s := range dockerStages { + if s.Name == dockerTargetStage { + return i, nil + } + } + + return -1, fmt.Errorf("%s is not a valid target build stage", dockerTargetStage) +} diff --git a/pkg/dockerfile/helpers.go b/pkg/dockerfile/helpers.go deleted file mode 100644 index ad1b38ae5b..0000000000 --- a/pkg/dockerfile/helpers.go +++ /dev/null @@ -1,63 +0,0 @@ -package dockerfile - -import ( - "fmt" - "strconv" - "strings" - - "github.com/moby/buildkit/frontend/dockerfile/instructions" -) - -func GetDockerStagesNameToIndexMap(stages []instructions.Stage) map[string]string { - nameToIndex := make(map[string]string) - for i, s := range stages { - name := strings.ToLower(s.Name) - index := strconv.Itoa(i) - if name != index { - nameToIndex[name] = index - } - } - return nameToIndex -} - -func ResolveDockerStagesFromValue(stages []instructions.Stage) { - nameToIndex := GetDockerStagesNameToIndexMap(stages) - - for _, s := range stages { - for _, cmd := range s.Commands { - switch typedCmd := cmd.(type) { - case *instructions.CopyCommand: - if typedCmd.From != "" { - from := strings.ToLower(typedCmd.From) - if val, ok := nameToIndex[from]; ok { - typedCmd.From = val - } - } - - case *instructions.RunCommand: - for _, mount := range instructions.GetMounts(typedCmd) { - if mount.From != "" { - from := strings.ToLower(mount.From) - if val, ok := nameToIndex[from]; ok { - mount.From = val - } - } - } - } - } - } -} - -func GetDockerTargetStageIndex(dockerStages []instructions.Stage, dockerTargetStage string) (int, error) { - if dockerTargetStage == "" { - return len(dockerStages) - 1, nil - } - - for i, s := range dockerStages { - if s.Name == dockerTargetStage { - return i, nil - } - } - - return -1, fmt.Errorf("%s is not a valid target build stage", dockerTargetStage) -}