feat(staged-dockerfile): Dockerfile and DockerfileStage primitives re…

…worked

* Initialize DockerfileStage with instructions.
* Build dependencies tree between different DockerfileStage-s based on FROM, COPY and RUN instructions usage.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>

pkg/build/image/dockerfile.go

@@ -16,6 +16,7 @@ import (
 	"github.com/werf/werf/pkg/config"
 	backend_instruction "github.com/werf/werf/pkg/container_backend/instruction"
 	"github.com/werf/werf/pkg/dockerfile"
+	"github.com/werf/werf/pkg/dockerfile/frontend"
 	dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
 	"github.com/werf/werf/pkg/path_matcher"
 	"github.com/werf/werf/pkg/util"
@@ -29,7 +30,7 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig
 			return nil, fmt.Errorf("unable to read dockerfile %s: %w", relDockerfilePath, err)
 		}
 
-		d, err := dockerfile.ParseDockerfile(dockerfileData, dockerfile.DockerfileOptions{
+		d, err := frontend.ParseDockerfileWithBuildkit(dockerfileData, dockerfile.DockerfileOptions{
 			Target:    dockerfileImageConfig.Target,
 			BuildArgs: util.MapStringInterfaceToMapStringString(dockerfileImageConfig.Args),
 			AddHost:   dockerfileImageConfig.AddHost,
@@ -178,9 +179,9 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf
 		return nil, err
 	}
 
-	dockerfile.ResolveDockerStagesFromValue(dockerStages)
+	frontend.ResolveDockerStagesFromValue(dockerStages)
 
-	dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target)
+	dockerTargetIndex, err := frontend.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target)
 	if err != nil {
 		return nil, err
 	}

pkg/build/stage/full_dockerfile_test.go

@@ -10,7 +10,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/werf/werf/pkg/container_backend/stage_builder"
-	"github.com/werf/werf/pkg/dockerfile"
+	"github.com/werf/werf/pkg/dockerfile/frontend"
 	"github.com/werf/werf/pkg/util"
 )
 
@@ -21,13 +21,13 @@ func testDockerfileToDockerStages(dockerfileData []byte) ([]instructions.Stage,
 	dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST)
 	Expect(err).To(Succeed())
 
-	dockerfile.ResolveDockerStagesFromValue(dockerStages)
+	frontend.ResolveDockerStagesFromValue(dockerStages)
 
 	return dockerStages, dockerMetaArgs
 }
 
 func newTestFullDockerfileStage(dockerfileData []byte, target string, buildArgs map[string]interface{}, dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dependencies []*TestDependency) *FullDockerfileStage {
-	dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, target)
+	dockerTargetIndex, err := frontend.GetDockerTargetStageIndex(dockerStages, target)
 	Expect(err).To(Succeed())
 
 	ds := NewDockerStages(

pkg/dockerfile/dockerfile.go

@@ -1,36 +1,9 @@
 package dockerfile
 
 import (
-	"bytes"
 	"context"
-	"fmt"
-
-	"github.com/moby/buildkit/frontend/dockerfile/instructions"
-	"github.com/moby/buildkit/frontend/dockerfile/parser"
 )
 
-func ParseDockerfile(dockerfile []byte, opts DockerfileOptions) (*Dockerfile, error) {
-	p, err := parser.Parse(bytes.NewReader(dockerfile))
-	if err != nil {
-		return nil, fmt.Errorf("parsing dockerfile data: %w", err)
-	}
-
-	dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST)
-	if err != nil {
-		return nil, fmt.Errorf("parsing instructions tree: %w", err)
-	}
-
-	// FIXME(staged-dockerfile): is this needed?
-	ResolveDockerStagesFromValue(dockerStages)
-
-	dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target)
-	if err != nil {
-		return nil, fmt.Errorf("determine target stage: %w", err)
-	}
-
-	return newDockerfile(dockerStages, dockerMetaArgs, dockerTargetIndex, opts), nil
-}
-
 type DockerfileOptions struct {
 	Target    string
 	BuildArgs map[string]string
@@ -39,26 +12,37 @@ type DockerfileOptions struct {
 	SSH       string
 }
 
-func newDockerfile(dockerStages []instructions.Stage, dockerMetaArgs []instructions.ArgCommand, dockerTargetStageIndex int, opts DockerfileOptions) *Dockerfile {
+func NewDockerfile(stages []*DockerfileStage, opts DockerfileOptions) *Dockerfile {
 	return &Dockerfile{
 		DockerfileOptions: opts,
-
-		dockerStages:           dockerStages,
-		dockerMetaArgs:         dockerMetaArgs,
-		dockerTargetStageIndex: dockerTargetStageIndex,
-		nameToIndex:            GetDockerStagesNameToIndexMap(dockerStages),
+		Stages:            stages,
 	}
 }
 
 type Dockerfile struct {
 	DockerfileOptions
 
-	dockerStages           []instructions.Stage
-	dockerMetaArgs         []instructions.ArgCommand
-	dockerTargetStageIndex int
-	nameToIndex            map[string]string
+	Stages []*DockerfileStage
 }
 
-func (dockerfile *Dockerfile) GroupStagesByIndependentSets(ctx context.Context) ([][]*DockerfileStage, error) {
+func (df *Dockerfile) GroupStagesByIndependentSets(ctx context.Context) ([][]*DockerfileStage, error) {
+	// FIXME(staged-dockerfile): build real dependencies tree
+
+	// var res [][]*DockerfileStage
+	// var curLevel []*DockerfileStage
+
+	// stagesQueue
+
+	// res = append(res, curLevel)
+
+	// for _, stg := range df.Stages {
+	// 	stg.Dependencies
+	// }
+
+	// var res [][]*DockerfileStage
+	// for _, stg := range df.Stages {
+	// 	res = append(res, []*DockerfileStage{stg})
+	// }
+	// return res, nil
 	return nil, nil
 }

pkg/dockerfile/dockerfile_stage.go

@@ -1,10 +1,85 @@
 package dockerfile
 
-func NewDockerfileStage(dockerfile *Dockerfile, instructions []InstructionInterface) *DockerfileStage {
-	return &DockerfileStage{Dockerfile: dockerfile, Instructions: instructions}
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
+)
+
+func NewDockerfileStage(index int, baseName, stageName string, instructions []InstructionInterface, platform string) *DockerfileStage {
+	return &DockerfileStage{BaseName: baseName, StageName: stageName, Instructions: instructions, Platform: platform}
 }
 
 type DockerfileStage struct {
 	Dockerfile   *Dockerfile
+	Dependencies []*DockerfileStage
+
+	BaseName     string
+	Index        int
+	StageName    string
+	Platform     string
 	Instructions []InstructionInterface
 }
+
+func (stage DockerfileStage) LogName() string {
+	if stage.HasStageName() {
+		return stage.StageName
+	} else {
+		return fmt.Sprintf("<%d>", stage.Index)
+	}
+}
+
+func (stage DockerfileStage) HasStageName() bool {
+	return stage.StageName != ""
+}
+
+func SetupDockerfileStagesDependencies(stages []*DockerfileStage) error {
+	stageByName := make(map[string]*DockerfileStage)
+	for _, stage := range stages {
+		if stage.HasStageName() {
+			stageByName[strings.ToLower(stage.StageName)] = stage
+		}
+	}
+
+	for _, stage := range stages {
+		// Base image dependency
+		if dependency, hasKey := stageByName[strings.ToLower(stage.BaseName)]; hasKey {
+			stage.Dependencies = append(stage.Dependencies, dependency)
+		}
+
+		for _, instr := range stage.Instructions {
+			switch typedInstr := instr.(type) {
+			case *dockerfile_instruction.Copy:
+				if dep := findStageByNameOrIndex(typedInstr.From, stages, stageByName); dep != nil {
+					stage.Dependencies = append(stage.Dependencies, dep)
+				} else {
+					return fmt.Errorf("unable to resolve stage %q instruction %s --from=%q: no such stage", stage.LogName(), instr.Name(), typedInstr.From)
+				}
+
+			case *dockerfile_instruction.Run:
+				for _, mount := range typedInstr.Mounts {
+					if mount.From != "" {
+						if dep := findStageByNameOrIndex(mount.From, stages, stageByName); dep != nil {
+							stage.Dependencies = append(stage.Dependencies, dep)
+						} else {
+							return fmt.Errorf("unable to resolve stage %q instruction %s --mount=from=%s: no such stage", stage.LogName(), instr.Name(), mount.From)
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
+func findStageByNameOrIndex(ref string, stages []*DockerfileStage, stageByName map[string]*DockerfileStage) *DockerfileStage {
+	if stg, found := stageByName[strings.ToLower(ref)]; found {
+		return stg
+	} else if ind, err := strconv.Atoi(ref); err == nil && ind >= 0 && ind < len(stages) {
+		return stages[ind]
+	}
+	return nil
+}

pkg/dockerfile/frontend/buildkit.go

@@ -1,15 +1,53 @@
 package frontend
 
 import (
+	"bytes"
 	"fmt"
+	"strconv"
+	"strings"
 
 	"github.com/moby/buildkit/frontend/dockerfile/instructions"
+	"github.com/moby/buildkit/frontend/dockerfile/parser"
 
 	"github.com/werf/werf/pkg/dockerfile"
 	dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
 )
 
-func DockerfileStageFromBuildkitStage(d *dockerfile.Dockerfile, stage instructions.Stage) (*dockerfile.DockerfileStage, error) {
+func ParseDockerfileWithBuildkit(dockerfileBytes []byte, opts dockerfile.DockerfileOptions) (*dockerfile.Dockerfile, error) {
+	p, err := parser.Parse(bytes.NewReader(dockerfileBytes))
+	if err != nil {
+		return nil, fmt.Errorf("parsing dockerfile data: %w", err)
+	}
+
+	dockerStages, dockerMetaArgs, err := instructions.Parse(p.AST)
+	if err != nil {
+		return nil, fmt.Errorf("parsing instructions tree: %w", err)
+	}
+
+	dockerTargetIndex, err := GetDockerTargetStageIndex(dockerStages, opts.Target)
+	if err != nil {
+		return nil, fmt.Errorf("determine target stage: %w", err)
+	}
+
+	var stages []*dockerfile.DockerfileStage
+	for i, dockerStage := range dockerStages {
+		stages = append(stages, DockerfileStageFromBuildkitStage(i, dockerStage))
+	}
+
+	// TODO(staged-dockerfile): convert meta-args and initialize into Dockerfile obj
+	_ = dockerMetaArgs
+	_ = dockerTargetIndex
+
+	dockerfile.SetupDockerfileStagesDependencies(stages)
+
+	d := dockerfile.NewDockerfile(stages, opts)
+	for _, stage := range d.Stages {
+		stage.Dockerfile = d
+	}
+	return d, nil
+}
+
+func DockerfileStageFromBuildkitStage(index int, stage instructions.Stage) *dockerfile.DockerfileStage {
 	var i []dockerfile.InstructionInterface
 
 	for _, cmd := range stage.Commands {
@@ -56,7 +94,7 @@ func DockerfileStageFromBuildkitStage(d *dockerfile.Dockerfile, stage instructio
 		}
 	}
 
-	return dockerfile.NewDockerfileStage(d, i), nil
+	return dockerfile.NewDockerfileStage(index, stage.BaseName, stage.Name, i, stage.Platform)
 }
 
 func extractSrcAndDst(sourcesAndDest instructions.SourcesAndDest) ([]string, string) {
@@ -75,3 +113,56 @@ func extractKeyValuePairsAsMap(pairs instructions.KeyValuePairs) (res map[string
 	}
 	return
 }
+
+func GetDockerStagesNameToIndexMap(stages []instructions.Stage) map[string]int {
+	nameToIndex := make(map[string]int)
+	for i, s := range stages {
+		name := strings.ToLower(s.Name)
+		if name != strconv.Itoa(i) {
+			nameToIndex[name] = i
+		}
+	}
+	return nameToIndex
+}
+
+func ResolveDockerStagesFromValue(stages []instructions.Stage) {
+	nameToIndex := GetDockerStagesNameToIndexMap(stages)
+
+	for _, s := range stages {
+		for _, cmd := range s.Commands {
+			switch typedCmd := cmd.(type) {
+			case *instructions.CopyCommand:
+				if typedCmd.From != "" {
+					from := strings.ToLower(typedCmd.From)
+					if val, ok := nameToIndex[from]; ok {
+						typedCmd.From = strconv.Itoa(val)
+					}
+				}
+
+			case *instructions.RunCommand:
+				for _, mount := range instructions.GetMounts(typedCmd) {
+					if mount.From != "" {
+						from := strings.ToLower(mount.From)
+						if val, ok := nameToIndex[from]; ok {
+							mount.From = strconv.Itoa(val)
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+func GetDockerTargetStageIndex(dockerStages []instructions.Stage, dockerTargetStage string) (int, error) {
+	if dockerTargetStage == "" {
+		return len(dockerStages) - 1, nil
+	}
+
+	for i, s := range dockerStages {
+		if s.Name == dockerTargetStage {
+			return i, nil
+		}
+	}
+
+	return -1, fmt.Errorf("%s is not a valid target build stage", dockerTargetStage)
+}