Skip to content

Commit

Permalink
feat(staged-dockerfile): map dockerfile stages with dependencies to w…
Browse files Browse the repository at this point in the history 
…erf internal images

* Support dependencies between dockerfile stages.
* Use generics for `build/stage/instruction`.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Oct 19, 2022
1 parent be18d80 commit f5f200e
Show file tree
Hide file tree
Showing 9 changed files with 208 additions and 115 deletions.
97 changes: 71 additions & 26 deletions pkg/build/image/dockerfile.go
View file
Expand Up @@ -14,7 +14,6 @@ import (
"github.com/werf/werf/pkg/build/stage"
stage_instruction "github.com/werf/werf/pkg/build/stage/instruction"
"github.com/werf/werf/pkg/config"
backend_instruction "github.com/werf/werf/pkg/container_backend/instruction"
"github.com/werf/werf/pkg/dockerfile"
"github.com/werf/werf/pkg/dockerfile/frontend"
dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
Expand Down Expand Up @@ -60,40 +59,86 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig
func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile, dockerfileImageConfig *config.ImageFromDockerfile, opts CommonImageOptions) (ImagesSets, error) {
var ret ImagesSets

stagesSets, err := cfg.GroupStagesByIndependentSets(ctx)
targetStage, err := cfg.GetTargetStage()
if err != nil {
return nil, fmt.Errorf("unable to group dockerfile stages by independent sets :%w", err)
return nil, fmt.Errorf("unable to get target dockerfile stage: %w", err)
}

for _, set := range stagesSets {
for _, stg := range set {
_ = stg
// TODO(staged-dockerfile): map stages to *Image+build.Stage objects
// ret = append(ret, stg.)
}
queue := []struct {
Stage *dockerfile.DockerfileStage
Level int
}{
{Stage: targetStage, Level: 0},
}

{
// TODO parse FROM instruction properly, set correct BaseImageReference here
appendQueue := func(stage *dockerfile.DockerfileStage, level int) {
queue = append(queue, struct {
Stage *dockerfile.DockerfileStage
Level int
}{Stage: stage, Level: level})
}

img, err := NewImage(ctx, "test", ImageFromRegistryAsBaseImage, ImageOptions{
IsDockerfileImage: true,
DockerfileImageConfig: dockerfileImageConfig,
CommonImageOptions: opts,
BaseImageReference: "ubuntu:22.04",
})
if err != nil {
return nil, fmt.Errorf("unable to create image %q: %w", "test", err)
for len(queue) > 0 {
item := queue[0]
queue = queue[1:]

appendImageToCurrentSet := func(img *Image) {
if item.Level == len(ret) {
ret = append([][]*Image{nil}, ret...)
}
ret[len(ret)-item.Level-1] = append(ret[len(ret)-item.Level-1], img)
}

stg := item.Stage

var img *Image
var err error
if baseStg := cfg.FindStage(stg.BaseName); baseStg != nil {
img, err = NewImage(ctx, dockerfileImageConfig.Name, StageAsBaseImage, ImageOptions{
IsDockerfileImage: true,
DockerfileImageConfig: dockerfileImageConfig,
CommonImageOptions: opts,
BaseImageName: baseStg.WerfImageName(),
})
if err != nil {
return nil, fmt.Errorf("unable to map stage %s to werf image %q: %w", stg.LogName(), dockerfileImageConfig.Name, err)
}

appendQueue(baseStg, item.Level+1)
} else {
img, err = NewImage(ctx, dockerfileImageConfig.Name, ImageFromRegistryAsBaseImage, ImageOptions{
IsDockerfileImage: true,
DockerfileImageConfig: dockerfileImageConfig,
CommonImageOptions: opts,
BaseImageReference: targetStage.BaseName,
})
if err != nil {
return nil, fmt.Errorf("unable to map stage %s to werf image %q: %w", targetStage.LogName(), dockerfileImageConfig.Name, err)
}
}

img.stages = append(img.stages, stage_instruction.NewRun(backend_instruction.NewRun(*dockerfile_instruction.NewRun([]string{"ls", "/"}, false, nil, "", "")), nil, false, &stage.BaseStageOptions{
ImageName: img.Name,
ImageTmpDir: img.TmpDir,
ContainerWerfDir: img.ContainerWerfDir,
ProjectName: opts.ProjectName,
}))
for ind, instr := range stg.Instructions {
switch typedInstr := any(instr).(type) {
case *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Run]:
isFirstStage := (len(img.stages) == 0)

img.stages = append(img.stages, stage_instruction.NewRun(stage.StageName(fmt.Sprintf("%d-%s", ind, typedInstr.Data.Name())), typedInstr, dockerfileImageConfig.Dependencies, !isFirstStage, &stage.BaseStageOptions{
ImageName: img.Name,
ImageTmpDir: img.TmpDir,
ContainerWerfDir: img.ContainerWerfDir,
ProjectName: opts.ProjectName,
}))

default:
panic(fmt.Sprintf("unsupported instruction type %#v", instr))
}

for _, dep := range instr.GetDependenciesByStageRef() {
appendQueue(dep, item.Level+1)
}
}

ret = append(ret, []*Image{img})
appendImageToCurrentSet(img)
}

return ret, nil
Expand Down
4 changes: 2 additions & 2 deletions pkg/build/image/image.go
View file
Expand Up @@ -22,8 +22,8 @@ import (
type BaseImageType string

const (
ImageFromRegistryAsBaseImage BaseImageType = "ImageFromRegistryBaseImage"
StageAsBaseImage BaseImageType = "StageBaseImage"
ImageFromRegistryAsBaseImage BaseImageType = "ImageFromRegistryAsBaseImage"
StageAsBaseImage BaseImageType = "StageAsBaseImage"
NoBaseImage BaseImageType = "NoBaseImage"
)

Expand Down
13 changes: 8 additions & 5 deletions pkg/build/stage/instruction/base.go
View file
Expand Up @@ -3,27 +3,30 @@ package instruction
import (
"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/dockerfile"
)

type Base struct {
type Base[T dockerfile.InstructionDataInterface] struct {
*stage.BaseStage

instruction *dockerfile.DockerfileStageInstruction[T]
dependencies []*config.Dependency
hasPrevStage bool
}

func NewBase(name stage.StageName, dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Base {
return &Base{
func NewBase[T dockerfile.InstructionDataInterface](name stage.StageName, instruction *dockerfile.DockerfileStageInstruction[T], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Base[T] {
return &Base[T]{
BaseStage: stage.NewBaseStage(name, opts),
instruction: instruction,
dependencies: dependencies,
hasPrevStage: hasPrevStage,
}
}

func (stage *Base) HasPrevStage() bool {
func (stage *Base[T]) HasPrevStage() bool {
return stage.hasPrevStage
}

func (s *Base) IsStapelStage() bool {
func (s *Base[T]) IsStapelStage() bool {
return false
}
21 changes: 0 additions & 21 deletions pkg/build/stage/instruction/instruction.go
View file

This file was deleted.

16 changes: 7 additions & 9 deletions pkg/build/stage/instruction/run.go
View file
Expand Up @@ -7,27 +7,25 @@ import (
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/container_backend"
backend_instruction "github.com/werf/werf/pkg/container_backend/instruction"
"github.com/werf/werf/pkg/dockerfile"
dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
"github.com/werf/werf/pkg/util"
)

type Run struct {
*Base
instruction *backend_instruction.Run
*Base[*dockerfile_instruction.Run]
}

func NewRun(i *backend_instruction.Run, dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Run {
return &Run{
Base: NewBase(InstructionRun, dependencies, hasPrevStage, opts),
instruction: i,
}
func NewRun(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Run], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Run {
return &Run{Base: NewBase(name, i, dependencies, hasPrevStage, opts)}
}

func (stage *Run) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
return util.Sha256Hash(append([]string{string(InstructionRun)}, stage.instruction.Command...)...), nil
return util.Sha256Hash(append([]string{stage.instruction.Data.Name()}, stage.instruction.Data.Command...)...), nil
}

func (stage *Run) PrepareImage(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error {
stageImage.Builder.DockerfileStageBuilder().SetBuildContextArchive(buildContextArchive)
stageImage.Builder.DockerfileStageBuilder().AppendInstruction(stage.instruction)
stageImage.Builder.DockerfileStageBuilder().AppendInstruction(backend_instruction.NewRun(*stage.instruction.Data))
return nil
}
37 changes: 19 additions & 18 deletions pkg/dockerfile/dockerfile.go
View file
@@ -1,7 +1,7 @@
package dockerfile

import (
"context"
"fmt"
)

type DockerfileOptions struct {
Expand All @@ -25,24 +25,25 @@ type Dockerfile struct {
Stages []*DockerfileStage
}

func (df *Dockerfile) GroupStagesByIndependentSets(ctx context.Context) ([][]*DockerfileStage, error) {
// FIXME(staged-dockerfile): build real dependencies tree

// var res [][]*DockerfileStage
// var curLevel []*DockerfileStage

// stagesQueue
func (df *Dockerfile) GetTargetStage() (*DockerfileStage, error) {
if df.Target == "" {
return df.Stages[len(df.Stages)-1], nil
}

// res = append(res, curLevel)
for _, s := range df.Stages {
if s.StageName == df.Target {
return s, nil
}
}

// for _, stg := range df.Stages {
// stg.Dependencies
// }
return nil, fmt.Errorf("%s is not a valid target dockerfile stage", df.Target)
}

// var res [][]*DockerfileStage
// for _, stg := range df.Stages {
// res = append(res, []*DockerfileStage{stg})
// }
// return res, nil
return nil, nil
func (df *Dockerfile) FindStage(name string) *DockerfileStage {
for _, s := range df.Stages {
if s.StageName == name {
return s
}
}
return nil
}
55 changes: 40 additions & 15 deletions pkg/dockerfile/dockerfile_stage.go
View file
Expand Up @@ -8,30 +8,48 @@ import (
dockerfile_instruction "github.com/werf/werf/pkg/dockerfile/instruction"
)

func NewDockerfileStage(index int, baseName, stageName string, instructions []InstructionInterface, platform string) *DockerfileStage {
func NewDockerfileStage(index int, baseName, stageName string, instructions []DockerfileStageInstructionInterface, platform string) *DockerfileStage {
return &DockerfileStage{BaseName: baseName, StageName: stageName, Instructions: instructions, Platform: platform}
}

type DockerfileStage struct {
Dockerfile *Dockerfile
Dependencies []*DockerfileStage
BaseStage *DockerfileStage

BaseName string
Index int
StageName string
Platform string
Instructions []InstructionInterface
Instructions []DockerfileStageInstructionInterface
}

func (stage DockerfileStage) LogName() string {
func (stage *DockerfileStage) AppendDependencyStage(dep *DockerfileStage) {
for _, d := range stage.Dependencies {
if d.Index == dep.Index {
return
}
}
stage.Dependencies = append(stage.Dependencies, dep)
}

func (stage *DockerfileStage) WerfImageName() string {
if stage.HasStageName() {
return fmt.Sprintf("dockerfile-stage-%s", stage.StageName)
} else {
return fmt.Sprintf("dockerfile-stage-%d", stage.Index)
}
}

func (stage *DockerfileStage) LogName() string {
if stage.HasStageName() {
return stage.StageName
} else {
return fmt.Sprintf("<%d>", stage.Index)
}
}

func (stage DockerfileStage) HasStageName() bool {
func (stage *DockerfileStage) HasStageName() bool {
return stage.StageName != ""
}

Expand All @@ -45,37 +63,44 @@ func SetupDockerfileStagesDependencies(stages []*DockerfileStage) error {

for _, stage := range stages {
// Base image dependency
if dependency, hasKey := stageByName[strings.ToLower(stage.BaseName)]; hasKey {
stage.Dependencies = append(stage.Dependencies, dependency)
if baseStage, hasKey := stageByName[strings.ToLower(stage.BaseName)]; hasKey {
stage.BaseStage = baseStage
stage.Dependencies = append(stage.Dependencies, baseStage)
}

for _, instr := range stage.Instructions {
switch typedInstr := instr.(type) {
switch typedInstr := instr.GetInstructionData().(type) {
case *dockerfile_instruction.Copy:
if dep := findStageByNameOrIndex(typedInstr.From, stages, stageByName); dep != nil {
stage.Dependencies = append(stage.Dependencies, dep)
} else {
return fmt.Errorf("unable to resolve stage %q instruction %s --from=%q: no such stage", stage.LogName(), instr.Name(), typedInstr.From)
if typedInstr.From != "" {
if dep := findStageByRef(typedInstr.From, stages, stageByName); dep != nil {
stage.AppendDependencyStage(dep)
instr.SetDependencyByStageRef(typedInstr.From, dep)
} else {
return fmt.Errorf("unable to resolve stage %q instruction %s --from=%q: no such stage", stage.LogName(), instr.GetInstructionData().Name(), typedInstr.From)
}
}

case *dockerfile_instruction.Run:
for _, mount := range typedInstr.Mounts {
if mount.From != "" {
if dep := findStageByNameOrIndex(mount.From, stages, stageByName); dep != nil {
stage.Dependencies = append(stage.Dependencies, dep)
if dep := findStageByRef(mount.From, stages, stageByName); dep != nil {
stage.AppendDependencyStage(dep)
instr.SetDependencyByStageRef(mount.From, dep)
} else {
return fmt.Errorf("unable to resolve stage %q instruction %s --mount=from=%s: no such stage", stage.LogName(), instr.Name(), mount.From)
return fmt.Errorf("unable to resolve stage %q instruction %s --mount=from=%s: no such stage", stage.LogName(), instr.GetInstructionData().Name(), mount.From)
}
}
}

}
}
}

return nil
}

func findStageByNameOrIndex(ref string, stages []*DockerfileStage, stageByName map[string]*DockerfileStage) *DockerfileStage {
// findStageByRef finds stage by stage reference which is stage index or stage name
func findStageByRef(ref string, stages []*DockerfileStage, stageByName map[string]*DockerfileStage) *DockerfileStage {
if stg, found := stageByName[strings.ToLower(ref)]; found {
return stg
} else if ind, err := strconv.Atoi(ref); err == nil && ind >= 0 && ind < len(stages) {
Expand Down

0 comments on commit f5f200e

Please sign in to comment.