Skip to content

Commit

Permalink
feat(staged-dockerfile): implement COPY --from stage to image name ex…
Browse files Browse the repository at this point in the history 
…pansion

* Refactored stage instructions: save backend instruction type by using generics.
* Implemented generic instruction "expansion" mechanism, which should resolve refs to docker dependency stages to built images refs
* Changed digests in all instructions: prepend field name when calculating field digest.

refs #2215

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Oct 27, 2022
1 parent 030044c commit 824c5bb
Show file tree
Hide file tree
Showing 17 changed files with 215 additions and 109 deletions.
33 changes: 18 additions & 15 deletions pkg/build/stage/instruction/add.go
View file
Expand Up @@ -14,33 +14,36 @@ import (
)

type Add struct {
*Base[*dockerfile_instruction.Add]
*Base[*dockerfile_instruction.Add, *backend_instruction.Add]
}

func NewAdd(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Add], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Add {
return &Add{Base: NewBase(name, i, backend_instruction.NewAdd(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Add) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
func (stg *Add) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())
args = append(args, "Raw", stg.instruction.Data.Raw)
args = append(args, append([]string{"Src"}, stg.instruction.Data.Src...)...)
args = append(args, "Dst", stg.instruction.Data.Dst)
args = append(args, "Chown", stg.instruction.Data.Chown)
args = append(args, "Chmod", stg.instruction.Data.Chmod)

args = append(args, stage.instruction.Data.Name())
args = append(args, stage.instruction.Data.Raw)
args = append(args, stage.instruction.Data.Src...)
args = append(args, stage.instruction.Data.Dst)
args = append(args, stage.instruction.Data.Chown)
args = append(args, stage.instruction.Data.Chmod)
pathsChecksum, err := buildContextArchive.CalculatePathsChecksum(ctx, stg.instruction.Data.Src)
if err != nil {
return "", fmt.Errorf("unable to calculate build context paths checksum: %w", err)
}
args = append(args, "SrcChecksum", pathsChecksum)

// TODO(staged-dockerfile): support http src and --checksum option: https://docs.docker.com/engine/reference/builder/#verifying-a-remote-file-checksum-add---checksumchecksum-http-src-dest
// TODO(staged-dockerfile): support git ref: https://docs.docker.com/engine/reference/builder/#adding-a-git-repository-add-git-ref-dir
// TODO(staged-dockerfile): support --keep-git-dir for git: https://docs.docker.com/engine/reference/builder/#adding-a-git-repository-add-git-ref-dir
// TODO(staged-dockerfile): support --link

pathsChecksum, err := buildContextArchive.CalculatePathsChecksum(ctx, stage.instruction.Data.Src)
if err != nil {
return "", fmt.Errorf("unable to calculate build context paths checksum: %w", err)
}
args = append(args, fmt.Sprintf("src-checksum=%s", pathsChecksum))

return util.Sha256Hash(args...), nil
}
32 changes: 24 additions & 8 deletions pkg/build/stage/instruction/base.go
View file
Expand Up @@ -2,24 +2,25 @@ package instruction

import (
"context"
"fmt"

"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/dockerfile"
)

type Base[T dockerfile.InstructionDataInterface] struct {
type Base[T dockerfile.InstructionDataInterface, BT container_backend.InstructionInterface] struct {
*stage.BaseStage

instruction *dockerfile.DockerfileStageInstruction[T]
backendInstruction container_backend.InstructionInterface
backendInstruction BT
dependencies []*config.Dependency
hasPrevStage bool
}

func NewBase[T dockerfile.InstructionDataInterface](name stage.StageName, instruction *dockerfile.DockerfileStageInstruction[T], backendInstruction container_backend.InstructionInterface, dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Base[T] {
return &Base[T]{
func NewBase[T dockerfile.InstructionDataInterface, BT container_backend.InstructionInterface](name stage.StageName, instruction *dockerfile.DockerfileStageInstruction[T], backendInstruction BT, dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Base[T, BT] {
return &Base[T, BT]{
BaseStage: stage.NewBaseStage(name, opts),
instruction: instruction,
backendInstruction: backendInstruction,
Expand All @@ -28,19 +29,34 @@ func NewBase[T dockerfile.InstructionDataInterface](name stage.StageName, instru
}
}

func (stg *Base[T]) HasPrevStage() bool {
func (stg *Base[T, BT]) HasPrevStage() bool {
return stg.hasPrevStage
}

func (stg *Base[T]) IsStapelStage() bool {
func (stg *Base[T, BT]) IsStapelStage() bool {
return false
}

func (stg *Base[T]) UsesBuildContext() bool {
func (stg *Base[T, BT]) UsesBuildContext() bool {
return stg.backendInstruction.UsesBuildContext()
}

func (stg *Base[T]) PrepareImage(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error {
func (stg *Base[T, BT]) getDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver, expander InstructionExpander) ([]string, error) {
if err := expander.ExpandInstruction(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive); err != nil {
return nil, fmt.Errorf("unable to expand instruction %q: %w", stg.instruction.Data.Name(), err)
}
return nil, nil
}

func (stg *Base[T, BT]) PrepareImage(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error {
stageImage.Builder.DockerfileStageBuilder().AppendInstruction(stg.backendInstruction)
return nil
}

func (stg *Base[T, BT]) ExpandInstruction(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error {
return nil
}

type InstructionExpander interface {
ExpandInstruction(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error
}
17 changes: 11 additions & 6 deletions pkg/build/stage/instruction/cmd.go
View file
Expand Up @@ -14,17 +14,22 @@ import (
)

type Cmd struct {
*Base[*dockerfile_instruction.Cmd]
*Base[*dockerfile_instruction.Cmd, *backend_instruction.Cmd]
}

func NewCmd(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Cmd], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Cmd {
return &Cmd{Base: NewBase(name, i, backend_instruction.NewCmd(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Cmd) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
args = append(args, stage.instruction.Data.Name())
args = append(args, stage.instruction.Data.Cmd...)
args = append(args, fmt.Sprintf("%v", stage.instruction.Data.PrependShell))
func (stg *Cmd) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())
args = append(args, append([]string{"Cmd"}, stg.instruction.Data.Cmd...)...)
args = append(args, "PrependShell", fmt.Sprintf("%v", stg.instruction.Data.PrependShell))

return util.Sha256Hash(args...), nil
}
37 changes: 28 additions & 9 deletions pkg/build/stage/instruction/copy.go
View file
Expand Up @@ -13,20 +13,39 @@ import (
)

type Copy struct {
*Base[*dockerfile_instruction.Copy]
*Base[*dockerfile_instruction.Copy, *backend_instruction.Copy]
}

func NewCopy(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Copy], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Copy {
return &Copy{Base: NewBase(name, i, backend_instruction.NewCopy(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Copy) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
args = append(args, stage.instruction.Data.Name())
args = append(args, stage.instruction.Data.From)
args = append(args, stage.instruction.Data.Src...)
args = append(args, stage.instruction.Data.Dst)
args = append(args, stage.instruction.Data.Chown)
args = append(args, stage.instruction.Data.Chmod)
func (stg *Copy) ExpandInstruction(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) error {
if stg.instruction.Data.From != "" {
if ds := stg.instruction.GetDependencyByStageRef(stg.instruction.Data.From); ds != nil {
depStageImageName := c.GetImageNameForLastImageStage(ds.WerfImageName())
stg.backendInstruction.From = depStageImageName
}
}

return nil
}

func (stg *Copy) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())
args = append(args, "From", stg.instruction.Data.From)
args = append(args, append([]string{"Src"}, stg.instruction.Data.Src...)...)
args = append(args, "Dst", stg.instruction.Data.Dst)
args = append(args, "Chown", stg.instruction.Data.Chown)
args = append(args, "Chmod", stg.instruction.Data.Chmod)
args = append(args, "ExpandedFrom", stg.backendInstruction.From)

// TODO(staged-dockerfile): support --link option: https://docs.docker.com/engine/reference/builder/#copy---link

return util.Sha256Hash(args...), nil
}
16 changes: 10 additions & 6 deletions pkg/build/stage/instruction/entrypoint.go
View file
Expand Up @@ -14,17 +14,21 @@ import (
)

type Entrypoint struct {
*Base[*dockerfile_instruction.Entrypoint]
*Base[*dockerfile_instruction.Entrypoint, *backend_instruction.Entrypoint]
}

func NewEntrypoint(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Entrypoint], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Entrypoint {
return &Entrypoint{Base: NewBase(name, i, backend_instruction.NewEntrypoint(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Entrypoint) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
args = append(args, stage.instruction.Data.Name())
args = append(args, stage.instruction.Data.Entrypoint...)
args = append(args, fmt.Sprintf("%v", stage.instruction.Data.PrependShell))
func (stg *Entrypoint) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())
args = append(args, append([]string{"Entrypoint"}, stg.instruction.Data.Entrypoint...)...)
args = append(args, "PrependShell", fmt.Sprintf("%v", stg.instruction.Data.PrependShell))
return util.Sha256Hash(args...), nil
}
19 changes: 13 additions & 6 deletions pkg/build/stage/instruction/env.go
View file
Expand Up @@ -13,19 +13,26 @@ import (
)

type Env struct {
*Base[*dockerfile_instruction.Env]
*Base[*dockerfile_instruction.Env, *backend_instruction.Env]
}

func NewEnv(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Env], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Env {
return &Env{Base: NewBase(name, i, backend_instruction.NewEnv(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Env) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
args = append(args, stage.instruction.Data.Name())
func (stg *Env) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())
// FIXME(staged-dockerfile): sort envs
for k, v := range stage.instruction.Data.Envs {
args = append(args, k, v)
if len(stg.instruction.Data.Envs) > 0 {
args = append(args, "Envs")
for k, v := range stg.instruction.Data.Envs {
args = append(args, k, v)
}
}
return util.Sha256Hash(args...), nil
}
14 changes: 9 additions & 5 deletions pkg/build/stage/instruction/expose.go
View file
Expand Up @@ -13,16 +13,20 @@ import (
)

type Expose struct {
*Base[*dockerfile_instruction.Expose]
*Base[*dockerfile_instruction.Expose, *backend_instruction.Expose]
}

func NewExpose(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Expose], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Expose {
return &Expose{Base: NewBase(name, i, backend_instruction.NewExpose(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Expose) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
args = append(args, stage.instruction.Data.Name())
args = append(args, stage.instruction.Data.Ports...)
func (stg *Expose) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())
args = append(args, append([]string{"Ports"}, stg.instruction.Data.Ports...)...)
return util.Sha256Hash(args...), nil
}
24 changes: 14 additions & 10 deletions pkg/build/stage/instruction/healthcheck.go
View file
Expand Up @@ -14,21 +14,25 @@ import (
)

type Healthcheck struct {
*Base[*dockerfile_instruction.Healthcheck]
*Base[*dockerfile_instruction.Healthcheck, *backend_instruction.Healthcheck]
}

func NewHealthcheck(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Healthcheck], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Healthcheck {
return &Healthcheck{Base: NewBase(name, i, backend_instruction.NewHealthcheck(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Healthcheck) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
args = append(args, stage.instruction.Data.Name())
args = append(args, string(stage.instruction.Data.Type))
args = append(args, stage.instruction.Data.Config.Test...)
args = append(args, stage.instruction.Data.Config.Interval.String())
args = append(args, stage.instruction.Data.Config.Timeout.String())
args = append(args, stage.instruction.Data.Config.StartPeriod.String())
args = append(args, fmt.Sprintf("%d", stage.instruction.Data.Config.Retries))
func (stg *Healthcheck) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())
args = append(args, "Type", string(stg.instruction.Data.Type))
args = append(args, append([]string{"Test"}, stg.instruction.Data.Config.Test...)...)
args = append(args, "Interval", stg.instruction.Data.Config.Interval.String())
args = append(args, "Timeout", stg.instruction.Data.Config.Timeout.String())
args = append(args, "StartPeriod", stg.instruction.Data.Config.StartPeriod.String())
args = append(args, "Retries", fmt.Sprintf("%d", stg.instruction.Data.Config.Retries))
return util.Sha256Hash(args...), nil
}
21 changes: 15 additions & 6 deletions pkg/build/stage/instruction/label.go
View file
Expand Up @@ -13,19 +13,28 @@ import (
)

type Label struct {
*Base[*dockerfile_instruction.Label]
*Base[*dockerfile_instruction.Label, *backend_instruction.Label]
}

func NewLabel(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Label], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Label {
return &Label{Base: NewBase(name, i, backend_instruction.NewLabel(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Label) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
args = append(args, stage.instruction.Data.Name())
func (stg *Label) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())

// FIXME(staged-dockerfile): sort labels map
for k, v := range stage.instruction.Data.Labels {
args = append(args, k, v)
if len(stg.instruction.Data.Labels) > 0 {
args = append(args, "Labels")
for k, v := range stg.instruction.Data.Labels {
args = append(args, k, v)
}
}

return util.Sha256Hash(args...), nil
}
14 changes: 9 additions & 5 deletions pkg/build/stage/instruction/maintainer.go
View file
Expand Up @@ -13,16 +13,20 @@ import (
)

type Maintainer struct {
*Base[*dockerfile_instruction.Maintainer]
*Base[*dockerfile_instruction.Maintainer, *backend_instruction.Maintainer]
}

func NewMaintainer(name stage.StageName, i *dockerfile.DockerfileStageInstruction[*dockerfile_instruction.Maintainer], dependencies []*config.Dependency, hasPrevStage bool, opts *stage.BaseStageOptions) *Maintainer {
return &Maintainer{Base: NewBase(name, i, backend_instruction.NewMaintainer(*i.Data), dependencies, hasPrevStage, opts)}
}

func (stage *Maintainer) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
var args []string
args = append(args, stage.instruction.Data.Name())
args = append(args, stage.instruction.Data.Maintainer)
func (stg *Maintainer) GetDependencies(ctx context.Context, c stage.Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *stage.StageImage, buildContextArchive container_backend.BuildContextArchiver) (string, error) {
args, err := stg.getDependencies(ctx, c, cb, prevImage, prevBuiltImage, buildContextArchive, stg)
if err != nil {
return "", err
}

args = append(args, "Instruction", stg.instruction.Data.Name())
args = append(args, "Maintainer", stg.instruction.Data.Maintainer)
return util.Sha256Hash(args...), nil
}

0 comments on commit 824c5bb

Please sign in to comment.