Skip to content

Commit

Permalink
feat(staged-dockerfile): prepare conveyor, stage and dockerfile parse…
Browse files Browse the repository at this point in the history 
…r for new impl

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Oct 5, 2022
1 parent 5261f5c commit db8d337
Show file tree
Hide file tree
Showing 14 changed files with 233 additions and 127 deletions.
2 changes: 1 addition & 1 deletion pkg/build/build_phase.go
View file
Expand Up @@ -631,7 +631,7 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *imag
}

switch stg.(type) {
case *stage.DockerfileStage:
case *stage.FullDockerfileStage:
var labels []string
for key, value := range serviceLabels {
labels = append(labels, fmt.Sprintf("%s=%v", key, value))
Expand Down
32 changes: 22 additions & 10 deletions pkg/build/image/dockerfile.go
View file
Expand Up @@ -11,7 +11,6 @@ import (
"github.com/moby/buildkit/frontend/dockerfile/parser"

"github.com/werf/logboek"
"github.com/werf/werf/pkg/build/dockerfile_helpers"
"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/dockerfile"
Expand All @@ -20,12 +19,25 @@ import (
)

func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig *config.ImageFromDockerfile, opts CommonImageOptions) (ImagesSets, error) {
// TODO: check dockerfile-config mode: use legacy (default) dockerfile mapper or new staged-dockerfile mapper
useLegacyMapper := true
if dockerfileImageConfig.Staged {
relDockerfilePath := filepath.Join(dockerfileImageConfig.Context, dockerfileImageConfig.Dockerfile)
dockerfileData, err := opts.GiterminismManager.FileReader().ReadDockerfile(ctx, relDockerfilePath)
if err != nil {
return nil, fmt.Errorf("unable to read dockerfile %s: %w", relDockerfilePath, err)
}

d, err := dockerfile.ParseDockerfile(dockerfileData, dockerfile.DockerfileOptions{
Target: dockerfileImageConfig.Target,
BuildArgs: util.MapStringInterfaceToMapStringString(dockerfileImageConfig.Args),
AddHost: dockerfileImageConfig.AddHost,
Network: dockerfileImageConfig.Network,
SSH: dockerfileImageConfig.SSH,
})
if err != nil {
return nil, fmt.Errorf("unable to parse dockerfile %s: %w", relDockerfilePath, err)
}

if !useLegacyMapper {
// TODO: dockerfileImageConfig to dockerfile.Dockerfile obj
return mapDockerfileToImagesSets(ctx, dockerfile.Dockerfile{})
return mapDockerfileToImagesSets(ctx, d)
}

img, err := mapLegacyDockerfileToImage(ctx, dockerfileImageConfig, opts)
Expand All @@ -40,7 +52,7 @@ func MapDockerfileConfigToImagesSets(ctx context.Context, dockerfileImageConfig
return ret, nil
}

func mapDockerfileToImagesSets(ctx context.Context, cfg dockerfile.Dockerfile) (ImagesSets, error) {
func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile) (ImagesSets, error) {
var ret ImagesSets

stagesSets, err := cfg.GroupStagesByIndependentSets(ctx)
Expand Down Expand Up @@ -136,9 +148,9 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf
return nil, err
}

dockerfile_helpers.ResolveDockerStagesFromValue(dockerStages)
dockerfile.ResolveDockerStagesFromValue(dockerStages)

dockerTargetIndex, err := dockerfile_helpers.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target)
dockerTargetIndex, err := dockerfile.GetDockerTargetStageIndex(dockerStages, dockerfileImageConfig.Target)
if err != nil {
return nil, err
}
Expand All @@ -155,7 +167,7 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf
ProjectName: opts.ProjectName,
}

dockerfileStage := stage.GenerateDockerfileStage(
dockerfileStage := stage.GenerateFullDockerfileStage(
stage.NewDockerRunArgs(
dockerfileData,
dockerfileImageConfig.Dockerfile,
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/image/stapel.go
View file
Expand Up @@ -108,7 +108,7 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap
stages = append(stages, stage.NewGitLatestPatchStage(gitPatchStageOptions, baseStageOptions))
}

stages = appendIfExist(ctx, stages, stage.GenerateDockerInstructionsStage(stapelImageConfig.(*config.StapelImage), baseStageOptions))
stages = appendIfExist(ctx, stages, stage.GenerateStapelDockerInstructionsStage(stapelImageConfig.(*config.StapelImage), baseStageOptions))
}

if len(gitMappings) != 0 {
Expand Down
40 changes: 40 additions & 0 deletions pkg/build/stage/dockerfile_instruction.go
View file
@@ -0,0 +1,40 @@
package stage

import (
"context"

"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/container_backend"
"github.com/werf/werf/pkg/dockerfile"
)

// TODO(staged-dockerfile): common implementation and possible implementation for each separate dockerfile instruction

type DockerfileInstruction interface {
Name() string
}

type DockerfileInstructionStage struct {
*BaseStage

instruction DockerfileInstruction
dependencies []*config.Dependency
}

func NewDockerfileInstructionStage(instruction DockerfileInstruction, dependencies []*config.Dependency, dockerfileStage *dockerfile.DockerfileStage, opts *NewBaseStageOptions) *DockerfileInstructionStage {
return &DockerfileInstructionStage{
instruction: instruction,
dependencies: dependencies,
BaseStage: newBaseStage(StageName(instruction.Name()), opts),
}
}

func (stage *DockerfileInstructionStage) GetDependencies(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevImage, prevBuiltImage *StageImage) (string, error) {
// TODO: digest
return "", nil
}

func (stage *DockerfileInstructionStage) PrepareImage(ctx context.Context, c Conveyor, cb container_backend.ContainerBackend, prevBuiltImage, stageImage *StageImage) error {
// TODO: setup builder
return nil
}
28 changes: 14 additions & 14 deletions pkg/build/stage/dockerfile.go → pkg/build/stage/full_dockerfile.go
View file
Expand Up @@ -34,12 +34,12 @@ func IsErrInvalidBaseImage(err error) bool {
return err != nil && errors.Is(err, ErrInvalidBaseImage)
}

func GenerateDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages, contextChecksum *ContextChecksum, baseStageOptions *NewBaseStageOptions, dependencies []*config.Dependency) *DockerfileStage {
return newDockerfileStage(dockerRunArgs, dockerStages, contextChecksum, baseStageOptions, dependencies)
func GenerateFullDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages, contextChecksum *ContextChecksum, baseStageOptions *NewBaseStageOptions, dependencies []*config.Dependency) *FullDockerfileStage {
return newFullDockerfileStage(dockerRunArgs, dockerStages, contextChecksum, baseStageOptions, dependencies)
}

func newDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages, contextChecksum *ContextChecksum, baseStageOptions *NewBaseStageOptions, dependencies []*config.Dependency) *DockerfileStage {
s := &DockerfileStage{}
func newFullDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages, contextChecksum *ContextChecksum, baseStageOptions *NewBaseStageOptions, dependencies []*config.Dependency) *FullDockerfileStage {
s := &FullDockerfileStage{}
s.DockerRunArgs = dockerRunArgs
s.DockerStages = dockerStages
s.ContextChecksum = contextChecksum
Expand All @@ -49,7 +49,7 @@ func newDockerfileStage(dockerRunArgs *DockerRunArgs, dockerStages *DockerStages
return s
}

type DockerfileStage struct {
type FullDockerfileStage struct {
dependencies []*config.Dependency

*DockerRunArgs
Expand Down Expand Up @@ -319,7 +319,7 @@ type dockerfileInstructionInterface interface {
Name() string
}

func (s *DockerfileStage) FetchDependencies(ctx context.Context, c Conveyor, containerBackend container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error {
func (s *FullDockerfileStage) FetchDependencies(ctx context.Context, c Conveyor, containerBackend container_backend.ContainerBackend, dockerRegistry docker_registry.ApiInterface) error {
resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c)

resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash)
Expand Down Expand Up @@ -411,7 +411,7 @@ func isUnsupportedMediaTypeError(err error) bool {

var errImageNotExistLocally = errors.New("IMAGE_NOT_EXIST_LOCALLY")

func (s *DockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
func (s *FullDockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ container_backend.ContainerBackend, _, _ *StageImage) (string, error) {
resolvedDependenciesArgsHash := resolveDependenciesArgsHash(s.dependencies, c)

resolvedDockerMetaArgsHash, err := s.resolveDockerMetaArgs(resolvedDependenciesArgsHash)
Expand Down Expand Up @@ -501,7 +501,7 @@ func (s *DockerfileStage) GetDependencies(ctx context.Context, c Conveyor, _ con
return util.Sha256Hash(dockerfileStageDependencies...), nil
}

func (s *DockerfileStage) dockerfileInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, cmd interface{}, isOnbuildInstruction, isBaseImageOnbuildInstruction bool) ([]string, []string, error) {
func (s *FullDockerfileStage) dockerfileInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, cmd interface{}, isOnbuildInstruction, isBaseImageOnbuildInstruction bool) ([]string, []string, error) {
var dependencies []string
var onBuildDependencies []string

Expand Down Expand Up @@ -666,7 +666,7 @@ func (s *DockerfileStage) dockerfileInstructionDependencies(ctx context.Context,
return dependencies, onBuildDependencies, nil
}

func (s *DockerfileStage) dockerfileOnBuildInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, expression string, isBaseImageOnbuildInstruction bool) ([]string, []string, error) {
func (s *FullDockerfileStage) dockerfileOnBuildInstructionDependencies(ctx context.Context, giterminismManager giterminism_manager.Interface, resolvedDockerMetaArgsHash, resolvedDependenciesArgsHash map[string]string, dockerStageID int, expression string, isBaseImageOnbuildInstruction bool) ([]string, []string, error) {
p, err := parser.Parse(bytes.NewReader([]byte(expression)))
if err != nil {
return nil, nil, err
Expand All @@ -690,7 +690,7 @@ func (s *DockerfileStage) dockerfileOnBuildInstructionDependencies(ctx context.C
return []string{expression}, onBuildDependencies, nil
}

func (s *DockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, _, stageImage *StageImage) error {
func (s *FullDockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cr container_backend.ContainerBackend, _, stageImage *StageImage) error {
archivePath, err := s.prepareContextArchive(ctx, c.GiterminismManager())
if err != nil {
return err
Expand All @@ -711,7 +711,7 @@ func (s *DockerfileStage) PrepareImage(ctx context.Context, c Conveyor, cr conta
return nil
}

func (s *DockerfileStage) prepareContextArchive(ctx context.Context, giterminismManager giterminism_manager.Interface) (string, error) {
func (s *FullDockerfileStage) prepareContextArchive(ctx context.Context, giterminismManager giterminism_manager.Interface) (string, error) {
contextPathRelativeToGitWorkTree := s.contextRelativeToGitWorkTree(giterminismManager)
contextPathMatcher := path_matcher.NewPathMatcher(path_matcher.PathMatcherOptions{BasePath: contextPathRelativeToGitWorkTree})

Expand Down Expand Up @@ -746,7 +746,7 @@ func (s *DockerfileStage) prepareContextArchive(ctx context.Context, giterminism
return archivePath, nil
}

func (s *DockerfileStage) SetupDockerImageBuilder(b stage_builder.DockerfileBuilderInterface, c Conveyor) error {
func (s *FullDockerfileStage) SetupDockerImageBuilder(b stage_builder.DockerfileBuilderInterface, c Conveyor) error {
b.SetDockerfile(s.dockerfile)
b.SetDockerfileCtxRelPath(s.dockerfilePath)

Expand Down Expand Up @@ -782,7 +782,7 @@ func (s *DockerfileStage) SetupDockerImageBuilder(b stage_builder.DockerfileBuil
return nil
}

func (s *DockerfileStage) calculateFilesChecksum(ctx context.Context, giterminismManager giterminism_manager.Interface, wildcards []string, dockerfileLine string) (string, error) {
func (s *FullDockerfileStage) calculateFilesChecksum(ctx context.Context, giterminismManager giterminism_manager.Interface, wildcards []string, dockerfileLine string) (string, error) {
var checksum string
var err error

Expand Down Expand Up @@ -827,7 +827,7 @@ func (s *DockerfileStage) calculateFilesChecksum(ctx context.Context, giterminis
return checksum, nil
}

func (s *DockerfileStage) calculateFilesChecksumWithGit(ctx context.Context, giterminismManager giterminism_manager.Interface, wildcards []string, dockerfileLine string) (string, error) {
func (s *FullDockerfileStage) calculateFilesChecksumWithGit(ctx context.Context, giterminismManager giterminism_manager.Interface, wildcards []string, dockerfileLine string) (string, error) {
contextPathRelativeToGitWorkTree := s.contextRelativeToGitWorkTree(giterminismManager)
wildcardsPathMatcher := path_matcher.NewPathMatcher(path_matcher.PathMatcherOptions{
BasePath: contextPathRelativeToGitWorkTree,
Expand Down

0 comments on commit db8d337

Please sign in to comment.