feat(staged-dockerfile): refactored conveyor, debug container backend…

… staged dockerfile builder

* Implemented test instruction building using new staged-dockerfile building method of container backend.
* Refactored conveyor so that new builder now correctly coupled to the existing building mechanics.

refs #2215

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>

pkg/build/build_phase.go

@@ -451,7 +451,7 @@ func (phase *BuildPhase) onImageStage(ctx context.Context, img *image.Image, stg
 		}
 
 		// Will build a new stage
-		i := phase.Conveyor.GetOrCreateStageImage(extractLegacyStageImage(phase.StagesIterator.GetPrevImage(img, stg)), uuid.New().String())
+		i := phase.Conveyor.GetOrCreateStageImage(uuid.New().String(), phase.StagesIterator.GetPrevImage(img, stg), stg, img)
 		stg.SetStageImage(i)
 
 		if err := phase.fetchBaseImageForStage(ctx, img, stg); err != nil {
@@ -502,7 +502,7 @@ func (phase *BuildPhase) findAndFetchStageFromSecondaryStagesStorage(ctx context
 			if copiedStageDesc, err := storageManager.CopySuitableByDigestStage(ctx, secondaryStageDesc, secondaryStagesStorage, storageManager.GetStagesStorage(), phase.Conveyor.ContainerBackend); err != nil {
 				return fmt.Errorf("unable to copy suitable stage %s from %s to %s: %w", secondaryStageDesc.StageID.String(), secondaryStagesStorage.String(), storageManager.GetStagesStorage().String(), err)
 			} else {
-				i := phase.Conveyor.GetOrCreateStageImage(extractLegacyStageImage(phase.StagesIterator.GetPrevImage(img, stg)), copiedStageDesc.Info.Name)
+				i := phase.Conveyor.GetOrCreateStageImage(copiedStageDesc.Info.Name, phase.StagesIterator.GetPrevImage(img, stg), stg, img)
 				i.Image.SetStageDescription(copiedStageDesc)
 				stg.SetStageImage(i)
 
@@ -547,27 +547,16 @@ ScanSecondaryStagesStorageList:
 }
 
 func (phase *BuildPhase) fetchBaseImageForStage(ctx context.Context, img *image.Image, stg stage.Interface) error {
-	switch {
-	case stg.Name() == "from":
+	if stg.HasPrevStage() {
+		return phase.Conveyor.StorageManager.FetchStage(ctx, phase.Conveyor.ContainerBackend, phase.StagesIterator.PrevBuiltStage)
+	} else {
 		if err := img.FetchBaseImage(ctx); err != nil {
-			return fmt.Errorf("unable to fetch base image %s for stage %s: %w", img.GetBaseImage().Image.Name(), stg.LogDetailedName(), err)
+			return fmt.Errorf("unable to fetch base image %s for stage %s: %w", img.GetBaseStageImage().Image.Name(), stg.LogDetailedName(), err)
 		}
-	case stg.Name() == "dockerfile":
-		return nil
-	case stg.HasPrevStage():
-		return phase.Conveyor.StorageManager.FetchStage(ctx, phase.Conveyor.ContainerBackend, phase.StagesIterator.PrevBuiltStage)
 	}
-
 	return nil
 }
 
-func extractLegacyStageImage(stageImage *stage.StageImage) *container_backend.LegacyStageImage {
-	if stageImage == nil || stageImage.Image == nil {
-		return nil
-	}
-	return stageImage.Image.(*container_backend.LegacyStageImage)
-}
-
 func (phase *BuildPhase) calculateStage(ctx context.Context, img *image.Image, stg stage.Interface) (bool, func(), error) {
 	// FIXME(stapel-to-buildah): store StageImage-s everywhere in stage and build pkgs
 	stageDependencies, err := stg.GetDependencies(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevImage(img, stg), phase.StagesIterator.GetPrevBuiltImage(img, stg))
@@ -597,7 +586,7 @@ func (phase *BuildPhase) calculateStage(ctx context.Context, img *image.Image, s
 		if stageDesc, err := storageManager.SelectSuitableStage(ctx, phase.Conveyor, stg, stages); err != nil {
 			return false, phase.Conveyor.GetStageDigestMutex(stg.GetDigest()).Unlock, err
 		} else if stageDesc != nil {
-			i := phase.Conveyor.GetOrCreateStageImage(extractLegacyStageImage(phase.StagesIterator.GetPrevImage(img, stg)), stageDesc.Info.Name)
+			i := phase.Conveyor.GetOrCreateStageImage(stageDesc.Info.Name, phase.StagesIterator.GetPrevImage(img, stg), stg, img)
 			i.Image.SetStageDescription(stageDesc)
 			stg.SetStageImage(i)
 			foundSuitableStage = true
@@ -630,19 +619,7 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *imag
 		imagePkg.WerfStageContentDigestLabel: stg.GetContentDigest(),
 	}
 
-	switch stg.(type) {
-	case *stage.FullDockerfileStage:
-		var labels []string
-		for key, value := range serviceLabels {
-			labels = append(labels, fmt.Sprintf("%s=%v", key, value))
-		}
-		stageImage.Builder.DockerfileBuilder().AppendLabels(labels...)
-
-		phase.Conveyor.AppendOnTerminateFunc(func() error {
-			return stageImage.Builder.DockerfileBuilder().Cleanup(ctx)
-		})
-
-	default:
+	if stg.IsStapelStage() {
 		if phase.Conveyor.UseLegacyStapelBuilder(phase.Conveyor.ContainerBackend) {
 			stageImage.Builder.LegacyStapelStageBuilder().Container().ServiceCommitChangeOptions().AddLabel(serviceLabels)
 		} else {
@@ -688,6 +665,19 @@ func (phase *BuildPhase) prepareStageInstructions(ctx context.Context, img *imag
 		} else {
 			stageImage.Builder.StapelStageBuilder().AddEnvs(commitEnvs)
 		}
+	} else if _, ok := stg.(*stage.FullDockerfileStage); ok {
+		var labels []string
+		for key, value := range serviceLabels {
+			labels = append(labels, fmt.Sprintf("%s=%v", key, value))
+		}
+		stageImage.Builder.DockerfileBuilder().AppendLabels(labels...)
+
+		phase.Conveyor.AppendOnTerminateFunc(func() error {
+			return stageImage.Builder.DockerfileBuilder().Cleanup(ctx)
+		})
+	} else {
+		stageImage.Builder.DockerfileStageBuilder().AppendPostCommands(&container_backend.InstructionLabel{Labels: serviceLabels})
+		// staged dockerfile
 	}
 
 	err := stg.PrepareImage(ctx, phase.Conveyor, phase.Conveyor.ContainerBackend, phase.StagesIterator.GetPrevBuiltImage(img, stg), stageImage)
@@ -789,7 +779,7 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *image.I
 				stg.LogDetailedName(), stg.GetDigest(), stageDesc.Info.Name,
 			)
 
-			i := phase.Conveyor.GetOrCreateStageImage(extractLegacyStageImage(phase.StagesIterator.GetPrevImage(img, stg)), stageDesc.Info.Name)
+			i := phase.Conveyor.GetOrCreateStageImage(stageDesc.Info.Name, phase.StagesIterator.GetPrevImage(img, stg), stg, img)
 			i.Image.SetStageDescription(stageDesc)
 			stg.SetStageImage(i)
 

pkg/build/conveyor.go

@@ -632,16 +632,34 @@ func (c *Conveyor) SetStageImage(stageImage *stage.StageImage) {
 	c.stageImages[stageImage.Image.Name()] = stageImage
 }
 
-func (c *Conveyor) GetOrCreateStageImage(fromImage *container_backend.LegacyStageImage, name string) *stage.StageImage {
-	if img := c.GetStageImage(name); img != nil {
-		return img
+func extractLegacyStageImage(stageImage *stage.StageImage) *container_backend.LegacyStageImage {
+	if stageImage == nil || stageImage.Image == nil {
+		return nil
+	}
+	return stageImage.Image.(*container_backend.LegacyStageImage)
+}
+
+func (c *Conveyor) GetOrCreateStageImage(name string, prevStageImage *stage.StageImage, stg stage.Interface, img *image.Image) *stage.StageImage {
+	if stageImage := c.GetStageImage(name); stageImage != nil {
+		return stageImage
 	}
 
-	i := container_backend.NewLegacyStageImage(fromImage, name, c.ContainerBackend)
-	img := stage.NewStageImage(c.ContainerBackend, fromImage, i)
+	i := container_backend.NewLegacyStageImage(extractLegacyStageImage(prevStageImage), name, c.ContainerBackend)
+
+	var baseImage string
+	if stg != nil {
+		if stg.HasPrevStage() {
+			baseImage = prevStageImage.Image.Name()
+		} else if stg.IsStapelStage() && stg.Name() == "from" {
+			baseImage = prevStageImage.Image.Name()
+		} else {
+			baseImage = img.GetBaseImageReference()
+		}
+	}
 
-	c.SetStageImage(img)
-	return img
+	stageImage := stage.NewStageImage(c.ContainerBackend, baseImage, i)
+	c.SetStageImage(stageImage)
+	return stageImage
 }
 
 func (c *Conveyor) GetImage(name string) *image.Image {

pkg/build/image/conveyor.go

@@ -4,15 +4,14 @@ import (
 	"sync"
 
 	"github.com/werf/werf/pkg/build/stage"
-	"github.com/werf/werf/pkg/container_backend"
 	"github.com/werf/werf/pkg/git_repo"
 )
 
 type Conveyor interface {
 	stage.Conveyor
 
 	GetImage(name string) *Image
-	GetOrCreateStageImage(fromImage *container_backend.LegacyStageImage, name string) *stage.StageImage
+	GetOrCreateStageImage(name string, prevStageImage *stage.StageImage, stg stage.Interface, img *Image) *stage.StageImage
 
 	IsBaseImagesRepoIdsCacheExist(key string) bool
 	GetBaseImagesRepoIdsCache(key string) string

pkg/build/image/dockerfile.go

@@ -14,6 +14,7 @@ import (
 	"github.com/werf/werf/pkg/build/stage"
 	"github.com/werf/werf/pkg/build/stage/dockerfile_instruction"
 	"github.com/werf/werf/pkg/config"
+	"github.com/werf/werf/pkg/container_backend"
 	"github.com/werf/werf/pkg/dockerfile"
 	"github.com/werf/werf/pkg/path_matcher"
 	"github.com/werf/werf/pkg/util"
@@ -70,12 +71,18 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile,
 	}
 
 	{
-		img := NewImage("test", ImageOptions{
+		// TODO parse FROM instruction properly, set correct BaseImageReference here
+
+		img, err := NewImage(ctx, "test", ImageFromRegistryAsBaseImage, ImageOptions{
 			IsDockerfileImage:  true,
 			CommonImageOptions: opts,
+			BaseImageReference: "ubuntu:22.04",
 		})
+		if err != nil {
+			return nil, fmt.Errorf("unable to create image %q: %w", "test", err)
+		}
 
-		img.stages = append(img.stages, dockerfile_instruction.NewRun(&dockerfile.InstructionRun{Command: []string{"ls", "/"}}, nil, false, &stage.BaseStageOptions{
+		img.stages = append(img.stages, dockerfile_instruction.NewRun(&container_backend.InstructionRun{Command: []string{"ls", "/"}}, nil, false, &stage.BaseStageOptions{
 			ImageName:        img.Name,
 			ImageTmpDir:      img.TmpDir,
 			ContainerWerfDir: img.ContainerWerfDir,
@@ -89,10 +96,13 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile,
 }
 
 func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *config.ImageFromDockerfile, opts CommonImageOptions) (*Image, error) {
-	img := NewImage(dockerfileImageConfig.Name, ImageOptions{
+	img, err := NewImage(ctx, dockerfileImageConfig.Name, NoBaseImage, ImageOptions{
 		CommonImageOptions: opts,
 		IsDockerfileImage:  true,
 	})
+	if err != nil {
+		return nil, fmt.Errorf("unable to create image %q: %w", dockerfileImageConfig.Name, err)
+	}
 
 	for _, contextAddFile := range dockerfileImageConfig.ContextAddFiles {
 		relContextAddFile := filepath.Join(dockerfileImageConfig.Context, contextAddFile)