feat(local-stages-storage): introduce local storage independent of co…

…ntainer backend implementation

* DockerServerStagesStorage removed, LocalStagesStorage introduced.
* Implemented missing methods of both DockerServerBackend and BuildahBackend needed for independent LocalStagesStorage.
* Multiarch manifests posting not implemented yet for docker server and buildah backends.
* Arbitrary manifests posting not implemented yet for buildah backend.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>

cmd/werf/common/common.go

@@ -857,8 +857,8 @@ func GetParallelTasksLimit(cmdData *CmdData) (int64, error) {
 	}
 }
 
-func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) *storage.DockerServerStagesStorage {
-	return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend))
+func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) *storage.LocalStagesStorage {
+	return storage.NewLocalStagesStorage(containerBackend)
 }
 
 func GetStagesStorage(ctx context.Context, containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.PrimaryStagesStorage, error) {
@@ -902,9 +902,9 @@ func GetCacheStagesStorageList(ctx context.Context, containerBackend container_b
 func GetSecondaryStagesStorageList(ctx context.Context, stagesStorage storage.StagesStorage, containerBackend container_backend.ContainerBackend, cmdData *CmdData) ([]storage.StagesStorage, error) {
 	var res []storage.StagesStorage
 
-	if dockerBackend, matched := containerBackend.(*container_backend.DockerServerBackend); matched {
+	if _, matched := containerBackend.(*container_backend.DockerServerBackend); matched {
 		if stagesStorage.Address() != storage.LocalStorageAddress {
-			res = append(res, storage.NewDockerServerStagesStorage(dockerBackend))
+			res = append(res, storage.NewLocalStagesStorage(containerBackend))
 		}
 	}
 

cmd/werf/common/repo_data.go

@@ -55,7 +55,7 @@ func (repoData *RepoData) CreateStagesStorage(ctx context.Context, containerBack
 	}
 
 	if addr == storage.LocalStorageAddress {
-		return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)), nil
+		return storage.NewLocalStagesStorage(containerBackend), nil
 	} else {
 		dockerRegistry, err := repoData.CreateDockerRegistry(ctx, insecureRegistry, skipTlsVerifyRegistry)
 		if err != nil {

pkg/build/build_phase.go

@@ -1008,7 +1008,7 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *image.I
 	if v := os.Getenv("WERF_TEST_ATOMIC_STAGE_BUILD__SLEEP_SECONDS_BEFORE_STAGE_SAVE"); v != "" {
 		seconds := 0
 		fmt.Sscanf(v, "%d", &seconds)
-		fmt.Printf("Sleeping %d seconds before saving newly built image %s into repo %s by digest %s...\n", seconds, stg.GetStageImage().Image.GetBuiltID(), phase.Conveyor.StorageManager.GetStagesStorage().String(), stg.GetDigest())
+		fmt.Printf("Sleeping %d seconds before saving newly built image %s into repo %s by digest %s...\n", seconds, stg.GetStageImage().Image.BuiltID(), phase.Conveyor.StorageManager.GetStagesStorage().String(), stg.GetDigest())
 		time.Sleep(time.Duration(seconds) * time.Second)
 	}
 

pkg/build/image/image_tree.go

@@ -124,7 +124,7 @@ func (tree *ImagesTree) GetImage(name string) *Image {
 	return nil
 }
 
-func (tree *ImagesTree) GetImagesByName(finalOnly bool) []*util.Pair[string, []*Image] {
+func (tree *ImagesTree) GetImagesByName(finalOnly bool) []util.Pair[string, []*Image] {
 	images := make(map[string]map[string]*Image)
 	var names []string
 
@@ -148,7 +148,7 @@ func (tree *ImagesTree) GetImagesByName(finalOnly bool) []*util.Pair[string, []*
 		}
 	}
 
-	var res []*util.Pair[string, []*Image]
+	var res []util.Pair[string, []*Image]
 
 	sort.Strings(names)
 	for _, name := range names {

pkg/buildah/common.go

@@ -14,6 +14,7 @@ import (
 	"github.com/opencontainers/runtime-spec/specs-go"
 
 	"github.com/werf/werf/pkg/buildah/thirdparty"
+	"github.com/werf/werf/pkg/image"
 	"github.com/werf/werf/pkg/util"
 	"github.com/werf/werf/pkg/werf"
 )
@@ -122,6 +123,16 @@ type AddOpts struct {
 	Ignores    []string
 }
 
+type ImagesOptions struct {
+	CommitOpts
+	Filters []util.Pair[string, string]
+}
+
+type ContainersOptions struct {
+	CommitOpts
+	Filters []image.ContainerFilter
+}
+
 type (
 	FromCommandOpts       CommonOpts
 	BuildFromCommandsOpts CommonOpts
@@ -151,6 +162,8 @@ type Buildah interface {
 	Config(ctx context.Context, container string, opts ConfigOpts) error
 	Copy(ctx context.Context, container, contextDir string, src []string, dst string, opts CopyOpts) error
 	Add(ctx context.Context, container string, src []string, dst string, opts AddOpts) error
+	Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error)
+	Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error)
 }
 
 type Mode string

pkg/buildah/native_linux.go

@@ -40,6 +40,7 @@ import (
 	"gopkg.in/errgo.v2/fmt/errors"
 
 	"github.com/werf/werf/pkg/buildah/thirdparty"
+	"github.com/werf/werf/pkg/image"
 )
 
 const (
@@ -792,6 +793,88 @@ func (b *NativeBuildah) NewSessionTmpDir() (string, error) {
 	return sessionTmpDir, nil
 }
 
+func (b *NativeBuildah) Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error) {
+	sysCtx, err := b.getSystemContext(opts.TargetPlatform)
+	if err != nil {
+		return nil, err
+	}
+
+	runtime, err := b.getRuntime(sysCtx)
+	if err != nil {
+		return nil, err
+	}
+
+	listOpts := &libimage.ListImagesOptions{}
+	for _, filter := range opts.Filters {
+		listOpts.Filters = append(listOpts.Filters, fmt.Sprintf("%s=%s", filter.First, filter.Second))
+	}
+	images, err := runtime.ListImages(ctx, nil, listOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	var res image.ImagesList
+	for _, img := range images {
+		repoTags, err := img.RepoTags()
+		if err != nil {
+			return nil, fmt.Errorf("unable to get image %s repo tags: %w", img.ID(), err)
+		}
+		res = append(res, image.Summary{RepoTags: repoTags})
+	}
+
+	return res, nil
+}
+
+func (b *NativeBuildah) Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error) {
+	builders, err := buildah.OpenAllBuilders(b.Store)
+	if err != nil {
+		return nil, err
+	}
+
+	seenImages := make(map[string]string)
+	imageNameForID := func(id string) string {
+		if id == "" {
+			return buildah.BaseImageFakeName
+		}
+		imageName, ok := seenImages[id]
+		if ok {
+			return imageName
+		}
+		img, err2 := b.Store.Image(id)
+		if err2 == nil && len(img.Names) > 0 {
+			seenImages[id] = img.Names[0]
+		}
+		return seenImages[id]
+	}
+
+	var res image.ContainerList
+SelectContainers:
+	for _, builder := range builders {
+		imgID := builder.FromImageID
+		imgName := imageNameForID(builder.FromImageID)
+
+		for _, filter := range opts.Filters {
+			if filter.ID != "" && !thirdparty.MatchesID(builder.ContainerID, filter.ID) {
+				continue SelectContainers
+			}
+			if filter.Name != "" && !thirdparty.MatchesCtrName(builder.Container, filter.Name) {
+				continue SelectContainers
+			}
+			if filter.Ancestor != "" && !thirdparty.MatchesAncestor(imgName, imgID, filter.Ancestor) {
+				continue SelectContainers
+			}
+		}
+
+		res = append(res, image.Container{
+			ID:      builder.ContainerID,
+			ImageID: builder.FromImageID,
+			Names:   []string{builder.Container},
+		})
+	}
+
+	return res, nil
+}
+
 func NewNativeStoreOptions(rootlessUID int, driver StorageDriver) (*thirdparty.StoreOptions, error) {
 	var (
 		runRoot string

pkg/buildah/thirdparty/utils.go

@@ -0,0 +1,31 @@
+package thirdparty
+
+import "strings"
+
+func MatchesAncestor(imgName, imgID, argName string) bool {
+	if MatchesID(imgID, argName) {
+		return true
+	}
+	return MatchesReference(imgName, argName)
+}
+
+func MatchesID(imageID, argID string) bool {
+	return strings.HasPrefix(imageID, argID)
+}
+
+func MatchesReference(name, argName string) bool {
+	if argName == "" {
+		return true
+	}
+	splitName := strings.Split(name, ":")
+	// If the arg contains a tag, we handle it differently than if it does not
+	if strings.Contains(argName, ":") {
+		splitArg := strings.Split(argName, ":")
+		return strings.HasSuffix(splitName[0], splitArg[0]) && (splitName[1] == splitArg[1])
+	}
+	return strings.HasSuffix(splitName[0], argName)
+}
+
+func MatchesCtrName(ctrName, argName string) bool {
+	return strings.Contains(ctrName, argName)
+}

pkg/container_backend/buildah_backend.go

@@ -648,6 +648,19 @@ func (backend *BuildahBackend) Push(ctx context.Context, ref string, opts PushOp
 	)
 }
 
+func (backend *BuildahBackend) TagImageByName(ctx context.Context, img LegacyImageInterface) error {
+	if img.BuiltID() != "" {
+		if err := backend.Tag(ctx, img.BuiltID(), img.Name(), TagOpts{}); err != nil {
+			return fmt.Errorf("unable to tag %q as %s: %w", img.BuiltID(), img.Name(), err)
+		}
+	} else {
+		if err := backend.RefreshImageObject(ctx, img); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (backend *BuildahBackend) BuildDockerfile(ctx context.Context, dockerfileContent []byte, opts BuildDockerfileOpts) (string, error) {
 	buildArgs := make(map[string]string)
 	for _, argStr := range opts.BuildArgs {
@@ -724,7 +737,9 @@ func (backend *BuildahBackend) RenameImage(ctx context.Context, img LegacyImageI
 
 	if removeOldName {
 		if err := logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Removing old image tag %s", img.Name())).DoError(func() error {
-			if err := backend.Rmi(ctx, img.Name(), RmiOpts{TargetPlatform: img.GetTargetPlatform()}); err != nil {
+			if err := backend.Rmi(ctx, img.Name(), RmiOpts{
+				CommonOpts: CommonOpts{TargetPlatform: img.GetTargetPlatform()},
+			}); err != nil {
 				return fmt.Errorf("unable to remove image %q: %w", img.Name(), err)
 			}
 			return nil
@@ -753,7 +768,9 @@ func (backend *BuildahBackend) RenameImage(ctx context.Context, img LegacyImageI
 
 func (backend *BuildahBackend) RemoveImage(ctx context.Context, img LegacyImageInterface) error {
 	if err := logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Removing image tag %s", img.Name())).DoError(func() error {
-		if err := backend.Rmi(ctx, img.Name(), RmiOpts{TargetPlatform: img.GetTargetPlatform()}); err != nil {
+		if err := backend.Rmi(ctx, img.Name(), RmiOpts{
+			CommonOpts: CommonOpts{TargetPlatform: img.GetTargetPlatform()},
+		}); err != nil {
 			return fmt.Errorf("unable to remove image %q: %w", img.Name(), err)
 		}
 		return nil
@@ -970,3 +987,21 @@ func newHealthConfigFromString(healthcheck string) (*thirdparty.BuildahHealthCon
 
 	return healthconfig, nil
 }
+
+func (runtime *BuildahBackend) Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error) {
+	imagesOpts := buildah.ImagesOptions{Filters: opts.Filters}
+	return runtime.buildah.Images(ctx, imagesOpts)
+}
+
+func (runtime *BuildahBackend) Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error) {
+	containersOpts := buildah.ContainersOptions{Filters: opts.Filters}
+	return runtime.buildah.Containers(ctx, containersOpts)
+}
+
+func (runtime *BuildahBackend) Rm(ctx context.Context, name string, opts RmOpts) error {
+	return runtime.buildah.Rm(ctx, name, buildah.RmOpts{})
+}
+
+func (runtime *BuildahBackend) PostManifest(ctx context.Context, ref string, opts PostManifestOpts) error {
+	return fmt.Errorf("not implemented")
+}