From e6aa7f14453be1eb2c5f0032a22255d5edf408bb Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Fri, 7 Apr 2023 15:53:46 +0300 Subject: [PATCH] feat(local-stages-storage): introduce local storage independent of container backend implementation * DockerServerStagesStorage removed, LocalStagesStorage introduced. * Implemented missing methods of both DockerServerBackend and BuildahBackend needed for independent LocalStagesStorage. * Multiarch manifests posting not implemented yet for docker server and buildah backends. * Arbitrary manifests posting not implemented yet for buildah backend. Signed-off-by: Timofey Kirillov --- cmd/werf/common/common.go | 8 +- cmd/werf/common/repo_data.go | 2 +- pkg/build/build_phase.go | 2 +- pkg/build/image/image_tree.go | 4 +- pkg/buildah/common.go | 13 + pkg/buildah/native_linux.go | 83 +++ pkg/buildah/thirdparty/utils.go | 31 + pkg/container_backend/buildah_backend.go | 39 +- .../docker_server_backend.go | 100 ++- pkg/container_backend/interface.go | 36 +- pkg/container_backend/legacy_interface.go | 2 - pkg/container_backend/legacy_stage_image.go | 13 +- .../perf_check_container_backend.go | 40 ++ pkg/docker/image.go | 8 +- pkg/image/container.go | 23 + pkg/image/info.go | 8 + pkg/image/summary.go | 49 ++ pkg/storage/docker_server_stages_storage.go | 570 ------------------ pkg/storage/import_metadata.go | 12 +- pkg/storage/local_stages_storage.go | 438 ++++++++++++++ pkg/storage/manager/storage_manager.go | 2 +- pkg/storage/repo_stages_storage.go | 14 +- pkg/storage/stages_storage.go | 7 + pkg/util/pair.go | 8 +- test/pkg/utils/storage.go | 2 +- 25 files changed, 872 insertions(+), 642 deletions(-) create mode 100644 pkg/buildah/thirdparty/utils.go create mode 100644 pkg/image/container.go create mode 100644 pkg/image/summary.go delete mode 100644 pkg/storage/docker_server_stages_storage.go create mode 100644 pkg/storage/local_stages_storage.go diff --git a/cmd/werf/common/common.go b/cmd/werf/common/common.go index 4633a3a3b5..cb0b2b1983 100644 --- a/cmd/werf/common/common.go +++ b/cmd/werf/common/common.go @@ -857,8 +857,8 @@ func GetParallelTasksLimit(cmdData *CmdData) (int64, error) { } } -func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) *storage.DockerServerStagesStorage { - return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)) +func GetLocalStagesStorage(containerBackend container_backend.ContainerBackend) *storage.LocalStagesStorage { + return storage.NewLocalStagesStorage(containerBackend) } func GetStagesStorage(ctx context.Context, containerBackend container_backend.ContainerBackend, cmdData *CmdData) (storage.PrimaryStagesStorage, error) { @@ -902,9 +902,9 @@ func GetCacheStagesStorageList(ctx context.Context, containerBackend container_b func GetSecondaryStagesStorageList(ctx context.Context, stagesStorage storage.StagesStorage, containerBackend container_backend.ContainerBackend, cmdData *CmdData) ([]storage.StagesStorage, error) { var res []storage.StagesStorage - if dockerBackend, matched := containerBackend.(*container_backend.DockerServerBackend); matched { + if _, matched := containerBackend.(*container_backend.DockerServerBackend); matched { if stagesStorage.Address() != storage.LocalStorageAddress { - res = append(res, storage.NewDockerServerStagesStorage(dockerBackend)) + res = append(res, storage.NewLocalStagesStorage(containerBackend)) } } diff --git a/cmd/werf/common/repo_data.go b/cmd/werf/common/repo_data.go index 9cc15f30d3..94d01b6cba 100644 --- a/cmd/werf/common/repo_data.go +++ b/cmd/werf/common/repo_data.go @@ -55,7 +55,7 @@ func (repoData *RepoData) CreateStagesStorage(ctx context.Context, containerBack } if addr == storage.LocalStorageAddress { - return storage.NewDockerServerStagesStorage(containerBackend.(*container_backend.DockerServerBackend)), nil + return storage.NewLocalStagesStorage(containerBackend), nil } else { dockerRegistry, err := repoData.CreateDockerRegistry(ctx, insecureRegistry, skipTlsVerifyRegistry) if err != nil { diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 03a4e45af9..aab1a5b97e 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -1008,7 +1008,7 @@ func (phase *BuildPhase) atomicBuildStageImage(ctx context.Context, img *image.I if v := os.Getenv("WERF_TEST_ATOMIC_STAGE_BUILD__SLEEP_SECONDS_BEFORE_STAGE_SAVE"); v != "" { seconds := 0 fmt.Sscanf(v, "%d", &seconds) - fmt.Printf("Sleeping %d seconds before saving newly built image %s into repo %s by digest %s...\n", seconds, stg.GetStageImage().Image.GetBuiltID(), phase.Conveyor.StorageManager.GetStagesStorage().String(), stg.GetDigest()) + fmt.Printf("Sleeping %d seconds before saving newly built image %s into repo %s by digest %s...\n", seconds, stg.GetStageImage().Image.BuiltID(), phase.Conveyor.StorageManager.GetStagesStorage().String(), stg.GetDigest()) time.Sleep(time.Duration(seconds) * time.Second) } diff --git a/pkg/build/image/image_tree.go b/pkg/build/image/image_tree.go index 603f703a75..a8678168e4 100644 --- a/pkg/build/image/image_tree.go +++ b/pkg/build/image/image_tree.go @@ -124,7 +124,7 @@ func (tree *ImagesTree) GetImage(name string) *Image { return nil } -func (tree *ImagesTree) GetImagesByName(finalOnly bool) []*util.Pair[string, []*Image] { +func (tree *ImagesTree) GetImagesByName(finalOnly bool) []util.Pair[string, []*Image] { images := make(map[string]map[string]*Image) var names []string @@ -148,7 +148,7 @@ func (tree *ImagesTree) GetImagesByName(finalOnly bool) []*util.Pair[string, []* } } - var res []*util.Pair[string, []*Image] + var res []util.Pair[string, []*Image] sort.Strings(names) for _, name := range names { diff --git a/pkg/buildah/common.go b/pkg/buildah/common.go index d761db6ba2..98a62d5a75 100644 --- a/pkg/buildah/common.go +++ b/pkg/buildah/common.go @@ -14,6 +14,7 @@ import ( "github.com/opencontainers/runtime-spec/specs-go" "github.com/werf/werf/pkg/buildah/thirdparty" + "github.com/werf/werf/pkg/image" "github.com/werf/werf/pkg/util" "github.com/werf/werf/pkg/werf" ) @@ -122,6 +123,16 @@ type AddOpts struct { Ignores []string } +type ImagesOptions struct { + CommitOpts + Filters []util.Pair[string, string] +} + +type ContainersOptions struct { + CommitOpts + Filters []image.ContainerFilter +} + type ( FromCommandOpts CommonOpts BuildFromCommandsOpts CommonOpts @@ -151,6 +162,8 @@ type Buildah interface { Config(ctx context.Context, container string, opts ConfigOpts) error Copy(ctx context.Context, container, contextDir string, src []string, dst string, opts CopyOpts) error Add(ctx context.Context, container string, src []string, dst string, opts AddOpts) error + Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error) + Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error) } type Mode string diff --git a/pkg/buildah/native_linux.go b/pkg/buildah/native_linux.go index ed2864ca41..0c9ee2dcbd 100644 --- a/pkg/buildah/native_linux.go +++ b/pkg/buildah/native_linux.go @@ -40,6 +40,7 @@ import ( "gopkg.in/errgo.v2/fmt/errors" "github.com/werf/werf/pkg/buildah/thirdparty" + "github.com/werf/werf/pkg/image" ) const ( @@ -792,6 +793,88 @@ func (b *NativeBuildah) NewSessionTmpDir() (string, error) { return sessionTmpDir, nil } +func (b *NativeBuildah) Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error) { + sysCtx, err := b.getSystemContext(opts.TargetPlatform) + if err != nil { + return nil, err + } + + runtime, err := b.getRuntime(sysCtx) + if err != nil { + return nil, err + } + + listOpts := &libimage.ListImagesOptions{} + for _, filter := range opts.Filters { + listOpts.Filters = append(listOpts.Filters, fmt.Sprintf("%s=%s", filter.First, filter.Second)) + } + images, err := runtime.ListImages(ctx, nil, listOpts) + if err != nil { + return nil, err + } + + var res image.ImagesList + for _, img := range images { + repoTags, err := img.RepoTags() + if err != nil { + return nil, fmt.Errorf("unable to get image %s repo tags: %w", img.ID(), err) + } + res = append(res, image.Summary{RepoTags: repoTags}) + } + + return res, nil +} + +func (b *NativeBuildah) Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error) { + builders, err := buildah.OpenAllBuilders(b.Store) + if err != nil { + return nil, err + } + + seenImages := make(map[string]string) + imageNameForID := func(id string) string { + if id == "" { + return buildah.BaseImageFakeName + } + imageName, ok := seenImages[id] + if ok { + return imageName + } + img, err2 := b.Store.Image(id) + if err2 == nil && len(img.Names) > 0 { + seenImages[id] = img.Names[0] + } + return seenImages[id] + } + + var res image.ContainerList +SelectContainers: + for _, builder := range builders { + imgID := builder.FromImageID + imgName := imageNameForID(builder.FromImageID) + + for _, filter := range opts.Filters { + if filter.ID != "" && !thirdparty.MatchesID(builder.ContainerID, filter.ID) { + continue SelectContainers + } + if filter.Name != "" && !thirdparty.MatchesCtrName(builder.Container, filter.Name) { + continue SelectContainers + } + if filter.Ancestor != "" && !thirdparty.MatchesAncestor(imgName, imgID, filter.Ancestor) { + continue SelectContainers + } + } + + res = append(res, image.Container{ + ID: builder.ContainerID, + ImageID: builder.FromImageID, + Names: []string{builder.Container}, + }) + } + + return res, nil +} + func NewNativeStoreOptions(rootlessUID int, driver StorageDriver) (*thirdparty.StoreOptions, error) { var ( runRoot string diff --git a/pkg/buildah/thirdparty/utils.go b/pkg/buildah/thirdparty/utils.go new file mode 100644 index 0000000000..fe97a3dc4f --- /dev/null +++ b/pkg/buildah/thirdparty/utils.go @@ -0,0 +1,31 @@ +package thirdparty + +import "strings" + +func MatchesAncestor(imgName, imgID, argName string) bool { + if MatchesID(imgID, argName) { + return true + } + return MatchesReference(imgName, argName) +} + +func MatchesID(imageID, argID string) bool { + return strings.HasPrefix(imageID, argID) +} + +func MatchesReference(name, argName string) bool { + if argName == "" { + return true + } + splitName := strings.Split(name, ":") + // If the arg contains a tag, we handle it differently than if it does not + if strings.Contains(argName, ":") { + splitArg := strings.Split(argName, ":") + return strings.HasSuffix(splitName[0], splitArg[0]) && (splitName[1] == splitArg[1]) + } + return strings.HasSuffix(splitName[0], argName) +} + +func MatchesCtrName(ctrName, argName string) bool { + return strings.Contains(ctrName, argName) +} diff --git a/pkg/container_backend/buildah_backend.go b/pkg/container_backend/buildah_backend.go index 0db313bbe3..888ef5ac86 100644 --- a/pkg/container_backend/buildah_backend.go +++ b/pkg/container_backend/buildah_backend.go @@ -648,6 +648,19 @@ func (backend *BuildahBackend) Push(ctx context.Context, ref string, opts PushOp ) } +func (backend *BuildahBackend) TagImageByName(ctx context.Context, img LegacyImageInterface) error { + if img.BuiltID() != "" { + if err := backend.Tag(ctx, img.BuiltID(), img.Name(), TagOpts{}); err != nil { + return fmt.Errorf("unable to tag %q as %s: %w", img.BuiltID(), img.Name(), err) + } + } else { + if err := backend.RefreshImageObject(ctx, img); err != nil { + return err + } + } + return nil +} + func (backend *BuildahBackend) BuildDockerfile(ctx context.Context, dockerfileContent []byte, opts BuildDockerfileOpts) (string, error) { buildArgs := make(map[string]string) for _, argStr := range opts.BuildArgs { @@ -724,7 +737,9 @@ func (backend *BuildahBackend) RenameImage(ctx context.Context, img LegacyImageI if removeOldName { if err := logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Removing old image tag %s", img.Name())).DoError(func() error { - if err := backend.Rmi(ctx, img.Name(), RmiOpts{TargetPlatform: img.GetTargetPlatform()}); err != nil { + if err := backend.Rmi(ctx, img.Name(), RmiOpts{ + CommonOpts: CommonOpts{TargetPlatform: img.GetTargetPlatform()}, + }); err != nil { return fmt.Errorf("unable to remove image %q: %w", img.Name(), err) } return nil @@ -753,7 +768,9 @@ func (backend *BuildahBackend) RenameImage(ctx context.Context, img LegacyImageI func (backend *BuildahBackend) RemoveImage(ctx context.Context, img LegacyImageInterface) error { if err := logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Removing image tag %s", img.Name())).DoError(func() error { - if err := backend.Rmi(ctx, img.Name(), RmiOpts{TargetPlatform: img.GetTargetPlatform()}); err != nil { + if err := backend.Rmi(ctx, img.Name(), RmiOpts{ + CommonOpts: CommonOpts{TargetPlatform: img.GetTargetPlatform()}, + }); err != nil { return fmt.Errorf("unable to remove image %q: %w", img.Name(), err) } return nil @@ -970,3 +987,21 @@ func newHealthConfigFromString(healthcheck string) (*thirdparty.BuildahHealthCon return healthconfig, nil } + +func (runtime *BuildahBackend) Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error) { + imagesOpts := buildah.ImagesOptions{Filters: opts.Filters} + return runtime.buildah.Images(ctx, imagesOpts) +} + +func (runtime *BuildahBackend) Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error) { + containersOpts := buildah.ContainersOptions{Filters: opts.Filters} + return runtime.buildah.Containers(ctx, containersOpts) +} + +func (runtime *BuildahBackend) Rm(ctx context.Context, name string, opts RmOpts) error { + return runtime.buildah.Rm(ctx, name, buildah.RmOpts{}) +} + +func (runtime *BuildahBackend) PostManifest(ctx context.Context, ref string, opts PostManifestOpts) error { + return fmt.Errorf("not implemented") +} diff --git a/pkg/container_backend/docker_server_backend.go b/pkg/container_backend/docker_server_backend.go index fdbb0bbe13..99d4c491ab 100644 --- a/pkg/container_backend/docker_server_backend.go +++ b/pkg/container_backend/docker_server_backend.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/docker/docker/api/types" + "github.com/docker/docker/api/types/filters" "github.com/docker/docker/client" "github.com/google/uuid" @@ -183,7 +184,9 @@ func (runtime *DockerServerBackend) RenameImage(ctx context.Context, img LegacyI func (runtime *DockerServerBackend) RemoveImage(ctx context.Context, img LegacyImageInterface) error { return logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Removing image tag %s", img.Name())).DoError(func() error { - return runtime.Rmi(ctx, img.Name(), RmiOpts{TargetPlatform: img.GetTargetPlatform()}) + return runtime.Rmi(ctx, img.Name(), RmiOpts{ + CommonOpts: CommonOpts{TargetPlatform: img.GetTargetPlatform()}, + }) }) } @@ -223,43 +226,29 @@ func (runtime *DockerServerBackend) Pull(ctx context.Context, ref string, opts P } func (runtime *DockerServerBackend) Rmi(ctx context.Context, ref string, opts RmiOpts) error { - return docker.CliRmi(ctx, ref, "--force") -} - -func (runtime *DockerServerBackend) PushImage(ctx context.Context, img LegacyImageInterface) error { - if err := logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Pushing %s", img.Name())).DoError(func() error { - return docker.CliPushWithRetries(ctx, img.Name()) - }); err != nil { - return err + args := []string{ref} + if opts.Force { + args = append(args, "--force") } - - return nil + return docker.CliRmi(ctx, args...) } -// PushBuiltImage is only available for DockerServerBackend -func (runtime *DockerServerBackend) PushBuiltImage(ctx context.Context, img LegacyImageInterface) error { - if err := logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Tagging built image by name %s", img.Name())).DoError(func() error { - if err := img.TagBuiltImage(ctx); err != nil { - return fmt.Errorf("unable to tag built image by name %s: %w", img.Name(), err) - } - return nil - }); err != nil { - return err - } +func (runtime *DockerServerBackend) Rm(ctx context.Context, ref string, opts RmOpts) error { + return docker.ContainerRemove(ctx, ref, types.ContainerRemoveOptions{Force: opts.Force}) +} +func (runtime *DockerServerBackend) PushImage(ctx context.Context, img LegacyImageInterface) error { if err := logboek.Context(ctx).Info().LogProcess(fmt.Sprintf("Pushing %s", img.Name())).DoError(func() error { - return img.Push(ctx) + return docker.CliPushWithRetries(ctx, img.Name()) }); err != nil { return err } - return nil } -// TagBuiltImageByName is only available for DockerServerBackend func (runtime *DockerServerBackend) TagImageByName(ctx context.Context, img LegacyImageInterface) error { - if img.GetBuiltID() != "" { - if err := img.TagBuiltImage(ctx); err != nil { + if img.BuiltID() != "" { + if err := docker.CliTag(ctx, img.BuiltID(), img.Name()); err != nil { return fmt.Errorf("unable to tag image %s: %w", img.Name(), err) } } else { @@ -267,7 +256,6 @@ func (runtime *DockerServerBackend) TagImageByName(ctx context.Context, img Lega return err } } - return nil } @@ -292,3 +280,61 @@ func (runtime *DockerServerBackend) RemoveHostDirs(ctx context.Context, mountDir return docker.CliRun(ctx, args...) } + +func (runtime *DockerServerBackend) Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error) { + filterSet := filters.NewArgs() + for _, item := range opts.Filters { + filterSet.Add(item.First, item.Second) + } + images, err := docker.Images(ctx, types.ImageListOptions{Filters: filterSet}) + if err != nil { + return nil, fmt.Errorf("unable to get docker images: %w", err) + } + + var res image.ImagesList + for _, img := range images { + res = append(res, image.Summary{ + RepoTags: img.RepoTags, + }) + } + return res, nil +} + +func (runtime *DockerServerBackend) Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error) { + filterSet := filters.NewArgs() + for _, filter := range opts.Filters { + if filter.ID != "" { + filterSet.Add("id", filter.ID) + } + if filter.Name != "" { + filterSet.Add("name", filter.Name) + } + if filter.Ancestor != "" { + filterSet.Add("ancestor", filter.Ancestor) + } + } + + containersOptions := types.ContainerListOptions{} + containersOptions.All = true + containersOptions.Filters = filterSet + + containers, err := docker.Containers(ctx, containersOptions) + if err != nil { + return nil, err + } + + var res image.ContainerList + for _, container := range containers { + res = append(res, image.Container{ + ID: container.ID, + ImageID: container.ImageID, + Names: container.Names, + }) + } + + return res, nil +} + +func (runtime *DockerServerBackend) PostManifest(ctx context.Context, ref string, opts PostManifestOpts) error { + return docker.CreateImage(ctx, ref, opts.Labels) +} diff --git a/pkg/container_backend/interface.go b/pkg/container_backend/interface.go index c27b9527ed..efa62b1239 100644 --- a/pkg/container_backend/interface.go +++ b/pkg/container_backend/interface.go @@ -4,6 +4,7 @@ import ( "context" "github.com/werf/werf/pkg/image" + "github.com/werf/werf/pkg/util" ) type CommonOpts struct { @@ -14,15 +15,23 @@ type ( TagOpts CommonOpts PushOpts CommonOpts PullOpts CommonOpts - RmiOpts CommonOpts GetImageInfoOpts CommonOpts CalculateDependencyImportChecksum CommonOpts ) +type RmOpts struct { + CommonOpts + Force bool +} + +type RmiOpts struct { + CommonOpts + Force bool +} + type BuildDockerfileOpts struct { CommonOpts - TargetPlatform string BuildContextArchive BuildContextArchiver DockerfileCtxRelPath string // TODO: remove this and instead write the []byte dockerfile to /Dockerfile in the ContextTar inDockerServerBackend.BuildDockerfile(). Target string @@ -36,7 +45,6 @@ type BuildDockerfileOpts struct { type BuildDockerfileStageOptions struct { CommonOpts - BuildContextArchive BuildContextArchiver } @@ -46,11 +54,29 @@ type BuildOptions struct { IntrospectAfterError bool } +type ImagesOptions struct { + CommonOpts + Filters []util.Pair[string, string] +} + +type ContainersOptions struct { + CommonOpts + Filters []image.ContainerFilter +} + +type PostManifestOpts struct { + CommonOpts + Labels []string + Manifests []*image.Info +} + type ContainerBackend interface { Tag(ctx context.Context, ref, newRef string, opts TagOpts) error Push(ctx context.Context, ref string, opts PushOpts) error Pull(ctx context.Context, ref string, opts PullOpts) error Rmi(ctx context.Context, ref string, opts RmiOpts) error + Rm(ctx context.Context, name string, opts RmOpts) error + PostManifest(ctx context.Context, ref string, opts PostManifestOpts) error GetImageInfo(ctx context.Context, ref string, opts GetImageInfoOpts) (*image.Info, error) BuildDockerfile(ctx context.Context, dockerfile []byte, opts BuildDockerfileOpts) (string, error) @@ -62,6 +88,9 @@ type ContainerBackend interface { GetDefaultPlatform() string GetRuntimePlatform() string + Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error) + Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error) + String() string // TODO: Util method for cleanup, which possibly should be avoided in the future @@ -73,4 +102,5 @@ type ContainerBackend interface { PullImageFromRegistry(ctx context.Context, img LegacyImageInterface) error RenameImage(ctx context.Context, img LegacyImageInterface, newImageName string, removeOldName bool) error RemoveImage(ctx context.Context, img LegacyImageInterface) error + TagImageByName(ctx context.Context, img LegacyImageInterface) error } diff --git a/pkg/container_backend/legacy_interface.go b/pkg/container_backend/legacy_interface.go index 41f00410a7..f76ba2f5a0 100644 --- a/pkg/container_backend/legacy_interface.go +++ b/pkg/container_backend/legacy_interface.go @@ -23,9 +23,7 @@ type LegacyImageInterface interface { Build(context.Context, BuildOptions) error SetBuiltID(builtID string) - GetBuiltID() string BuiltID() string - TagBuiltImage(ctx context.Context) error Introspect(ctx context.Context) error diff --git a/pkg/container_backend/legacy_stage_image.go b/pkg/container_backend/legacy_stage_image.go index 376d710797..125eff93f1 100644 --- a/pkg/container_backend/legacy_stage_image.go +++ b/pkg/container_backend/legacy_stage_image.go @@ -201,7 +201,7 @@ func (i *LegacyStageImage) GetInfo() *image.Info { } func (i *LegacyStageImage) MustGetBuiltID() string { - builtId := i.GetBuiltID() + builtId := i.BuiltID() if builtId == "" { panic(fmt.Sprintf("image %s built id is not available", i.Name())) } @@ -216,19 +216,8 @@ func (i *LegacyStageImage) BuiltID() string { return i.builtID } -func (i *LegacyStageImage) GetBuiltID() string { - return i.BuiltID() -} - -func (i *LegacyStageImage) TagBuiltImage(ctx context.Context) error { - _ = i.ContainerBackend.(*DockerServerBackend) - - return docker.CliTag(ctx, i.MustGetBuiltID(), i.name) -} - func (i *LegacyStageImage) Tag(ctx context.Context, name string) error { _ = i.ContainerBackend.(*DockerServerBackend) - return docker.CliTag(ctx, i.GetID(), name) } diff --git a/pkg/container_backend/perf_check_container_backend.go b/pkg/container_backend/perf_check_container_backend.go index 6ca5c94589..32772c3129 100644 --- a/pkg/container_backend/perf_check_container_backend.go +++ b/pkg/container_backend/perf_check_container_backend.go @@ -146,3 +146,43 @@ func (runtime *PerfCheckContainerBackend) RemoveHostDirs(ctx context.Context, mo }) return } + +func (runtime *PerfCheckContainerBackend) Images(ctx context.Context, opts ImagesOptions) (res image.ImagesList, resErr error) { + logboek.Context(ctx).Default().LogProcess("ContainerBackend.Images %v", opts). + Do(func() { + res, resErr = runtime.ContainerBackend.Images(ctx, opts) + }) + return +} + +func (runtime *PerfCheckContainerBackend) Containers(ctx context.Context, opts ContainersOptions) (res image.ContainerList, resErr error) { + logboek.Context(ctx).Default().LogProcess("ContainerBackend.Containers %v", opts). + Do(func() { + res, resErr = runtime.ContainerBackend.Containers(ctx, opts) + }) + return +} + +func (runtime *PerfCheckContainerBackend) Rm(ctx context.Context, name string, opts RmOpts) (resErr error) { + logboek.Context(ctx).Default().LogProcess("ContainerBackend.Rm %q %v", name, opts). + Do(func() { + resErr = runtime.ContainerBackend.Rm(ctx, name, opts) + }) + return +} + +func (runtime *PerfCheckContainerBackend) PostManifest(ctx context.Context, ref string, opts PostManifestOpts) (resErr error) { + logboek.Context(ctx).Default().LogProcess("ContainerBackend.PostManifest %q %v", ref, opts). + Do(func() { + resErr = runtime.ContainerBackend.PostManifest(ctx, ref, opts) + }) + return +} + +func (runtime *PerfCheckContainerBackend) TagImageByName(ctx context.Context, img LegacyImageInterface) (resErr error) { + logboek.Context(ctx).Default().LogProcess("ContainerBackend.TagImageByName %q", img.Name()). + Do(func() { + resErr = runtime.ContainerBackend.TagImageByName(ctx, img) + }) + return +} diff --git a/pkg/docker/image.go b/pkg/docker/image.go index ee60cd94f8..937a5ad557 100644 --- a/pkg/docker/image.go +++ b/pkg/docker/image.go @@ -20,16 +20,14 @@ import ( parallelConstant "github.com/werf/werf/pkg/util/parallel/constant" ) -func CreateImage(ctx context.Context, ref string, labels map[string]string) error { +func CreateImage(ctx context.Context, ref string, labels []string) error { var opts types.ImageImportOptions if len(labels) > 0 { changeOption := "LABEL" - - for k, v := range labels { - changeOption += fmt.Sprintf(" %s=%s", k, v) + for _, label := range labels { + changeOption += fmt.Sprintf(" %s", label) } - opts.Changes = append(opts.Changes, changeOption) } diff --git a/pkg/image/container.go b/pkg/image/container.go new file mode 100644 index 0000000000..92ab42b090 --- /dev/null +++ b/pkg/image/container.go @@ -0,0 +1,23 @@ +package image + +type Container struct { + ID string + ImageID string + Names []string +} + +func (container Container) LogName() string { + name := container.ID + if len(container.Names) != 0 { + name = container.Names[0] + } + return name +} + +type ContainerList []Container + +type ContainerFilter struct { + ID string + Name string + Ancestor string +} diff --git a/pkg/image/info.go b/pkg/image/info.go index 63ae40b0d6..762f82eb97 100644 --- a/pkg/image/info.go +++ b/pkg/image/info.go @@ -54,6 +54,14 @@ func (info *Info) GetCopy() *Info { } } +func (info *Info) LogName() string { + if info.Name == ":" { + return info.ID + } else { + return info.Name + } +} + func NewInfoFromInspect(ref string, inspect *types.ImageInspect) *Info { repository, tag := ParseRepositoryAndTag(ref) diff --git a/pkg/image/summary.go b/pkg/image/summary.go new file mode 100644 index 0000000000..2773f7b67a --- /dev/null +++ b/pkg/image/summary.go @@ -0,0 +1,49 @@ +package image + +import ( + "fmt" + "strings" +) + +type Summary struct { + RepoTags []string +} + +type ImagesList []Summary + +// FIXME(multiarch): take into account multiarch stages, which does not use uniqueID +func (list ImagesList) ConvertToStages() ([]StageID, error) { + var stagesList []StageID + + for _, summary := range list { + repoTags := summary.RepoTags + if len(repoTags) == 0 { + repoTags = append(repoTags, ":") + } + + for _, repoTag := range repoTags { + _, tag := ParseRepositoryAndTag(repoTag) + + if len(tag) != 70 || len(strings.Split(tag, "-")) != 2 { // 2604b86b2c7a1c6d19c62601aadb19e7d5c6bb8f17bc2bf26a390ea7-1611836746968 + continue + } + + if digest, uniqueID, err := getDigestAndUniqueIDFromLocalStageImageTag(tag); err != nil { + return nil, err + } else { + stagesList = append(stagesList, StageID{Digest: digest, UniqueID: uniqueID}) + } + } + } + + return stagesList, nil +} + +func getDigestAndUniqueIDFromLocalStageImageTag(repoStageImageTag string) (string, int64, error) { + parts := strings.SplitN(repoStageImageTag, "-", 2) + if uniqueID, err := ParseUniqueIDAsTimestamp(parts[1]); err != nil { + return "", 0, fmt.Errorf("unable to parse unique id %s as timestamp: %w", parts[1], err) + } else { + return parts[0], uniqueID, nil + } +} diff --git a/pkg/storage/docker_server_stages_storage.go b/pkg/storage/docker_server_stages_storage.go deleted file mode 100644 index aec8d992f7..0000000000 --- a/pkg/storage/docker_server_stages_storage.go +++ /dev/null @@ -1,570 +0,0 @@ -package storage - -import ( - "context" - "fmt" - "strconv" - "strings" - - "github.com/docker/docker/api/types" - "github.com/docker/docker/api/types/filters" - v1 "github.com/google/go-containerregistry/pkg/v1" - - "github.com/werf/logboek" - "github.com/werf/werf/pkg/container_backend" - "github.com/werf/werf/pkg/docker" - "github.com/werf/werf/pkg/docker_registry" - "github.com/werf/werf/pkg/image" - "github.com/werf/werf/pkg/util" -) - -const ( - LocalStage_ImageRepoFormat = "%s" - LocalStage_ImageFormat = "%s:%s-%d" - - LocalImportMetadata_ImageNameFormat = "werf-import-metadata/%s" - LocalImportMetadata_TagFormat = "%s" - - LocalClientIDRecord_ImageNameFormat = "werf-client-id/%s" - LocalClientIDRecord_ImageFormat = "werf-client-id/%s:%s-%d" -) - -const ImageDeletionFailedDueToUsedByContainerErrorTip = "Use --force option to remove all containers that are based on deleting werf docker images" - -func IsImageDeletionFailedDueToUsingByContainerErr(err error) bool { - return strings.HasSuffix(err.Error(), ImageDeletionFailedDueToUsedByContainerErrorTip) -} - -func getDigestAndUniqueIDFromLocalStageImageTag(repoStageImageTag string) (string, int64, error) { - parts := strings.SplitN(repoStageImageTag, "-", 2) - if uniqueID, err := image.ParseUniqueIDAsTimestamp(parts[1]); err != nil { - return "", 0, fmt.Errorf("unable to parse unique id %s as timestamp: %w", parts[1], err) - } else { - return parts[0], uniqueID, nil - } -} - -type DockerServerStagesStorage struct { - // Local stages storage is compatible only with docker-server backed runtime - DockerServerBackend *container_backend.DockerServerBackend -} - -func NewDockerServerStagesStorage(dockerServerBackend *container_backend.DockerServerBackend) *DockerServerStagesStorage { - return &DockerServerStagesStorage{DockerServerBackend: dockerServerBackend} -} - -func (storage *DockerServerStagesStorage) ConstructStageImageName(projectName, digest string, uniqueID int64) string { - return fmt.Sprintf(LocalStage_ImageFormat, projectName, digest, uniqueID) -} - -func (storage *DockerServerStagesStorage) GetStagesIDs(ctx context.Context, projectName string, _ ...Option) ([]image.StageID, error) { - filterSet := localStagesStorageFilterSetBase(projectName) - images, err := docker.Images(ctx, types.ImageListOptions{Filters: filterSet}) - if err != nil { - return nil, fmt.Errorf("unable to get docker images: %w", err) - } - - return convertToStagesList(images) -} - -func (storage *DockerServerStagesStorage) ExportStage(ctx context.Context, stageDescription *image.StageDescription, destinationReference string, mutateConfigFunc func(config v1.Config) (v1.Config, error)) error { - if err := docker.CliTag(ctx, stageDescription.Info.Name, destinationReference); err != nil { - return err - } - defer func() { _ = docker.CliRmi(ctx, destinationReference) }() - - if err := docker.CliPushWithRetries(ctx, destinationReference); err != nil { - return err - } - - return docker_registry.API().MutateAndPushImage(ctx, destinationReference, destinationReference, mutateExportStageConfig(mutateConfigFunc)) -} - -func (storage *DockerServerStagesStorage) DeleteStage(ctx context.Context, stageDescription *image.StageDescription, options DeleteImageOptions) error { - return deleteRepoImageListInDockerServerStagesStorage(ctx, stageDescription, options.RmiForce) -} - -func (storage *DockerServerStagesStorage) RejectStage(_ context.Context, _, _ string, _ int64) error { - return nil -} - -type FilterStagesAndProcessRelatedDataOptions struct { - SkipUsedImage bool - RmForce bool - RmContainersThatUseImage bool -} - -func (storage *DockerServerStagesStorage) FilterStagesAndProcessRelatedData(ctx context.Context, stageDescriptions []*image.StageDescription, options FilterStagesAndProcessRelatedDataOptions) ([]*image.StageDescription, error) { - return processRelatedContainers(ctx, stageDescriptions, processRelatedContainersOptions{ - skipUsedImages: options.SkipUsedImage, - rmContainersThatUseImage: options.RmContainersThatUseImage, - rmForce: options.RmForce, - }) -} - -func (storage *DockerServerStagesStorage) CreateRepo(_ context.Context) error { - return nil -} - -func (storage *DockerServerStagesStorage) DeleteRepo(_ context.Context) error { - return nil -} - -func (storage *DockerServerStagesStorage) GetStageDescription(ctx context.Context, projectName, digest string, uniqueID int64) (*image.StageDescription, error) { - stageImageName := storage.ConstructStageImageName(projectName, digest, uniqueID) - - inspect, err := storage.DockerServerBackend.GetImageInspect(ctx, stageImageName) - if err != nil { - return nil, fmt.Errorf("unable to get image %s inspect: %w", stageImageName, err) - } - - if inspect != nil { - return &image.StageDescription{ - StageID: &image.StageID{Digest: digest, UniqueID: uniqueID}, - Info: image.NewInfoFromInspect(stageImageName, inspect), - }, nil - } - - return nil, nil -} - -func (storage *DockerServerStagesStorage) AddStageCustomTag(_ context.Context, _ *image.StageDescription, _ string) error { - return fmt.Errorf("not implemented") -} - -func (storage *DockerServerStagesStorage) CheckStageCustomTag(_ context.Context, _ *image.StageDescription, _ string) error { - return fmt.Errorf("not implemented") -} - -func (storage *DockerServerStagesStorage) DeleteStageCustomTag(_ context.Context, _ string) error { - return fmt.Errorf("not implemented") -} - -func (storage *DockerServerStagesStorage) GetStageCustomTagMetadata(_ context.Context, _ string) (*CustomTagMetadata, error) { - return nil, fmt.Errorf("not implemented") -} - -func (storage *DockerServerStagesStorage) GetStageCustomTagMetadataIDs(_ context.Context, _ ...Option) ([]string, error) { - return nil, nil -} - -func (storage *DockerServerStagesStorage) RegisterStageCustomTag(_ context.Context, _ *image.StageDescription, _ string) error { - return nil -} - -func (storage *DockerServerStagesStorage) UnregisterStageCustomTag(_ context.Context, _ string) error { - return nil -} - -func (storage *DockerServerStagesStorage) AddManagedImage(_ context.Context, _, _ string) error { - return nil -} - -func (storage *DockerServerStagesStorage) RmManagedImage(_ context.Context, _, _ string) error { - return nil -} - -func (storage *DockerServerStagesStorage) IsManagedImageExist(_ context.Context, _, _ string, _ ...Option) (bool, error) { - return false, nil -} - -func (storage *DockerServerStagesStorage) GetManagedImages(_ context.Context, _ string, _ ...Option) ([]string, error) { - return []string{}, nil -} - -func (storage *DockerServerStagesStorage) GetStagesIDsByDigest(ctx context.Context, projectName, digest string, _ ...Option) ([]image.StageID, error) { - filterSet := filters.NewArgs() - filterSet.Add("reference", fmt.Sprintf(LocalStage_ImageRepoFormat, projectName)) - // NOTE digest already depends on build-cache-version - filterSet.Add("label", fmt.Sprintf("%s=%s", image.WerfStageDigestLabel, digest)) - - images, err := docker.Images(ctx, types.ImageListOptions{Filters: filterSet}) - if err != nil { - return nil, fmt.Errorf("unable to get docker images: %w", err) - } - - return convertToStagesList(images) -} - -func (storage *DockerServerStagesStorage) ShouldFetchImage(_ context.Context, _ container_backend.LegacyImageInterface) (bool, error) { - return false, nil -} - -func (storage *DockerServerStagesStorage) FetchImage(_ context.Context, _ container_backend.LegacyImageInterface) error { - return nil -} - -func (storage *DockerServerStagesStorage) StoreImage(ctx context.Context, img container_backend.LegacyImageInterface) error { - return storage.DockerServerBackend.TagImageByName(ctx, img) -} - -func (storage *DockerServerStagesStorage) PutImageMetadata(_ context.Context, _, _, _, _ string) error { - return nil -} - -func (storage *DockerServerStagesStorage) RmImageMetadata(_ context.Context, _, _, _, _ string) error { - return nil -} - -func (storage *DockerServerStagesStorage) selectFullImageMetadataName(_ context.Context, _, _, _, _ string) (string, error) { - return "", nil -} - -func (storage *DockerServerStagesStorage) IsImageMetadataExist(_ context.Context, _, _, _, _ string, _ ...Option) (bool, error) { - return false, nil -} - -func (storage *DockerServerStagesStorage) GetAllAndGroupImageMetadataByImageName(_ context.Context, _ string, _ []string, _ ...Option) (map[string]map[string][]string, map[string]map[string][]string, error) { - return map[string]map[string][]string{}, map[string]map[string][]string{}, nil -} - -func (storage *DockerServerStagesStorage) GetImportMetadata(ctx context.Context, projectName, id string) (*ImportMetadata, error) { - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.GetImportMetadata %s %s\n", projectName, id) - - fullImageName := makeLocalImportMetadataName(projectName, id) - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.GetImportMetadata full image name: %s\n", fullImageName) - - inspect, err := storage.DockerServerBackend.GetImageInspect(ctx, fullImageName) - if err != nil { - return nil, fmt.Errorf("unable to get image %s inspect: %w", fullImageName, err) - } - - if inspect != nil { - return newImportMetadataFromLabels(inspect.Config.Labels), nil - } - - return nil, nil -} - -func (storage *DockerServerStagesStorage) PutImportMetadata(ctx context.Context, projectName string, metadata *ImportMetadata) error { - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.PutImportMetadata %s %v\n", projectName, metadata) - - fullImageName := makeLocalImportMetadataName(projectName, metadata.ImportSourceID) - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.PutImportMetadata full image name: %s\n", fullImageName) - - if exists, err := docker.ImageExist(ctx, fullImageName); err != nil { - return fmt.Errorf("unable to check existence of image %q: %w", fullImageName, err) - } else if exists { - return nil - } - - labels := metadata.ToLabels() - labels[image.WerfLabel] = projectName - - if err := docker.CreateImage(ctx, fullImageName, labels); err != nil { - return fmt.Errorf("unable to create image %q: %w", fullImageName, err) - } - - return nil -} - -func (storage *DockerServerStagesStorage) RmImportMetadata(ctx context.Context, projectName, id string) error { - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.RmImportMetadata %s %s\n", projectName, id) - - fullImageName := makeLocalImportMetadataName(projectName, id) - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.RmImportMetadata full image name: %s\n", fullImageName) - - if exists, err := docker.ImageExist(ctx, fullImageName); err != nil { - return fmt.Errorf("unable to check existence of image %s: %w", fullImageName, err) - } else if !exists { - return nil - } - - if err := docker.CliRmi(ctx, "--force", fullImageName); err != nil { - return fmt.Errorf("unable to remove image %s: %w", fullImageName, err) - } - - return nil -} - -func (storage *DockerServerStagesStorage) GetImportMetadataIDs(ctx context.Context, projectName string, _ ...Option) ([]string, error) { - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.GetImportMetadataIDs %s\n", projectName) - - filterSet := filters.NewArgs() - filterSet.Add("reference", fmt.Sprintf(LocalImportMetadata_ImageNameFormat, projectName)) - - images, err := docker.Images(ctx, types.ImageListOptions{Filters: filterSet}) - if err != nil { - return nil, fmt.Errorf("unable to get docker images: %w", err) - } - - var tags []string - for _, img := range images { - for _, repoTag := range img.RepoTags { - _, tag := image.ParseRepositoryAndTag(repoTag) - tags = append(tags, tag) - } - } - - return tags, nil -} - -func makeLocalImportMetadataName(projectName, importSourceID string) string { - return strings.Join( - []string{ - fmt.Sprintf(LocalImportMetadata_ImageNameFormat, projectName), - fmt.Sprintf(LocalImportMetadata_TagFormat, importSourceID), - }, ":", - ) -} - -func (storage *DockerServerStagesStorage) String() string { - return LocalStorageAddress -} - -func (storage *DockerServerStagesStorage) Address() string { - return LocalStorageAddress -} - -func (storage *DockerServerStagesStorage) GetClientIDRecords(ctx context.Context, projectName string, _ ...Option) ([]*ClientIDRecord, error) { - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.GetClientID for project %s\n", projectName) - - filterSet := filters.NewArgs() - filterSet.Add("reference", fmt.Sprintf(LocalClientIDRecord_ImageNameFormat, projectName)) - - images, err := docker.Images(ctx, types.ImageListOptions{Filters: filterSet}) - if err != nil { - return nil, fmt.Errorf("unable to get docker images: %w", err) - } - - var res []*ClientIDRecord - for _, img := range images { - for _, repoTag := range img.RepoTags { - _, tag := image.ParseRepositoryAndTag(repoTag) - - tagParts := strings.SplitN(util.Reverse(tag), "-", 2) - if len(tagParts) != 2 { - continue - } - - clientID, timestampMillisecStr := util.Reverse(tagParts[1]), util.Reverse(tagParts[0]) - - timestampMillisec, err := strconv.ParseInt(timestampMillisecStr, 10, 64) - if err != nil { - continue - } - - rec := &ClientIDRecord{ClientID: clientID, TimestampMillisec: timestampMillisec} - res = append(res, rec) - - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.GetClientID got clientID record: %s\n", rec) - } - } - - return res, nil -} - -func (storage *DockerServerStagesStorage) PostClientIDRecord(ctx context.Context, projectName string, rec *ClientIDRecord) error { - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.PostClientID %s for project %s\n", rec.ClientID, projectName) - - fullImageName := fmt.Sprintf(LocalClientIDRecord_ImageFormat, projectName, rec.ClientID, rec.TimestampMillisec) - - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.PostClientID full image name: %s\n", fullImageName) - - labels := map[string]string{image.WerfLabel: projectName} - if err := docker.CreateImage(ctx, fullImageName, labels); err != nil { - return fmt.Errorf("unable to create image %q: %w", fullImageName, err) - } - - logboek.Context(ctx).Info().LogF("Posted new clientID %q for project %s\n", rec.ClientID, projectName) - - return nil -} - -func (storage *DockerServerStagesStorage) PostMultiplatformImage(ctx context.Context, projectName, tag string, allPlatformsImages []*image.Info) error { - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.PostMultiplatformImage by tag %s for project %s\n", tag, projectName) - - fullImageName := fmt.Sprintf("%s:%s", projectName, tag) - - logboek.Context(ctx).Debug().LogF("-- DockerServerStagesStorage.PostMultiplatformImage full image name: %s\n", fullImageName) - - // opts := docker_registry.ManifestListOptions{ - // PushImageOptions: docker_registry.PushImageOptions{ - // Labels: map[string]string{image.WerfLabel: projectName}, - // }, - // Manifests: allPlatformsImages, - // } - - // if err := storage.DockerRegistry.PushManifestList(ctx, fullImageName, opts); err != nil { - // return fmt.Errorf("unable to push image %s: %w", fullImageName, err) - // } - - // FIXME(multiarch): use container backend to create manifest list - - // logboek.Context(ctx).Info().LogF("Created manifest list %s for project %s\n", fullImageName, projectName) - - return nil -} - -type processRelatedContainersOptions struct { - skipUsedImages bool - rmContainersThatUseImage bool - rmForce bool -} - -func processRelatedContainers(ctx context.Context, stageDescriptionList []*image.StageDescription, options processRelatedContainersOptions) ([]*image.StageDescription, error) { - filterSet := filters.NewArgs() - for _, stageDescription := range stageDescriptionList { - filterSet.Add("ancestor", stageDescription.Info.ID) - } - - containerList, err := containerListByFilterSet(ctx, filterSet) - if err != nil { - return nil, err - } - - var stageDescriptionListToExcept []*image.StageDescription - var containerListToRemove []types.Container - for _, container := range containerList { - for _, stageDescription := range stageDescriptionList { - imageInfo := stageDescription.Info - - if imageInfo.ID == container.ImageID { - switch { - case options.skipUsedImages: - logboek.Context(ctx).Default().LogFDetails("Skip image %s (used by container %s)\n", logImageName(imageInfo), logContainerName(container)) - stageDescriptionListToExcept = append(stageDescriptionListToExcept, stageDescription) - case options.rmContainersThatUseImage: - containerListToRemove = append(containerListToRemove, container) - default: - return nil, fmt.Errorf("cannot remove image %s used by container %s\n%s", logImageName(imageInfo), logContainerName(container), ImageDeletionFailedDueToUsedByContainerErrorTip) - } - } - } - } - - if err := deleteContainers(ctx, containerListToRemove, options.rmForce); err != nil { - return nil, err - } - - return exceptStageDescriptionList(stageDescriptionList, stageDescriptionListToExcept...), nil -} - -func containerListByFilterSet(ctx context.Context, filterSet filters.Args) ([]types.Container, error) { - containersOptions := types.ContainerListOptions{} - containersOptions.All = true - containersOptions.Filters = filterSet - - return docker.Containers(ctx, containersOptions) -} - -func deleteContainers(ctx context.Context, containers []types.Container, rmForce bool) error { - for _, container := range containers { - if err := docker.ContainerRemove(ctx, container.ID, types.ContainerRemoveOptions{Force: rmForce}); err != nil { - return err - } - } - - return nil -} - -func exceptStageDescriptionList(stageDescriptionList []*image.StageDescription, stageDescriptionListToExcept ...*image.StageDescription) []*image.StageDescription { - var result []*image.StageDescription - -loop: - for _, sd1 := range stageDescriptionList { - for _, sd2 := range stageDescriptionListToExcept { - if sd2 == sd1 { - continue loop - } - } - - result = append(result, sd1) - } - - return result -} - -func localStagesStorageFilterSetBase(projectName string) filters.Args { - filterSet := filters.NewArgs() - filterSet.Add("reference", fmt.Sprintf(LocalStage_ImageRepoFormat, projectName)) - filterSet.Add("label", fmt.Sprintf("%s=%s", image.WerfLabel, projectName)) - filterSet.Add("label", fmt.Sprintf("%s=%s", image.WerfCacheVersionLabel, image.BuildCacheVersion)) - return filterSet -} - -func logImageName(image *image.Info) string { - if image.Name == ":" { - return image.ID - } else { - return image.Name - } -} - -func logContainerName(container types.Container) string { - name := container.ID - if len(container.Names) != 0 { - name = container.Names[0] - } - - return name -} - -func convertToStagesList(imageSummaryList []types.ImageSummary) ([]image.StageID, error) { - var stagesList []image.StageID - - for _, imageSummary := range imageSummaryList { - repoTags := imageSummary.RepoTags - if len(repoTags) == 0 { - repoTags = append(repoTags, ":") - } - - for _, repoTag := range repoTags { - _, tag := image.ParseRepositoryAndTag(repoTag) - - if len(tag) != 70 || len(strings.Split(tag, "-")) != 2 { // 2604b86b2c7a1c6d19c62601aadb19e7d5c6bb8f17bc2bf26a390ea7-1611836746968 - continue - } - - if digest, uniqueID, err := getDigestAndUniqueIDFromLocalStageImageTag(tag); err != nil { - return nil, err - } else { - stagesList = append(stagesList, image.StageID{Digest: digest, UniqueID: uniqueID}) - } - } - } - - return stagesList, nil -} - -func deleteRepoImageListInDockerServerStagesStorage(ctx context.Context, stageDescription *image.StageDescription, rmiForce bool) error { - var imageReferences []string - imageInfo := stageDescription.Info - - if imageInfo.Name == "" { - imageReferences = append(imageReferences, imageInfo.ID) - } else { - isDanglingImage := imageInfo.Name == ":" - isTaglessImage := !isDanglingImage && imageInfo.Tag == "" - - if isDanglingImage || isTaglessImage { - imageReferences = append(imageReferences, imageInfo.ID) - } else { - imageReferences = append(imageReferences, imageInfo.Name) - } - } - - if err := imageReferencesRemove(ctx, imageReferences, rmiForce); err != nil { - return err - } - - return nil -} - -func imageReferencesRemove(ctx context.Context, references []string, rmiForce bool) error { - if len(references) == 0 { - return nil - } - - var args []string - if rmiForce { - args = append(args, "--force") - } - args = append(args, references...) - - if err := docker.CliRmi_LiveOutput(ctx, args...); err != nil { - return err - } - - return nil -} diff --git a/pkg/storage/import_metadata.go b/pkg/storage/import_metadata.go index b3c88de925..67e4a8a54c 100644 --- a/pkg/storage/import_metadata.go +++ b/pkg/storage/import_metadata.go @@ -1,6 +1,8 @@ package storage import ( + "fmt" + "github.com/werf/werf/pkg/image" ) @@ -10,7 +12,15 @@ type ImportMetadata struct { Checksum string } -func (m *ImportMetadata) ToLabels() map[string]string { +func (m *ImportMetadata) ToLabels() []string { + return []string{ + fmt.Sprintf("%s=%s", image.WerfImportMetadataImportSourceIDLabel, m.ImportSourceID), + fmt.Sprintf("%s=%s", image.WerfImportMetadataSourceImageIDLabel, m.SourceImageID), + fmt.Sprintf("%s=%s", image.WerfImportMetadataChecksumLabel, m.Checksum), + } +} + +func (m *ImportMetadata) ToLabelsMap() map[string]string { return map[string]string{ image.WerfImportMetadataImportSourceIDLabel: m.ImportSourceID, image.WerfImportMetadataSourceImageIDLabel: m.SourceImageID, diff --git a/pkg/storage/local_stages_storage.go b/pkg/storage/local_stages_storage.go new file mode 100644 index 0000000000..5ea3f87f41 --- /dev/null +++ b/pkg/storage/local_stages_storage.go @@ -0,0 +1,438 @@ +package storage + +import ( + "context" + "fmt" + "strconv" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + + "github.com/werf/logboek" + "github.com/werf/werf/pkg/container_backend" + "github.com/werf/werf/pkg/docker_registry" + "github.com/werf/werf/pkg/image" + "github.com/werf/werf/pkg/util" +) + +const ( + LocalStage_ImageRepoFormat = "%s" + LocalStage_ImageFormatWithUniqueID = "%s:%s-%d" + LocalStage_ImageFormat = "%s:%s" + + LocalImportMetadata_ImageNameFormat = "werf-import-metadata/%s" + LocalImportMetadata_TagFormat = "%s" + + LocalClientIDRecord_ImageNameFormat = "werf-client-id/%s" + LocalClientIDRecord_ImageFormat = "werf-client-id/%s:%s-%d" + + ImageDeletionFailedDueToUsedByContainerErrorTip = "Use --force option to remove all containers that are based on deleting werf docker images" +) + +func IsImageDeletionFailedDueToUsingByContainerErr(err error) bool { + return strings.HasSuffix(err.Error(), ImageDeletionFailedDueToUsedByContainerErrorTip) +} + +type LocalStagesStorage struct { + ContainerBackend container_backend.ContainerBackend +} + +func NewLocalStagesStorage(containerBackend container_backend.ContainerBackend) *LocalStagesStorage { + return &LocalStagesStorage{ContainerBackend: containerBackend} +} + +func (storage *LocalStagesStorage) FilterStagesAndProcessRelatedData(ctx context.Context, stageDescriptions []*image.StageDescription, opts FilterStagesAndProcessRelatedDataOptions) ([]*image.StageDescription, error) { + containersOpts := container_backend.ContainersOptions{} + for _, stageDescription := range stageDescriptions { + containersOpts.Filters = append(containersOpts.Filters, image.ContainerFilter{Ancestor: stageDescription.Info.ID}) + } + containers, err := storage.ContainerBackend.Containers(ctx, containersOpts) + if err != nil { + return nil, err + } + + var stageDescriptionListToExcept []*image.StageDescription + var containerListToRemove []image.Container + for _, container := range containers { + for _, stageDescription := range stageDescriptions { + imageInfo := stageDescription.Info + + if imageInfo.ID == container.ImageID { + switch { + case opts.SkipUsedImage: + logboek.Context(ctx).Default().LogFDetails("Skip image %s (used by container %s)\n", imageInfo.LogName(), container.LogName()) + stageDescriptionListToExcept = append(stageDescriptionListToExcept, stageDescription) + case opts.RmContainersThatUseImage: + containerListToRemove = append(containerListToRemove, container) + default: + return nil, fmt.Errorf("cannot remove image %s used by container %s\n%s", imageInfo.LogName(), container.LogName(), ImageDeletionFailedDueToUsedByContainerErrorTip) + } + } + } + } + + if err := storage.deleteContainers(ctx, containerListToRemove, opts.RmForce); err != nil { + return nil, err + } + + return exceptStageDescriptionList(stageDescriptions, stageDescriptionListToExcept...), nil +} + +func exceptStageDescriptionList(stageDescriptionList []*image.StageDescription, stageDescriptionListToExcept ...*image.StageDescription) []*image.StageDescription { + var result []*image.StageDescription + +loop: + for _, sd1 := range stageDescriptionList { + for _, sd2 := range stageDescriptionListToExcept { + if sd2 == sd1 { + continue loop + } + } + + result = append(result, sd1) + } + + return result +} + +func (storage *LocalStagesStorage) deleteContainers(ctx context.Context, containers []image.Container, rmForce bool) error { + for _, container := range containers { + if err := storage.ContainerBackend.Rm(ctx, container.ID, container_backend.RmOpts{Force: rmForce}); err != nil { + return fmt.Errorf("unable to remove container %q: %w", container.ID, err) + } + } + return nil +} + +func (storage *LocalStagesStorage) GetStagesIDs(ctx context.Context, projectName string, opts ...Option) ([]image.StageID, error) { + imagesOpts := container_backend.ImagesOptions{} + imagesOpts.Filters = append(imagesOpts.Filters, util.NewPair("reference", fmt.Sprintf(LocalStage_ImageRepoFormat, projectName))) + imagesOpts.Filters = append(imagesOpts.Filters, util.NewPair("label", fmt.Sprintf("%s=%s", image.WerfLabel, projectName))) + imagesOpts.Filters = append(imagesOpts.Filters, util.NewPair("label", fmt.Sprintf("%s=%s", image.WerfCacheVersionLabel, image.BuildCacheVersion))) + + images, err := storage.ContainerBackend.Images(ctx, imagesOpts) + if err != nil { + return nil, fmt.Errorf("unable to list images: %w", err) + } + return images.ConvertToStages() +} + +func (storage *LocalStagesStorage) GetStagesIDsByDigest(ctx context.Context, projectName, digest string, opts ...Option) ([]image.StageID, error) { + imagesOpts := container_backend.ImagesOptions{} + imagesOpts.Filters = append(imagesOpts.Filters, util.NewPair("reference", fmt.Sprintf(LocalStage_ImageRepoFormat, projectName))) + // NOTE digest already depends on build-cache-version + imagesOpts.Filters = append(imagesOpts.Filters, util.NewPair("label", fmt.Sprintf("%s=%s", image.WerfStageDigestLabel, digest))) + + images, err := storage.ContainerBackend.Images(ctx, imagesOpts) + if err != nil { + return nil, fmt.Errorf("unable to get docker images: %w", err) + } + return images.ConvertToStages() +} + +func (storage *LocalStagesStorage) GetStageDescription(ctx context.Context, projectName, digest string, uniqueID int64) (*image.StageDescription, error) { + stageImageName := storage.ConstructStageImageName(projectName, digest, uniqueID) + info, err := storage.ContainerBackend.GetImageInfo(ctx, stageImageName, container_backend.GetImageInfoOpts{}) + if err != nil { + return nil, fmt.Errorf("unable to get image %s info: %w", stageImageName, err) + } + + if info != nil { + return &image.StageDescription{ + StageID: &image.StageID{Digest: digest, UniqueID: uniqueID}, + Info: info, + }, nil + } + return nil, nil +} + +func (storage *LocalStagesStorage) ExportStage(ctx context.Context, stageDescription *image.StageDescription, destinationReference string, mutateConfigFunc func(config v1.Config) (v1.Config, error)) error { + if err := storage.ContainerBackend.Tag(ctx, stageDescription.Info.Name, destinationReference, container_backend.TagOpts{}); err != nil { + return fmt.Errorf("unable to tag %q as %q: %w", stageDescription.Info.Name, destinationReference, err) + } + defer func() { + _ = storage.ContainerBackend.Rmi(ctx, destinationReference, container_backend.RmiOpts{Force: true}) + }() + + if err := storage.ContainerBackend.Push(ctx, destinationReference, container_backend.PushOpts{}); err != nil { + return fmt.Errorf("unable to push %q: %w", destinationReference, err) + } + return docker_registry.API().MutateAndPushImage(ctx, destinationReference, destinationReference, mutateExportStageConfig(mutateConfigFunc)) +} + +func (storage *LocalStagesStorage) DeleteStage(ctx context.Context, stageDescription *image.StageDescription, options DeleteImageOptions) error { + var imageReferences []string + imageInfo := stageDescription.Info + + if imageInfo.Name == "" { + imageReferences = append(imageReferences, imageInfo.ID) + } else { + isDanglingImage := imageInfo.Name == ":" + isTaglessImage := !isDanglingImage && imageInfo.Tag == "" + + if isDanglingImage || isTaglessImage { + imageReferences = append(imageReferences, imageInfo.ID) + } else { + imageReferences = append(imageReferences, imageInfo.Name) + } + } + + for _, ref := range imageReferences { + if err := storage.ContainerBackend.Rmi(ctx, ref, container_backend.RmiOpts{Force: options.RmiForce}); err != nil { + return fmt.Errorf("unable to remove %q: %w", ref, err) + } + } + return nil +} + +func (storage *LocalStagesStorage) AddStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error { + return fmt.Errorf("not implemented") +} + +func (storage *LocalStagesStorage) CheckStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error { + return fmt.Errorf("not implemented") +} + +func (storage *LocalStagesStorage) DeleteStageCustomTag(ctx context.Context, tag string) error { + return fmt.Errorf("not implemented") +} + +func (storage *LocalStagesStorage) RejectStage(ctx context.Context, projectName, digest string, uniqueID int64) error { + return nil +} + +func (storage *LocalStagesStorage) ConstructStageImageName(projectName, digest string, uniqueID int64) string { + if uniqueID == 0 { + return fmt.Sprintf(LocalStage_ImageFormat, projectName, digest) + } + return fmt.Sprintf(LocalStage_ImageFormatWithUniqueID, projectName, digest, uniqueID) +} + +func (storage *LocalStagesStorage) FetchImage(ctx context.Context, img container_backend.LegacyImageInterface) error { + return nil +} + +func (storage *LocalStagesStorage) StoreImage(ctx context.Context, img container_backend.LegacyImageInterface) error { + return storage.ContainerBackend.TagImageByName(ctx, img) +} + +func (storage *LocalStagesStorage) ShouldFetchImage(ctx context.Context, img container_backend.LegacyImageInterface) (bool, error) { + return false, nil +} + +func (storage *LocalStagesStorage) CreateRepo(ctx context.Context) error { return nil } + +func (storage *LocalStagesStorage) DeleteRepo(ctx context.Context) error { return nil } + +func (storage *LocalStagesStorage) AddManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error { + return nil +} + +func (storage *LocalStagesStorage) RmManagedImage(ctx context.Context, projectName, imageNameOrManagedImageName string) error { + return nil +} + +func (storage *LocalStagesStorage) IsManagedImageExist(ctx context.Context, projectName, imageNameOrManagedImageName string, opts ...Option) (bool, error) { + return false, nil +} + +func (storage *LocalStagesStorage) GetManagedImages(ctx context.Context, projectName string, opts ...Option) ([]string, error) { + return []string{}, nil +} + +func (storage *LocalStagesStorage) PutImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string) error { + return nil +} + +func (storage *LocalStagesStorage) RmImageMetadata(ctx context.Context, projectName, imageNameOrManagedImageNameOrImageMetadataID, commit, stageID string) error { + return nil +} + +func (storage *LocalStagesStorage) IsImageMetadataExist(ctx context.Context, projectName, imageNameOrManagedImageName, commit, stageID string, opts ...Option) (bool, error) { + return false, nil +} + +func (storage *LocalStagesStorage) GetAllAndGroupImageMetadataByImageName(ctx context.Context, projectName string, imageNameOrManagedImageList []string, opts ...Option) (map[string]map[string][]string, map[string]map[string][]string, error) { + return map[string]map[string][]string{}, map[string]map[string][]string{}, nil +} + +func (storage *LocalStagesStorage) GetImportMetadata(ctx context.Context, projectName, id string) (*ImportMetadata, error) { + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.GetImportMetadata %s %s\n", projectName, id) + + fullImageName := makeLocalImportMetadataName(projectName, id) + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.GetImportMetadata full image name: %s\n", fullImageName) + + info, err := storage.ContainerBackend.GetImageInfo(ctx, fullImageName, container_backend.GetImageInfoOpts{}) + if err != nil { + return nil, fmt.Errorf("unable to get image %s info: %w", fullImageName, err) + } + if info == nil { + return nil, nil + } + return newImportMetadataFromLabels(info.Labels), nil +} + +func (storage *LocalStagesStorage) PutImportMetadata(ctx context.Context, projectName string, metadata *ImportMetadata) error { + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.PutImportMetadata %s %v\n", projectName, metadata) + + fullImageName := makeLocalImportMetadataName(projectName, metadata.ImportSourceID) + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.PutImportMetadata full image name: %s\n", fullImageName) + + if info, err := storage.ContainerBackend.GetImageInfo(ctx, fullImageName, container_backend.GetImageInfoOpts{}); err != nil { + return fmt.Errorf("unable to check existence of image %s: %w", fullImageName, err) + } else if info != nil { + return nil + } + + labels := metadata.ToLabels() + labels = append(labels, fmt.Sprintf("%s=%s", image.WerfLabel, projectName)) + if err := storage.ContainerBackend.PostManifest(ctx, fullImageName, container_backend.PostManifestOpts{Labels: labels}); err != nil { + return fmt.Errorf("unable to post manifest %q: %w", fullImageName, err) + } + return nil +} + +func (storage *LocalStagesStorage) RmImportMetadata(ctx context.Context, projectName, id string) error { + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.RmImportMetadata %s %s\n", projectName, id) + + fullImageName := makeLocalImportMetadataName(projectName, id) + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.RmImportMetadata full image name: %s\n", fullImageName) + + if info, err := storage.ContainerBackend.GetImageInfo(ctx, fullImageName, container_backend.GetImageInfoOpts{}); err != nil { + return fmt.Errorf("unable to check existence of image %s: %w", fullImageName, err) + } else if info != nil { + return nil + } + + if err := storage.ContainerBackend.Rmi(ctx, fullImageName, container_backend.RmiOpts{Force: true}); err != nil { + return fmt.Errorf("unable to remove image %s: %w", fullImageName, err) + } + return nil +} + +func (storage *LocalStagesStorage) GetImportMetadataIDs(ctx context.Context, projectName string, opts ...Option) ([]string, error) { + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.GetImportMetadataIDs %s\n", projectName) + + imagesOpts := container_backend.ImagesOptions{} + imagesOpts.Filters = append(imagesOpts.Filters, util.NewPair("reference", fmt.Sprintf(LocalImportMetadata_ImageNameFormat, projectName))) + images, err := storage.ContainerBackend.Images(ctx, imagesOpts) + if err != nil { + return nil, fmt.Errorf("unable to list images: %w", err) + } + + var tags []string + for _, img := range images { + for _, repoTag := range img.RepoTags { + _, tag := image.ParseRepositoryAndTag(repoTag) + tags = append(tags, tag) + } + } + + return tags, nil +} + +func (storage *LocalStagesStorage) GetClientIDRecords(ctx context.Context, projectName string, opts ...Option) ([]*ClientIDRecord, error) { + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.GetClientID for project %s\n", projectName) + + imagesOpts := container_backend.ImagesOptions{} + imagesOpts.Filters = append(imagesOpts.Filters, util.NewPair("reference", fmt.Sprintf(LocalClientIDRecord_ImageNameFormat, projectName))) + images, err := storage.ContainerBackend.Images(ctx, imagesOpts) + if err != nil { + return nil, fmt.Errorf("unable to list images: %w", err) + } + + var res []*ClientIDRecord + for _, img := range images { + for _, repoTag := range img.RepoTags { + _, tag := image.ParseRepositoryAndTag(repoTag) + + tagParts := strings.SplitN(util.Reverse(tag), "-", 2) + if len(tagParts) != 2 { + continue + } + + clientID, timestampMillisecStr := util.Reverse(tagParts[1]), util.Reverse(tagParts[0]) + + timestampMillisec, err := strconv.ParseInt(timestampMillisecStr, 10, 64) + if err != nil { + continue + } + + rec := &ClientIDRecord{ClientID: clientID, TimestampMillisec: timestampMillisec} + res = append(res, rec) + + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.GetClientID got clientID record: %s\n", rec) + } + } + return res, nil +} + +func (storage *LocalStagesStorage) PostClientIDRecord(ctx context.Context, projectName string, rec *ClientIDRecord) error { + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.PostClientID %s for project %s\n", rec.ClientID, projectName) + + fullImageName := fmt.Sprintf(LocalClientIDRecord_ImageFormat, projectName, rec.ClientID, rec.TimestampMillisec) + labels := []string{fmt.Sprintf("%s=%s", image.WerfLabel, projectName)} + + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.PostClientID full image name: %s\n", fullImageName) + + if err := storage.ContainerBackend.PostManifest(ctx, fullImageName, container_backend.PostManifestOpts{Labels: labels}); err != nil { + return fmt.Errorf("unable to post %q: %w", fullImageName, err) + } + + logboek.Context(ctx).Info().LogF("Posted new clientID %q for project %s\n", rec.ClientID, projectName) + + return nil +} + +func (storage *LocalStagesStorage) PostMultiplatformImage(ctx context.Context, projectName, tag string, allPlatformsImages []*image.Info) error { + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.PostMultiplatformImage by tag %s for project %s\n", tag, projectName) + + fullImageName := fmt.Sprintf("%s:%s", projectName, tag) + postManifestOpts := container_backend.PostManifestOpts{ + Labels: []string{fmt.Sprintf("%s=%s", image.WerfLabel, projectName)}, + Manifests: allPlatformsImages, + } + + logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.PostMultiplatformImage full image name: %s\n", fullImageName) + + if err := storage.ContainerBackend.PostManifest(ctx, fullImageName, postManifestOpts); err != nil { + return fmt.Errorf("unable to post manifest %q: %w", fullImageName, err) + } + + logboek.Context(ctx).Info().LogF("Created manifest list %s for project %s\n", fullImageName, projectName) + + return nil +} + +func (storage *LocalStagesStorage) String() string { + return LocalStorageAddress +} + +func (storage *LocalStagesStorage) Address() string { + return LocalStorageAddress +} + +func (storage *LocalStagesStorage) GetStageCustomTagMetadataIDs(ctx context.Context, opts ...Option) ([]string, error) { + return nil, nil +} + +func (storage *LocalStagesStorage) GetStageCustomTagMetadata(ctx context.Context, tagOrID string) (*CustomTagMetadata, error) { + return nil, fmt.Errorf("not implemented") +} + +func (storage *LocalStagesStorage) RegisterStageCustomTag(ctx context.Context, stageDescription *image.StageDescription, tag string) error { + return nil +} + +func (storage *LocalStagesStorage) UnregisterStageCustomTag(ctx context.Context, tag string) error { + return nil +} + +func makeLocalImportMetadataName(projectName, importSourceID string) string { + return strings.Join( + []string{ + fmt.Sprintf(LocalImportMetadata_ImageNameFormat, projectName), + fmt.Sprintf(LocalImportMetadata_TagFormat, importSourceID), + }, ":", + ) +} diff --git a/pkg/storage/manager/storage_manager.go b/pkg/storage/manager/storage_manager.go index 7de821a6f5..867f61aa48 100644 --- a/pkg/storage/manager/storage_manager.go +++ b/pkg/storage/manager/storage_manager.go @@ -321,7 +321,7 @@ func (m *StorageManager) ForEachDeleteFinalStage(ctx context.Context, options Fo } func (m *StorageManager) ForEachDeleteStage(ctx context.Context, options ForEachDeleteStageOptions, stagesDescriptions []*image.StageDescription, f func(ctx context.Context, stageDesc *image.StageDescription, err error) error) error { - if localStagesStorage, isLocal := m.StagesStorage.(*storage.DockerServerStagesStorage); isLocal { + if localStagesStorage, isLocal := m.StagesStorage.(*storage.LocalStagesStorage); isLocal { filteredStagesDescriptions, err := localStagesStorage.FilterStagesAndProcessRelatedData(ctx, stagesDescriptions, options.FilterStagesAndProcessRelatedDataOptions) if err != nil { return fmt.Errorf("error filtering local docker server stages: %w", err) diff --git a/pkg/storage/repo_stages_storage.go b/pkg/storage/repo_stages_storage.go index 6bd464d476..1577afca6a 100644 --- a/pkg/storage/repo_stages_storage.go +++ b/pkg/storage/repo_stages_storage.go @@ -486,9 +486,9 @@ func (storage *RepoStagesStorage) FetchImage(ctx context.Context, img container_ // FIXME(stapel-to-buildah): use ImageInterface instead of LegacyImageInterface // FIXME(stapel-to-buildah): possible optimization would be to push buildah container directly into registry wihtout committing a local image func (storage *RepoStagesStorage) StoreImage(ctx context.Context, img container_backend.LegacyImageInterface) error { - if img.GetBuiltID() != "" { - if err := storage.ContainerBackend.Tag(ctx, img.GetBuiltID(), img.Name(), container_backend.TagOpts{TargetPlatform: img.GetTargetPlatform()}); err != nil { - return fmt.Errorf("unable to tag built image %q by %q: %w", img.GetBuiltID(), img.Name(), err) + if img.BuiltID() != "" { + if err := storage.ContainerBackend.Tag(ctx, img.BuiltID(), img.Name(), container_backend.TagOpts{TargetPlatform: img.GetTargetPlatform()}); err != nil { + return fmt.Errorf("unable to tag built image %q by %q: %w", img.BuiltID(), img.Name(), err) } } @@ -628,9 +628,7 @@ func (storage *RepoStagesStorage) PutImportMetadata(ctx context.Context, project fullImageName := makeRepoImportMetadataName(storage.RepoAddress, metadata.ImportSourceID) logboek.Context(ctx).Debug().LogF("-- RepoStagesStorage.PutImportMetadata full image name: %s\n", fullImageName) - opts := &docker_registry.PushImageOptions{ - Labels: metadata.ToLabels(), - } + opts := &docker_registry.PushImageOptions{Labels: metadata.ToLabelsMap()} opts.Labels[image.WerfLabel] = projectName if err := storage.DockerRegistry.PushImage(ctx, fullImageName, opts); err != nil { @@ -893,3 +891,7 @@ func (storage *RepoStagesStorage) PostMultiplatformImage(ctx context.Context, pr return nil } + +func (storage *RepoStagesStorage) FilterStagesAndProcessRelatedData(ctx context.Context, stageDescriptions []*image.StageDescription, options FilterStagesAndProcessRelatedDataOptions) ([]*image.StageDescription, error) { + return stageDescriptions, nil +} diff --git a/pkg/storage/stages_storage.go b/pkg/storage/stages_storage.go index 4ff2039cbd..8b1ded335e 100644 --- a/pkg/storage/stages_storage.go +++ b/pkg/storage/stages_storage.go @@ -25,6 +25,12 @@ func IsErrBrokenImage(err error) bool { return err != nil && strings.HasSuffix(err.Error(), ErrBrokenImage.Error()) } +type FilterStagesAndProcessRelatedDataOptions struct { + SkipUsedImage bool + RmForce bool + RmContainersThatUseImage bool +} + type StagesStorage interface { GetStagesIDs(ctx context.Context, projectName string, opts ...Option) ([]image.StageID, error) GetStagesIDsByDigest(ctx context.Context, projectName, digest string, opts ...Option) ([]image.StageID, error) @@ -73,6 +79,7 @@ type StagesStorage interface { GetClientIDRecords(ctx context.Context, projectName string, opts ...Option) ([]*ClientIDRecord, error) PostClientIDRecord(ctx context.Context, projectName string, rec *ClientIDRecord) error PostMultiplatformImage(ctx context.Context, projectName, tag string, allPlatformsImages []*image.Info) error + FilterStagesAndProcessRelatedData(ctx context.Context, stageDescriptions []*image.StageDescription, options FilterStagesAndProcessRelatedDataOptions) ([]*image.StageDescription, error) String() string Address() string diff --git a/pkg/util/pair.go b/pkg/util/pair.go index d44a8835df..e849f4850f 100644 --- a/pkg/util/pair.go +++ b/pkg/util/pair.go @@ -7,14 +7,14 @@ type Pair[T1, T2 any] struct { Second T2 } -func NewPair[T1, T2 any](first T1, second T2) *Pair[T1, T2] { - return &Pair[T1, T2]{First: first, Second: second} +func NewPair[T1, T2 any](first T1, second T2) Pair[T1, T2] { + return Pair[T1, T2]{First: first, Second: second} } -func (p *Pair[T1, T2]) Unpair() (T1, T2) { +func (p Pair[T1, T2]) Unpair() (T1, T2) { return p.First, p.Second } -func (p *Pair[T1, T2]) String() string { +func (p Pair[T1, T2]) String() string { return fmt.Sprintf("[%v %v]", p.First, p.Second) } diff --git a/test/pkg/utils/storage.go b/test/pkg/utils/storage.go index 013f194c03..80dadeea9a 100644 --- a/test/pkg/utils/storage.go +++ b/test/pkg/utils/storage.go @@ -12,7 +12,7 @@ import ( func NewStagesStorage(stagesStorageAddress, implementationName string, dockerRegistryOptions docker_registry.DockerRegistryOptions) storage.PrimaryStagesStorage { if stagesStorageAddress == storage.LocalStorageAddress { - return storage.NewDockerServerStagesStorage(&container_backend.DockerServerBackend{}) + return storage.NewLocalStagesStorage(container_backend.NewDockerServerBackend()) } else { dockerRegistry, err := docker_registry.NewDockerRegistry(stagesStorageAddress, implementationName, dockerRegistryOptions) Expect(err).ShouldNot(HaveOccurred())