diff --git a/cmd/werf/common/container_backend.go b/cmd/werf/common/container_backend.go index 1789ee67b7..c60bc70a54 100644 --- a/cmd/werf/common/container_backend.go +++ b/cmd/werf/common/container_backend.go @@ -135,7 +135,6 @@ func InitProcessContainerBackend(ctx context.Context, cmdData *CmdData) (contain func InitProcessDocker(ctx context.Context, cmdData *CmdData) (context.Context, error) { opts := docker.InitOptions{ DockerConfigDir: *cmdData.DockerConfig, - ClaimPlatforms: cmdData.GetPlatform(), Verbose: *cmdData.LogVerbose, Debug: *cmdData.LogDebug, } diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index aab1a5b97e..8f919ee996 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -215,21 +215,49 @@ func (phase *BuildPhase) AfterImages(ctx context.Context) error { } if len(targetPlatforms) == 1 { - if err := phase.publishImageMetadata(ctx, name, images[0]); err != nil { - return fmt.Errorf("unable to publish image %q metadata: %w", name, err) + img := images[0] + + if img.IsFinal() { + if err := phase.publishFinalImage(ctx, name, img); err != nil { + return err + } + } + + // TODO: Separate LocalStagesStorage and RepoStagesStorage interfaces, local should not include metadata publishing methods at all + if _, isLocal := phase.Conveyor.StorageManager.GetStagesStorage().(*storage.LocalStagesStorage); !isLocal { + if err := phase.publishImageMetadata(ctx, name, img); err != nil { + return fmt.Errorf("unable to publish image %q metadata: %w", name, err) + } } } else { - if err := logboek.Context(ctx).LogProcess(logging.ImageLogProcessName(name, false, "")). - Options(func(options types.LogProcessOptionsInterface) { - options.Style(logging.ImageMetadataStyle()) - }). - DoError(func() error { - if err := phase.publishMultiplatformImageMetadata(ctx, name, images); err != nil { - return fmt.Errorf("unable to publish image %q multiplatform metadata: %w", name, err) - } - return nil - }); err != nil { - return err + opts := image.MultiplatformImageOptions{ + IsArtifact: images[0].IsArtifact, + IsDockerfileImage: images[0].IsDockerfileImage, + IsDockerfileTargetStage: images[0].IsDockerfileTargetStage, + } + img := image.NewMultiplatformImage(name, images, phase.Conveyor.StorageManager, opts) + phase.Conveyor.imagesTree.SetMultiplatformImage(img) + + if img.IsFinal() { + if err := phase.publishMultiplatformFinalImage(ctx, name, images); err != nil { + return err + } + } + + // TODO: Separate LocalStagesStorage and RepoStagesStorage interfaces, local should not include metadata publishing methods at all + if _, isLocal := phase.Conveyor.StorageManager.GetStagesStorage().(*storage.LocalStagesStorage); !isLocal { + if err := logboek.Context(ctx).LogProcess(logging.ImageLogProcessName(name, false, "")). + Options(func(options types.LogProcessOptionsInterface) { + options.Style(logging.ImageMetadataStyle()) + }). + DoError(func() error { + if err := phase.publishMultiplatformImageMetadata(ctx, name, img); err != nil { + return fmt.Errorf("unable to publish image %q multiplatform metadata: %w", name, err) + } + return nil + }); err != nil { + return err + } } } } @@ -237,6 +265,34 @@ func (phase *BuildPhase) AfterImages(ctx context.Context) error { return phase.createReport(ctx) } +func (phase *BuildPhase) publishFinalImage(ctx context.Context, name string, img *image.Image) error { + if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { + if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}); err != nil { + return err + } + } + return nil +} + +func (phase *BuildPhase) publishMultiplatformFinalImage(ctx context.Context, name string, images []*image.Image) error { + // FIXME(multiarch): copy manifest list into final stages storage with all dependant images + if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { + // if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage( + // ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, + // manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}, + // ); err != nil { + // return err + // } + + // desc, err := phase.Conveyor.StorageManager.GetFinalStagesStorage().GetStageDescription(ctx, phase.Conveyor.ProjectName(), img.GetStageID().Digest, img.GetStageID().UniqueID) + // if err != nil { + // return fmt.Errorf("unable to get image %s %s descriptor: %w", name, img.GetStageID(), err) + // } + // img.SetFinalStageDescription(desc) + } + return nil +} + func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string, img *image.Image) error { if err := phase.addManagedImage(ctx, name); err != nil { return err @@ -248,7 +304,6 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string, DoError(func() error { return phase.publishImageGitMetadata( ctx, img.GetName(), - img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.Repository, *img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().StageID, ) }); err != nil { @@ -256,19 +311,10 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string, } } - if img.IsArtifact { - return nil - } - if img.IsDockerfileImage && !img.IsDockerfileTargetStage { + if !img.IsFinal() { return nil } - if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { - if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}); err != nil { - return err - } - } - var customTagStorage storage.StagesStorage var customTagStage *imagePkg.StageDescription if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { @@ -292,26 +338,18 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string, return nil } -func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context, name string, images []*image.Image) error { +func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context, name string, img *image.MultiplatformImage) error { if err := phase.addManagedImage(ctx, name); err != nil { return err } - opts := image.MultiplatformImageOptions{ - IsArtifact: images[0].IsArtifact, - IsDockerfileImage: images[0].IsDockerfileImage, - IsDockerfileTargetStage: images[0].IsDockerfileTargetStage, - } - - img := image.NewMultiplatformImage(name, images, phase.Conveyor.StorageManager, opts) - phase.Conveyor.imagesTree.SetMultiplatformImage(img) - stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage() if len(phase.CustomTagFuncList) == 0 { - platforms := util.MapFuncToSlice(images, func(img *image.Image) string { return img.TargetPlatform }) + fullImageName := stagesStorage.ConstructStageImageName(phase.Conveyor.ProjectName(), img.GetStageID().Digest, img.GetStageID().UniqueID) + platforms := img.GetPlatforms() - container_backend.LogImageName(ctx, fmt.Sprintf("%s:%s", stagesStorage.Address(), img.GetStageID())) + container_backend.LogImageName(ctx, fullImageName) container_backend.LogMultiplatformImageInfo(ctx, platforms) if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), img.GetStageID().String(), img.GetImagesInfoList()); err != nil { @@ -346,35 +384,11 @@ func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context, if err := logboek.Context(ctx).Info(). LogProcess(fmt.Sprintf("Publish multiarch image %s git metadata", name)). DoError(func() error { - return phase.publishImageGitMetadata(ctx, name, stagesStorage.Address(), img.GetStageID()) + return phase.publishImageGitMetadata(ctx, name, img.GetStageID()) }); err != nil { return err } } - - if img.IsArtifact { - return nil - } - if img.IsDockerfileImage && !img.IsDockerfileTargetStage { - return nil - } - - if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil { - // FIXME(multiarch): copy manifest list into final stages storage with all dependant images - // if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage( - // ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, - // manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}, - // ); err != nil { - // return err - // } - - // desc, err := phase.Conveyor.StorageManager.GetFinalStagesStorage().GetStageDescription(ctx, phase.Conveyor.ProjectName(), img.GetStageID().Digest, img.GetStageID().UniqueID) - // if err != nil { - // return fmt.Errorf("unable to get image %s %s descriptor: %w", name, img.GetStageID(), err) - // } - // img.SetFinalStageDescription(desc) - } - return nil } @@ -388,10 +402,7 @@ func (phase *BuildPhase) createReport(ctx context.Context) error { } for _, img := range phase.Conveyor.imagesTree.GetImages() { - if img.IsArtifact { - continue - } - if img.IsDockerfileImage && !img.IsDockerfileTargetStage { + if !img.IsFinal() { continue } @@ -419,36 +430,34 @@ func (phase *BuildPhase) createReport(ctx context.Context) error { } } - if len(targetPlatforms) > 1 { - for _, img := range phase.Conveyor.imagesTree.GetMultiplatformImages() { - if img.IsArtifact { - continue - } - if img.IsDockerfileImage && !img.IsDockerfileTargetStage { - continue - } + if _, isLocal := phase.Conveyor.StorageManager.GetStagesStorage().(*storage.LocalStagesStorage); !isLocal { + if len(targetPlatforms) > 1 { + for _, img := range phase.Conveyor.imagesTree.GetMultiplatformImages() { + if !img.IsFinal() { + continue + } - isRebuilt := false - for _, pImg := range img.Images { - isRebuilt = (isRebuilt || pImg.GetRebuilt()) - } + isRebuilt := false + for _, pImg := range img.Images { + isRebuilt = (isRebuilt || pImg.GetRebuilt()) + } - desc := img.GetFinalStageDescription() - if desc == nil { - desc = img.GetStageDescription() - } + desc := img.GetFinalStageDescription() + if desc == nil { + desc = img.GetStageDescription() + } - record := ReportImageRecord{ - WerfImageName: img.Name, - DockerRepo: desc.Info.Repository, - DockerTag: desc.Info.Tag, - DockerImageID: desc.Info.ID, - DockerImageDigest: desc.Info.RepoDigest, - DockerImageName: desc.Info.Name, - Rebuilt: isRebuilt, + record := ReportImageRecord{ + WerfImageName: img.Name, + DockerRepo: desc.Info.Repository, + DockerTag: desc.Info.Tag, + DockerImageID: desc.Info.ID, + DockerImageDigest: desc.Info.RepoDigest, + DockerImageName: desc.Info.Name, + Rebuilt: isRebuilt, + } + phase.ImagesReport.SetImageRecord(img.Name, record) } - - phase.ImagesReport.SetImageRecord(img.Name, record) } } @@ -537,7 +546,7 @@ func (phase *BuildPhase) addManagedImage(ctx context.Context, name string) error return nil } -func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, imageName, repository string, stageID imagePkg.StageID) error { +func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, imageName string, stageID imagePkg.StageID) error { var commits []string headCommit := phase.Conveyor.giterminismManager.HeadCommit() @@ -554,7 +563,8 @@ func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, imageName, stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage() - logboek.Context(ctx).Info().LogF("name: %s:%s\n", repository, stageID) + fullImageName := stagesStorage.ConstructStageImageName(phase.Conveyor.ProjectName(), stageID.Digest, stageID.UniqueID) + logboek.Context(ctx).Info().LogF("name: %s\n", fullImageName) logboek.Context(ctx).Info().LogF("commits:\n") for _, commit := range commits { diff --git a/pkg/build/conveyor.go b/pkg/build/conveyor.go index 103baddd94..fa511b95ec 100644 --- a/pkg/build/conveyor.go +++ b/pkg/build/conveyor.go @@ -402,10 +402,7 @@ func (c *Conveyor) GetImageInfoGetters(opts imagePkg.InfoGetterOptions) ([]*imag } } else { for _, img := range c.imagesTree.GetMultiplatformImages() { - if img.IsArtifact { - continue - } - if img.IsDockerfileImage && !img.IsDockerfileTargetStage { + if !img.IsFinal() { continue } @@ -445,7 +442,13 @@ func (c *Conveyor) GetImagesEnvArray() []string { return envArray } -func (c *Conveyor) checkContainerBackendSupported(_ context.Context) error { +func (c *Conveyor) checkContainerBackendSupported(ctx context.Context) error { + targetPlatforms, err := c.GetTargetPlatforms() + if err != nil { + return fmt.Errorf("error getting target platforms: %w", err) + } + c.ContainerBackend.ClaimTargetPlatforms(ctx, targetPlatforms) + if _, isBuildah := c.ContainerBackend.(*container_backend.BuildahBackend); !isBuildah { return nil } diff --git a/pkg/build/image/image.go b/pkg/build/image/image.go index a9a2971fc5..0f0260d064 100644 --- a/pkg/build/image/image.go +++ b/pkg/build/image/image.go @@ -145,6 +145,16 @@ func ImageLogTagStyle(isArtifact bool) color.Style { return logging.ImageDefaultStyle(isArtifact) } +func (i *Image) IsFinal() bool { + if i.IsArtifact { + return false + } + if i.IsDockerfileImage && !i.IsDockerfileTargetStage { + return false + } + return true +} + func (i *Image) SetStages(stages []stage.Interface) { i.stages = stages } diff --git a/pkg/build/image/multiplatform_image.go b/pkg/build/image/multiplatform_image.go index dc0c946276..9d3bf05a36 100644 --- a/pkg/build/image/multiplatform_image.go +++ b/pkg/build/image/multiplatform_image.go @@ -74,3 +74,13 @@ func (img *MultiplatformImage) GetStageDescription() *image.StageDescription { func (img *MultiplatformImage) SetStageDescription(desc *common_image.StageDescription) { img.stageDescription = desc } + +func (img *MultiplatformImage) IsFinal() bool { + if img.IsArtifact { + return false + } + if img.IsDockerfileImage && !img.IsDockerfileTargetStage { + return false + } + return true +} diff --git a/pkg/container_backend/buildah_backend.go b/pkg/container_backend/buildah_backend.go index 888ef5ac86..3e34764d00 100644 --- a/pkg/container_backend/buildah_backend.go +++ b/pkg/container_backend/buildah_backend.go @@ -1005,3 +1005,5 @@ func (runtime *BuildahBackend) Rm(ctx context.Context, name string, opts RmOpts) func (runtime *BuildahBackend) PostManifest(ctx context.Context, ref string, opts PostManifestOpts) error { return fmt.Errorf("not implemented") } + +func (runtime *BuildahBackend) ClaimTargetPlatforms(ctx context.Context, targetPlatforms []string) {} diff --git a/pkg/container_backend/docker_server_backend.go b/pkg/container_backend/docker_server_backend.go index 99d4c491ab..129081ad6b 100644 --- a/pkg/container_backend/docker_server_backend.go +++ b/pkg/container_backend/docker_server_backend.go @@ -23,6 +23,10 @@ func NewDockerServerBackend() *DockerServerBackend { return &DockerServerBackend{} } +func (runtime *DockerServerBackend) ClaimTargetPlatforms(ctx context.Context, targetPlatforms []string) { + docker.ClaimTargetPlatforms(targetPlatforms) +} + func (runtime *DockerServerBackend) GetDefaultPlatform() string { return docker.GetDefaultPlatform() } @@ -336,5 +340,5 @@ func (runtime *DockerServerBackend) Containers(ctx context.Context, opts Contain } func (runtime *DockerServerBackend) PostManifest(ctx context.Context, ref string, opts PostManifestOpts) error { - return docker.CreateImage(ctx, ref, opts.Labels) + return docker.CreateImage(ctx, ref, docker.CreateImageOptions{Labels: opts.Labels}) } diff --git a/pkg/container_backend/interface.go b/pkg/container_backend/interface.go index efa62b1239..b9b4e0813e 100644 --- a/pkg/container_backend/interface.go +++ b/pkg/container_backend/interface.go @@ -91,6 +91,8 @@ type ContainerBackend interface { Images(ctx context.Context, opts ImagesOptions) (image.ImagesList, error) Containers(ctx context.Context, opts ContainersOptions) (image.ContainerList, error) + ClaimTargetPlatforms(ctx context.Context, targetPlatforms []string) + String() string // TODO: Util method for cleanup, which possibly should be avoided in the future diff --git a/pkg/container_backend/perf_check_container_backend.go b/pkg/container_backend/perf_check_container_backend.go index 32772c3129..b9560be52f 100644 --- a/pkg/container_backend/perf_check_container_backend.go +++ b/pkg/container_backend/perf_check_container_backend.go @@ -186,3 +186,8 @@ func (runtime *PerfCheckContainerBackend) TagImageByName(ctx context.Context, im }) return } + +func (runtime *PerfCheckContainerBackend) ClaimTargetPlatforms(ctx context.Context, targetPlatforms []string) { + logboek.Context(ctx).Default().LogProcess("ContainerBackend.ClaimTargetPlatforms %v", targetPlatforms). + Do(func() { runtime.ContainerBackend.ClaimTargetPlatforms(ctx, targetPlatforms) }) +} diff --git a/pkg/docker/image.go b/pkg/docker/image.go index 937a5ad557..1eda7936fc 100644 --- a/pkg/docker/image.go +++ b/pkg/docker/image.go @@ -20,18 +20,20 @@ import ( parallelConstant "github.com/werf/werf/pkg/util/parallel/constant" ) -func CreateImage(ctx context.Context, ref string, labels []string) error { - var opts types.ImageImportOptions +type CreateImageOptions struct { + Labels []string +} - if len(labels) > 0 { +func CreateImage(ctx context.Context, ref string, opts CreateImageOptions) error { + var importOpts types.ImageImportOptions + if len(opts.Labels) > 0 { changeOption := "LABEL" - for _, label := range labels { + for _, label := range opts.Labels { changeOption += fmt.Sprintf(" %s", label) } - opts.Changes = append(opts.Changes, changeOption) + importOpts.Changes = append(importOpts.Changes, changeOption) } - - _, err := apiCli(ctx).ImageImport(ctx, types.ImageImportSource{SourceName: "-"}, ref, opts) + _, err := apiCli(ctx).ImageImport(ctx, types.ImageImportSource{SourceName: "-"}, ref, importOpts) return err } diff --git a/pkg/docker/main.go b/pkg/docker/main.go index 6dca6d0325..cb86b652d1 100644 --- a/pkg/docker/main.go +++ b/pkg/docker/main.go @@ -98,6 +98,18 @@ func Init(ctx context.Context, opts InitOptions) error { return nil } +func ClaimTargetPlatforms(claimPlatforms []string) { + if defaultPlatform != "" { + claimPlatforms = append(claimPlatforms, defaultPlatform) + } + for _, claimPlatform := range claimPlatforms { + if claimPlatform != runtimePlatform { + useBuildx = true + break + } + } +} + func GetDefaultPlatform() string { return defaultPlatform } diff --git a/pkg/docker/manifest.go b/pkg/docker/manifest.go new file mode 100644 index 0000000000..6274c6230f --- /dev/null +++ b/pkg/docker/manifest.go @@ -0,0 +1,17 @@ +package docker + +import ( + "github.com/docker/cli/cli/command" + "github.com/docker/cli/cli/command/manifest" + "golang.org/x/net/context" +) + +func doCliManifest(c command.Cli, args ...string) error { + return prepareCliCmd(manifest.NewManifestCommand(c), args...).Execute() +} + +func CliManifest(ctx context.Context, args ...string) error { + return callCliWithAutoOutput(ctx, func(c command.Cli) error { + return doCliManifest(c, args...) + }) +} diff --git a/pkg/storage/local_stages_storage.go b/pkg/storage/local_stages_storage.go index 5ea3f87f41..c9e55aee90 100644 --- a/pkg/storage/local_stages_storage.go +++ b/pkg/storage/local_stages_storage.go @@ -385,22 +385,6 @@ func (storage *LocalStagesStorage) PostClientIDRecord(ctx context.Context, proje } func (storage *LocalStagesStorage) PostMultiplatformImage(ctx context.Context, projectName, tag string, allPlatformsImages []*image.Info) error { - logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.PostMultiplatformImage by tag %s for project %s\n", tag, projectName) - - fullImageName := fmt.Sprintf("%s:%s", projectName, tag) - postManifestOpts := container_backend.PostManifestOpts{ - Labels: []string{fmt.Sprintf("%s=%s", image.WerfLabel, projectName)}, - Manifests: allPlatformsImages, - } - - logboek.Context(ctx).Debug().LogF("-- LocalStagesStorage.PostMultiplatformImage full image name: %s\n", fullImageName) - - if err := storage.ContainerBackend.PostManifest(ctx, fullImageName, postManifestOpts); err != nil { - return fmt.Errorf("unable to post manifest %q: %w", fullImageName, err) - } - - logboek.Context(ctx).Info().LogF("Created manifest list %s for project %s\n", fullImageName, projectName) - return nil }