Skip to content

Commit

Permalink
fix(multiarch): publish git metadata for multiplatform mode images
Browse files Browse the repository at this point in the history 
Introduced MultiplatformImage descriptor to manage multiplatform images and maintain list of multiplatform images in the ImagesTree primitive.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Apr 6, 2023
1 parent e685e5e commit 1913c95
Show file tree
Hide file tree
Showing 3 changed files with 131 additions and 58 deletions.
110 changes: 52 additions & 58 deletions pkg/build/build_phase.go
View file
Expand Up @@ -245,7 +245,13 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string,
if !phase.BuildPhaseOptions.SkipImageMetadataPublication {
if err := logboek.Context(ctx).Info().
LogProcess(fmt.Sprintf("Publish image %s git metadata", img.GetName())).
DoError(func() error { return phase.publishImageGitMetadata(ctx, img) }); err != nil {
DoError(func() error {
return phase.publishImageGitMetadata(
ctx, img.GetName(),
img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.Repository,
*img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().StageID,
)
}); err != nil {
return err
}
}
Expand Down Expand Up @@ -286,72 +292,35 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, name string,
return nil
}

func calculateMuiltiplatformStageDigest(images []*image.Image) string {
metaStageDeps := util.MapFuncToSlice(images, func(img *image.Image) string {
stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription()
return stageDesc.StageID.String()
})
return util.Sha3_224Hash(metaStageDeps...)
}

func getImagesInfoList(images []*image.Image) []*imagePkg.Info {
return util.MapFuncToSlice(images, func(img *image.Image) *imagePkg.Info {
stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription()
return stageDesc.Info
})
}

func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context, name string, images []*image.Image) error {
if err := phase.addManagedImage(ctx, name); err != nil {
return err
}

digest := calculateMuiltiplatformStageDigest(images)
allPlatformsImages := getImagesInfoList(images)
_, uniqueID := phase.Conveyor.StorageManager.GenerateStageUniqueID(digest, nil)
metaStageID := imagePkg.StageID{Digest: digest, UniqueID: uniqueID}
opts := image.MultiplatformImageOptions{
IsArtifact: images[0].IsArtifact,
IsDockerfileImage: images[0].IsDockerfileImage,
IsDockerfileTargetStage: images[0].IsDockerfileTargetStage,
}

img := image.NewMultiplatformImage(name, images, phase.Conveyor.StorageManager, opts)
stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage()

// FIXME(multiarch): Copy manifest list into final stages storage.
// FIXME(multiarch): Git metadata for artifacts and stages per platform or

// if !phase.BuildPhaseOptions.SkipImageMetadataPublication {
// if err := logboek.Context(ctx).Info().
// LogProcess(fmt.Sprintf("Publish image %s git metadata", img.GetName())).
// DoError(func() error { return phase.publishImageGitMetadata(ctx, img) }); err != nil {
// return err
// }
// }

// if img.IsArtifact {
// return nil
// }
// if img.IsDockerfileImage && !img.IsDockerfileTargetStage {
// return nil
// }

// if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil {
// if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend, manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode}); err != nil {
// return err
// }
// }

if len(phase.CustomTagFuncList) == 0 {
platforms := util.MapFuncToSlice(images, func(img *image.Image) string { return img.TargetPlatform })

container_backend.LogImageName(ctx, fmt.Sprintf("%s:%s", stagesStorage.Address(), metaStageID))
container_backend.LogImageName(ctx, fmt.Sprintf("%s:%s", stagesStorage.Address(), img.GetStageID()))
container_backend.LogMultiplatformImageInfo(ctx, platforms)

if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), metaStageID.String(), allPlatformsImages); err != nil {
return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, metaStageID, err)
if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), img.GetStageID().String(), img.GetImagesInfoList()); err != nil {
return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, img.GetStageID(), err)
}
} else {
for _, tagFunc := range phase.CustomTagFuncList {
tag := tagFunc(name, metaStageID.String())
tag := tagFunc(name, img.GetStageID().String())

if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), tag, allPlatformsImages); err != nil {
return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, metaStageID, err)
if err := stagesStorage.PostMultiplatformImage(ctx, phase.Conveyor.ProjectName(), tag, img.GetImagesInfoList()); err != nil {
return fmt.Errorf("unable to post multiplatform image %s %s: %w", name, img.GetStageID(), err)
}

// FIXME(multiarch): custom tag registration for cleanup, shall we do it?
Expand All @@ -361,6 +330,33 @@ func (phase *BuildPhase) publishMultiplatformImageMetadata(ctx context.Context,
}
}

if !phase.BuildPhaseOptions.SkipImageMetadataPublication {
if err := logboek.Context(ctx).Info().
LogProcess(fmt.Sprintf("Publish multiarch image %s git metadata", name)).
DoError(func() error {
return phase.publishImageGitMetadata(ctx, name, stagesStorage.Address(), img.GetStageID())
}); err != nil {
return err
}
}

if img.IsArtifact {
return nil
}
if img.IsDockerfileImage && !img.IsDockerfileTargetStage {
return nil
}

if phase.Conveyor.StorageManager.GetFinalStagesStorage() != nil {
// FIXME(multiarch): copy manifest list into final stages storage with all dependant images
// if err := phase.Conveyor.StorageManager.CopyStageIntoFinalStorage(
// ctx, img.GetLastNonEmptyStage(), phase.Conveyor.ContainerBackend,
// manager.CopyStageIntoFinalStorageOptions{ShouldBeBuiltMode: phase.ShouldBeBuiltMode},
// ); err != nil {
// return err
// }
}

return nil
}

Expand Down Expand Up @@ -490,7 +486,7 @@ func (phase *BuildPhase) addManagedImage(ctx context.Context, name string) error
return nil
}

func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, img *image.Image) error {
func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, imageName, repository string, stageID imagePkg.StageID) error {
var commits []string

headCommit := phase.Conveyor.giterminismManager.HeadCommit()
Expand All @@ -507,22 +503,20 @@ func (phase *BuildPhase) publishImageGitMetadata(ctx context.Context, img *image

stagesStorage := phase.Conveyor.StorageManager.GetStagesStorage()

info := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info

logboek.Context(ctx).Info().LogF("name: %s:%s\n", info.Repository, info.Tag)
logboek.Context(ctx).Info().LogF("name: %s:%s\n", repository, stageID)
logboek.Context(ctx).Info().LogF("commits:\n")

for _, commit := range commits {
logboek.Context(ctx).Info().LogF(" %s\n", commit)

exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID(), storage.WithCache())
exist, err := stagesStorage.IsImageMetadataExist(ctx, phase.Conveyor.ProjectName(), imageName, commit, stageID.String(), storage.WithCache())
if err != nil {
return fmt.Errorf("unable to get image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err)
return fmt.Errorf("unable to get image %s metadata by commit %s and stage ID %s: %w", imageName, commit, stageID.String(), err)
}

if !exist {
if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.ProjectName(), img.GetName(), commit, img.GetStageID()); err != nil {
return fmt.Errorf("unable to put image %s metadata by commit %s and stage ID %s: %w", img.GetName(), commit, img.GetStageID(), err)
if err := stagesStorage.PutImageMetadata(ctx, phase.Conveyor.ProjectName(), imageName, commit, stageID.String()); err != nil {
return fmt.Errorf("unable to put image %s metadata by commit %s and stage ID %s: %w", imageName, commit, stageID.String(), err)
}
}
}
Expand Down
20 changes: 20 additions & 0 deletions pkg/build/image/image_tree.go
View file
Expand Up @@ -27,6 +27,8 @@ type ImagesTree struct {

allImages []*Image
imagesSets ImagesSets

multiplatformImages []*MultiplatformImage
}

type ImagesTreeOptions struct {
Expand Down Expand Up @@ -187,6 +189,24 @@ func (tree *ImagesTree) GetImagesSets() ImagesSets {
return tree.imagesSets
}

func (tree *ImagesTree) GetMultiplatformImage(name string) *MultiplatformImage {
for _, img := range tree.multiplatformImages {
if img.Name == name {
return img
}
}
return nil
}

func (tree *ImagesTree) SetMultiplatformImage(newImg *MultiplatformImage) {
for _, img := range tree.multiplatformImages {
if img.Name == newImg.Name {
return
}
}
tree.multiplatformImages = append(tree.multiplatformImages, newImg)
}

func getFromFields(imageBaseConfig *config.StapelImageBase) (string, string, bool) {
var from string
var fromImageName string
Expand Down
59 changes: 59 additions & 0 deletions pkg/build/image/multiplatform_image.go
View file
@@ -0,0 +1,59 @@
package image

import (
common_image "github.com/werf/werf/pkg/image"
"github.com/werf/werf/pkg/storage/manager"
"github.com/werf/werf/pkg/util"
)

type MultiplatformImage struct {
Name string
Images []*Image

MultiplatformImageOptions

calculatedDigest string
stageID common_image.StageID
}

type MultiplatformImageOptions struct {
IsArtifact, IsDockerfileImage, IsDockerfileTargetStage bool
}

func NewMultiplatformImage(name string, images []*Image, storageManager manager.StorageManagerInterface, opts MultiplatformImageOptions) *MultiplatformImage {
img := &MultiplatformImage{
Name: name,
Images: images,
MultiplatformImageOptions: opts,
}

metaStageDeps := util.MapFuncToSlice(images, func(img *Image) string {
stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription()
return stageDesc.StageID.String()
})
img.calculatedDigest = util.Sha3_224Hash(metaStageDeps...)

_, uniqueID := storageManager.GenerateStageUniqueID(img.GetDigest(), nil)
img.stageID = common_image.StageID{Digest: img.GetDigest(), UniqueID: uniqueID}

return img
}

func (img *MultiplatformImage) GetPlatforms() []string {
return util.MapFuncToSlice(img.Images, func(img *Image) string { return img.TargetPlatform })
}

func (img *MultiplatformImage) GetDigest() string {
return img.calculatedDigest
}

func (img *MultiplatformImage) GetStageID() common_image.StageID {
return img.stageID
}

func (img *MultiplatformImage) GetImagesInfoList() []*common_image.Info {
return util.MapFuncToSlice(img.Images, func(img *Image) *common_image.Info {
stageDesc := img.GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription()
return stageDesc.Info
})
}

0 comments on commit 1913c95

Please sign in to comment.