Skip to content

Commit

Permalink
fix(multiarch): fix werf converge, kube-run, run and other commands w…
Browse files Browse the repository at this point in the history 
…hen platform param specified

Conveyor images storage now can query image by name and target platform.

Signed-off-by: Timofey Kirillov <timofey.kirillov@flant.com>
  • Loading branch information
@distorhead
distorhead committed Mar 13, 2023
1 parent c329a16 commit 3e2add1
Show file tree
Hide file tree
Showing 23 changed files with 158 additions and 117 deletions.
5 changes: 4 additions & 1 deletion cmd/werf/bundle/export/export.go
View file
Expand Up @@ -311,7 +311,10 @@ func runExport(ctx context.Context, imagesToProcess build.ImagesToProcess) error
}
}

imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
5 changes: 4 additions & 1 deletion cmd/werf/bundle/publish/publish.go
View file
Expand Up @@ -317,7 +317,10 @@ func runPublish(ctx context.Context, imagesToProcess build.ImagesToProcess) erro
}
}

imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/werf/compose/main.go
View file
Expand Up @@ -418,8 +418,8 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken
}
}

for _, imageName := range c.GetExportedImagesNames() {
if err := c.FetchLastImageStage(ctx, imageName); err != nil {
for _, img := range c.GetExportedImages() {
if err := c.FetchLastImageStage(ctx, img.TargetPlatform, img.Name); err != nil {
return err
}
}
Expand Down
5 changes: 4 additions & 1 deletion cmd/werf/converge/converge.go
View file
Expand Up @@ -359,7 +359,10 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken
}
}

imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
5 changes: 4 additions & 1 deletion cmd/werf/helm/get_autogenerated_values.go
View file
Expand Up @@ -227,7 +227,10 @@ func runGetServiceValues(ctx context.Context, imagesToProcess build.ImagesToProc
}

imagesRepository = storageManager.StagesStorage.String()
imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
6 changes: 4 additions & 2 deletions cmd/werf/kube_run/kube_run.go
View file
Expand Up @@ -416,11 +416,13 @@ func run(ctx context.Context, pod, secret, namespace string, werfConfig *config.
}
}

if err := c.FetchLastImageStage(ctx, imageName); err != nil {
// FIXME(multiarch): specify multiarch manifest here
if err := c.FetchLastImageStage(ctx, "", imageName); err != nil {
return err
}

image = c.GetImageNameForLastImageStage(imageName)
// FIXME(multiarch): specify multiarch manifest here
image = c.GetImageNameForLastImageStage("", imageName)
return nil
}); err != nil {
return err
Expand Down
5 changes: 4 additions & 1 deletion cmd/werf/render/render.go
View file
Expand Up @@ -348,7 +348,10 @@ func runRender(ctx context.Context, imagesToProcess build.ImagesToProcess) error
}
}

imagesInfoGetters = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
imagesInfoGetters, err = c.GetImageInfoGetters(image.InfoGetterOptions{CustomTagFunc: useCustomTagFunc})
if err != nil {
return err
}

return nil
}); err != nil {
Expand Down
4 changes: 2 additions & 2 deletions cmd/werf/run/run.go
View file
Expand Up @@ -402,11 +402,11 @@ func run(ctx context.Context, containerBackend container_backend.ContainerBacken
}
}

if err := c.FetchLastImageStage(ctx, imageName); err != nil {
if err := c.FetchLastImageStage(ctx, "", imageName); err != nil {
return err
}

dockerImageName = c.GetImageNameForLastImageStage(imageName)
dockerImageName = c.GetImageNameForLastImageStage("", imageName)
return nil
}); err != nil {
return err
Expand Down
3 changes: 2 additions & 1 deletion cmd/werf/stage/image/main.go
View file
Expand Up @@ -207,7 +207,8 @@ func run(ctx context.Context, imageName string) error {
return err
}

fmt.Println(c.GetImageNameForLastImageStage(imageName))
// FIXME(multiarch): specify multiarch manifest here
fmt.Println(c.GetImageNameForLastImageStage("", imageName))

return nil
}); err != nil {
Expand Down
105 changes: 59 additions & 46 deletions pkg/build/conveyor.go
View file
Expand Up @@ -215,14 +215,18 @@ func (c *Conveyor) GetLocalGitRepoVirtualMergeOptions() stage.VirtualMergeOption
return c.ConveyorOptions.LocalGitRepoVirtualMergeOptions
}

func (c *Conveyor) GetImportServer(ctx context.Context, imageName, stageName string) (import_server.ImportServer, error) {
func (c *Conveyor) GetImportServer(ctx context.Context, targetPlatform, imageName, stageName string) (import_server.ImportServer, error) {
c.GetServiceRWMutex("ImportServer").Lock()
defer c.GetServiceRWMutex("ImportServer").Unlock()

importServerName := imageName
importServerName := fmt.Sprintf("%s")
if stageName != "" {
importServerName += "/" + stageName
}
// FIXME(multiarch): in this place we should get our current platform from the container backend in the case when targetPlatform is empty
if targetPlatform != "" && targetPlatform != "linux/amd64" {
importServerName += "[" + targetPlatform + "]"
}
if srv, hasKey := c.importServers[importServerName]; hasKey {
return srv, nil
}
Expand All @@ -232,9 +236,9 @@ func (c *Conveyor) GetImportServer(ctx context.Context, imageName, stageName str
var stg stage.Interface

if stageName != "" {
stg = c.getImageStage(imageName, stageName)
stg = c.getImageStage(targetPlatform, imageName, stageName)
} else {
stg = c.GetImage(imageName).GetLastNonEmptyStage()
stg = c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage()
}

if err := c.StorageManager.FetchStage(ctx, c.ContainerBackend, stg); err != nil {
Expand All @@ -256,9 +260,9 @@ func (c *Conveyor) GetImportServer(ctx context.Context, imageName, stageName str

var dockerImageName string
if stageName == "" {
dockerImageName = c.GetImageNameForLastImageStage(imageName)
dockerImageName = c.GetImageNameForLastImageStage(targetPlatform, imageName)
} else {
dockerImageName = c.GetImageNameForImageStage(imageName, stageName)
dockerImageName = c.GetImageNameForImageStage(targetPlatform, imageName, stageName)
}

var err error
Expand Down Expand Up @@ -356,36 +360,45 @@ func (c *Conveyor) ShouldBeBuilt(ctx context.Context, opts ShouldBeBuiltOptions)
return nil
}

func (c *Conveyor) FetchLastImageStage(ctx context.Context, imageName string) error {
lastImageStage := c.GetImage(imageName).GetLastNonEmptyStage()
func (c *Conveyor) FetchLastImageStage(ctx context.Context, targetPlatform, imageName string) error {
lastImageStage := c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage()
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, lastImageStage)
}

func (c *Conveyor) GetImageInfoGetters(opts imagePkg.InfoGetterOptions) (images []*imagePkg.InfoGetter) {
func (c *Conveyor) GetImageInfoGetters(opts imagePkg.InfoGetterOptions) ([]*imagePkg.InfoGetter, error) {
targetPlatforms, err := c.GetTargetPlatforms()
if err != nil {
return nil, fmt.Errorf("unable to get target platforms: %w", err)
}
var targetPlatform string
if len(targetPlatforms) > 0 {
// FIXME(multiarch): instead of using first specified platform use multiarch-manifest
targetPlatform = targetPlatforms[0]
}

var images []*imagePkg.InfoGetter
for _, img := range c.imagesTree.GetImages() {
if img.IsArtifact {
continue
}
if img.TargetPlatform != targetPlatform {
continue
}

getter := c.StorageManager.GetImageInfoGetter(img.Name, img.GetLastNonEmptyStage(), opts)
images = append(images, getter)
}

return
return images, nil
}

func (c *Conveyor) GetExportedImagesNames() []string {
var res []string

func (c *Conveyor) GetExportedImages() (res []*image.Image) {
for _, img := range c.imagesTree.GetImages() {
if img.IsArtifact {
continue
}

res = append(res, img.Name)
res = append(res, img)
}

return res
return
}

func (c *Conveyor) GetImagesEnvArray() []string {
Expand All @@ -395,7 +408,7 @@ func (c *Conveyor) GetImagesEnvArray() []string {
continue
}

envArray = append(envArray, generateImageEnv(img.Name, c.GetImageNameForLastImageStage(img.Name)))
envArray = append(envArray, generateImageEnv(img.Name, c.GetImageNameForLastImageStage(img.TargetPlatform, img.Name)))
}

return envArray
Expand Down Expand Up @@ -687,67 +700,67 @@ func (c *Conveyor) GetOrCreateStageImage(name string, prevStageImage *stage.Stag
return stageImage
}

func (c *Conveyor) GetImage(name string) *image.Image {
func (c *Conveyor) GetImage(targetPlatform, name string) *image.Image {
for _, img := range c.imagesTree.GetImages() {
if img.GetName() == name {
if img.GetName() == name && img.TargetPlatform == targetPlatform {
return img
}
}

panic(fmt.Sprintf("Image %q not found!", name))
}

func (c *Conveyor) GetImageStageContentDigest(imageName, stageName string) string {
return c.getImageStage(imageName, stageName).GetContentDigest()
func (c *Conveyor) GetImageStageContentDigest(targetPlatform, imageName, stageName string) string {
return c.getImageStage(targetPlatform, imageName, stageName).GetContentDigest()
}

func (c *Conveyor) GetImageContentDigest(imageName string) string {
return c.GetImage(imageName).GetContentDigest()
func (c *Conveyor) GetImageContentDigest(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetContentDigest()
}

func (c *Conveyor) getImageStage(imageName, stageName string) stage.Interface {
if stg := c.GetImage(imageName).GetStage(stage.StageName(stageName)); stg != nil {
func (c *Conveyor) getImageStage(targetPlatform, imageName, stageName string) stage.Interface {
if stg := c.GetImage(targetPlatform, imageName).GetStage(stage.StageName(stageName)); stg != nil {
return stg
} else {
return c.getLastNonEmptyImageStage(imageName)
return c.getLastNonEmptyImageStage(targetPlatform, imageName)
}
}

func (c *Conveyor) getLastNonEmptyImageStage(imageName string) stage.Interface {
func (c *Conveyor) getLastNonEmptyImageStage(targetPlatform, imageName string) stage.Interface {
// FIXME: find first existing stage after specified unexisting
return c.GetImage(imageName).GetLastNonEmptyStage()
return c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage()
}

func (c *Conveyor) FetchImageStage(ctx context.Context, imageName, stageName string) error {
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, c.getImageStage(imageName, stageName))
func (c *Conveyor) FetchImageStage(ctx context.Context, targetPlatform, imageName, stageName string) error {
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, c.getImageStage(targetPlatform, imageName, stageName))
}

func (c *Conveyor) FetchLastNonEmptyImageStage(ctx context.Context, imageName string) error {
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, c.getLastNonEmptyImageStage(imageName))
func (c *Conveyor) FetchLastNonEmptyImageStage(ctx context.Context, targetPlatform, imageName string) error {
return c.StorageManager.FetchStage(ctx, c.ContainerBackend, c.getLastNonEmptyImageStage(targetPlatform, imageName))
}

func (c *Conveyor) GetImageNameForLastImageStage(imageName string) string {
return c.GetImage(imageName).GetLastNonEmptyStage().GetStageImage().Image.Name()
func (c *Conveyor) GetImageNameForLastImageStage(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage().GetStageImage().Image.Name()
}

func (c *Conveyor) GetImageNameForImageStage(imageName, stageName string) string {
return c.getImageStage(imageName, stageName).GetStageImage().Image.Name()
func (c *Conveyor) GetImageNameForImageStage(targetPlatform, imageName, stageName string) string {
return c.getImageStage(targetPlatform, imageName, stageName).GetStageImage().Image.Name()
}

func (c *Conveyor) GetStageID(imageName string) string {
return c.GetImage(imageName).GetStageID()
func (c *Conveyor) GetStageID(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetStageID()
}

func (c *Conveyor) GetImageIDForLastImageStage(imageName string) string {
return c.GetImage(imageName).GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.ID
func (c *Conveyor) GetImageIDForLastImageStage(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.ID
}

func (c *Conveyor) GetImageDigestForLastImageStage(imageName string) string {
return c.GetImage(imageName).GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.RepoDigest
func (c *Conveyor) GetImageDigestForLastImageStage(targetPlatform, imageName string) string {
return c.GetImage(targetPlatform, imageName).GetLastNonEmptyStage().GetStageImage().Image.GetStageDescription().Info.RepoDigest
}

func (c *Conveyor) GetImageIDForImageStage(imageName, stageName string) string {
return c.getImageStage(imageName, stageName).GetStageImage().Image.GetStageDescription().Info.ID
func (c *Conveyor) GetImageIDForImageStage(targetPlatform, imageName, stageName string) string {
return c.getImageStage(targetPlatform, imageName, stageName).GetStageImage().Image.GetStageDescription().Info.ID
}

func (c *Conveyor) GetImportMetadata(ctx context.Context, projectName, id string) (*storage.ImportMetadata, error) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/build/image/conveyor.go
View file
Expand Up @@ -10,7 +10,7 @@ import (
type Conveyor interface {
stage.Conveyor

GetImage(name string) *Image
GetImage(targetPlatform, name string) *Image
GetOrCreateStageImage(name string, prevStageImage *stage.StageImage, stg stage.Interface, img *Image) *stage.StageImage
GetTargetPlatforms() ([]string, error)

Expand Down
6 changes: 4 additions & 2 deletions pkg/build/image/dockerfile.go
View file
Expand Up @@ -137,6 +137,7 @@ func mapDockerfileToImagesSets(ctx context.Context, cfg *dockerfile.Dockerfile,
stageLogName := fmt.Sprintf("%s%d", strings.ToUpper(instr.GetInstructionData().Name()), ind+1)
isFirstStage := (len(img.stages) == 0)
baseStageOptions := &stage.BaseStageOptions{
TargetPlatform: img.TargetPlatform,
ImageName: img.Name,
LogName: stageLogName,
ImageTmpDir: img.TmpDir,
Expand Down Expand Up @@ -257,8 +258,9 @@ func mapLegacyDockerfileToImage(ctx context.Context, dockerfileImageConfig *conf
)

baseStageOptions := &stage.BaseStageOptions{
ImageName: dockerfileImageConfig.Name,
ProjectName: opts.ProjectName,
TargetPlatform: opts.TargetPlatform,
ImageName: dockerfileImageConfig.Name,
ProjectName: opts.ProjectName,
}

dockerfileStage := stage.GenerateFullDockerfileStage(
Expand Down
4 changes: 2 additions & 2 deletions pkg/build/image/image.go
View file
Expand Up @@ -223,13 +223,13 @@ func isUnsupportedMediaTypeError(err error) bool {
func (i *Image) SetupBaseImage(ctx context.Context, storageManager manager.StorageManagerInterface, storageOpts manager.StorageOptions) error {
switch i.baseImageType {
case StageAsBaseImage:
i.stageAsBaseImage = i.Conveyor.GetImage(i.baseImageName).GetLastNonEmptyStage()
i.stageAsBaseImage = i.Conveyor.GetImage(i.TargetPlatform, i.baseImageName).GetLastNonEmptyStage()
i.baseImageReference = i.stageAsBaseImage.GetStageImage().Image.Name()
i.baseStageImage = i.stageAsBaseImage.GetStageImage()

case ImageFromRegistryAsBaseImage:
if i.IsDockerfileImage && i.dockerfileExpanderFactory != nil {
dependenciesArgs := stage.ResolveDependenciesArgs(i.DockerfileImageConfig.Dependencies, i.Conveyor)
dependenciesArgs := stage.ResolveDependenciesArgs(i.TargetPlatform, i.DockerfileImageConfig.Dependencies, i.Conveyor)
ref, err := i.dockerfileExpanderFactory.GetExpander(dockerfile.ExpandOptions{SkipUnsetEnv: false}).ProcessWordWithMap(i.baseImageReference, dependenciesArgs)
if err != nil {
return fmt.Errorf("unable to expand dockerfile base image reference %q: %w", i.baseImageReference, err)
Expand Down
1 change: 1 addition & 0 deletions pkg/build/image/stapel.go
View file
Expand Up @@ -66,6 +66,7 @@ func initStages(ctx context.Context, image *Image, metaConfig *config.Meta, stap
imageArtifact := stapelImageConfig.IsArtifact()

baseStageOptions := &stage.BaseStageOptions{
TargetPlatform: image.TargetPlatform,
ImageName: imageName,
ConfigMounts: imageBaseConfig.Mount,
ImageTmpDir: filepath.Join(opts.TmpDir, "image", imageBaseConfig.Name),
Expand Down

0 comments on commit 3e2add1

Please sign in to comment.