Skip to content

Commit

Permalink
feat(images-dependencies): implement images dependencies for dockerfi…
Browse files Browse the repository at this point in the history
…le builder
  • Loading branch information
distorhead committed Jan 28, 2022
1 parent db03a32 commit f8b0204
Show file tree
Hide file tree
Showing 19 changed files with 992 additions and 472 deletions.
2 changes: 1 addition & 1 deletion cmd/werf/cleanup/cleanup.go
Expand Up @@ -240,7 +240,7 @@ It is worth noting that auto-cleaning is enabled by default, and manual use is u

cleanupOptions := cleaning.CleanupOptions{
ImageNameList: imagesNames,
LocalGit: giterminismManager.LocalGitRepo(),
LocalGit: giterminismManager.LocalGitRepo().(*git_repo.Local),
KubernetesContextClients: kubernetesContextClients,
KubernetesNamespaceRestrictionByContext: common.GetKubernetesNamespaceRestrictionByContext(&commonCmdData, kubernetesContextClients),
WithoutKube: *commonCmdData.WithoutKube,
Expand Down
12 changes: 6 additions & 6 deletions cmd/werf/common/common.go
Expand Up @@ -1134,8 +1134,8 @@ func GetCustomWerfConfigRelPath(giterminismManager giterminism_manager.Interface
}

customConfigPath = util.GetAbsoluteFilepath(customConfigPath)
if !util.IsSubpathOfBasePath(giterminismManager.LocalGitRepo().WorkTreeDir, customConfigPath) {
return "", fmt.Errorf("the werf config %q must be in the project git work tree %q", customConfigPath, giterminismManager.LocalGitRepo().WorkTreeDir)
if !util.IsSubpathOfBasePath(giterminismManager.LocalGitRepo().GetWorkTreeDir(), customConfigPath) {
return "", fmt.Errorf("the werf config %q must be in the project git work tree %q", customConfigPath, giterminismManager.LocalGitRepo().GetWorkTreeDir())
}

return util.GetRelativeToBaseFilepath(giterminismManager.ProjectDir(), customConfigPath), nil
Expand All @@ -1148,8 +1148,8 @@ func GetCustomWerfConfigTemplatesDirRelPath(giterminismManager giterminism_manag
}

customConfigTemplatesDirPath = util.GetAbsoluteFilepath(customConfigTemplatesDirPath)
if !util.IsSubpathOfBasePath(giterminismManager.LocalGitRepo().WorkTreeDir, customConfigTemplatesDirPath) {
return "", fmt.Errorf("the werf configuration templates directory %q must be in the project git work tree %q", customConfigTemplatesDirPath, giterminismManager.LocalGitRepo().WorkTreeDir)
if !util.IsSubpathOfBasePath(giterminismManager.LocalGitRepo().GetWorkTreeDir(), customConfigTemplatesDirPath) {
return "", fmt.Errorf("the werf configuration templates directory %q must be in the project git work tree %q", customConfigTemplatesDirPath, giterminismManager.LocalGitRepo().GetWorkTreeDir())
}

return util.GetRelativeToBaseFilepath(giterminismManager.ProjectDir(), customConfigTemplatesDirPath), nil
Expand Down Expand Up @@ -1240,8 +1240,8 @@ func GetHelmChartDir(werfConfigPath string, werfConfig *config.WerfConfig, giter
}

absHelmChartDir := filepath.Join(giterminismManager.ProjectDir(), helmChartDir)
if !util.IsSubpathOfBasePath(giterminismManager.LocalGitRepo().WorkTreeDir, absHelmChartDir) {
return "", fmt.Errorf("the chart directory %s must be in the project git work tree %s", absHelmChartDir, giterminismManager.LocalGitRepo().WorkTreeDir)
if !util.IsSubpathOfBasePath(giterminismManager.LocalGitRepo().GetWorkTreeDir(), absHelmChartDir) {
return "", fmt.Errorf("the chart directory %s must be in the project git work tree %s", absHelmChartDir, giterminismManager.LocalGitRepo().GetWorkTreeDir())
}

return helmChartDir, nil
Expand Down
58 changes: 5 additions & 53 deletions pkg/build/conveyor.go
Expand Up @@ -9,7 +9,6 @@ import (
"path"
"path/filepath"
"reflect"
"strconv"
"strings"
"sync"

Expand All @@ -21,6 +20,7 @@ import (
"github.com/werf/logboek"
stylePkg "github.com/werf/logboek/pkg/style"
"github.com/werf/logboek/pkg/types"
"github.com/werf/werf/pkg/build/dockerfile_helpers"
"github.com/werf/werf/pkg/build/import_server"
"github.com/werf/werf/pkg/build/stage"
"github.com/werf/werf/pkg/config"
Expand Down Expand Up @@ -1257,33 +1257,19 @@ func prepareImageBasedOnImageFromDockerfile(ctx context.Context, imageFromDocker
return nil, err
}

resolveDockerStagesFromValue(dockerStages)
dockerfile_helpers.ResolveDockerStagesFromValue(dockerStages)

dockerTargetIndex, err := getDockerTargetStageIndex(dockerStages, imageFromDockerfileConfig.Target)
dockerTargetIndex, err := dockerfile_helpers.GetDockerTargetStageIndex(dockerStages, imageFromDockerfileConfig.Target)
if err != nil {
return nil, err
}

dockerTargetStage := dockerStages[dockerTargetIndex]

ds, err := stage.NewDockerStages(
ds := stage.NewDockerStages(
dockerStages,
util.MapStringInterfaceToMapStringString(imageFromDockerfileConfig.Args),
dockerMetaArgs,
dockerTargetIndex,
)
if err != nil {
return nil, err
}

resolvedBaseName, err := ds.ShlexProcessWordWithMetaArgs(dockerTargetStage.BaseName)
if err != nil {
return nil, err
}

if err := handleImageFromName(ctx, resolvedBaseName, false, img, c); err != nil {
return nil, err
}

baseStageOptions := &stage.NewBaseStageOptions{
ImageName: imageFromDockerfileConfig.Name,
Expand All @@ -1305,6 +1291,7 @@ func prepareImageBasedOnImageFromDockerfile(ctx context.Context, imageFromDocker
ds,
stage.NewContextChecksum(dockerignorePathMatcher),
baseStageOptions,
imageFromDockerfileConfig.Dependencies,
)

img.stages = append(img.stages, dockerfileStage)
Expand All @@ -1313,38 +1300,3 @@ func prepareImageBasedOnImageFromDockerfile(ctx context.Context, imageFromDocker

return img, nil
}

func resolveDockerStagesFromValue(stages []instructions.Stage) {
nameToIndex := make(map[string]string)
for i, s := range stages {
name := strings.ToLower(s.Name)
index := strconv.Itoa(i)
if name != index {
nameToIndex[name] = index
}

for _, cmd := range s.Commands {
copyCmd, ok := cmd.(*instructions.CopyCommand)
if ok && copyCmd.From != "" {
from := strings.ToLower(copyCmd.From)
if val, ok := nameToIndex[from]; ok {
copyCmd.From = val
}
}
}
}
}

func getDockerTargetStageIndex(dockerStages []instructions.Stage, dockerTargetStage string) (int, error) {
if dockerTargetStage == "" {
return len(dockerStages) - 1, nil
}

for i, s := range dockerStages {
if s.Name == dockerTargetStage {
return i, nil
}
}

return -1, fmt.Errorf("%s is not a valid target build stage", dockerTargetStage)
}
44 changes: 44 additions & 0 deletions pkg/build/dockerfile_helpers/dockerfile_helpers.go
@@ -0,0 +1,44 @@
package dockerfile_helpers

import (
"fmt"
"strconv"
"strings"

"github.com/moby/buildkit/frontend/dockerfile/instructions"
)

func ResolveDockerStagesFromValue(stages []instructions.Stage) {
nameToIndex := make(map[string]string)
for i, s := range stages {
name := strings.ToLower(s.Name)
index := strconv.Itoa(i)
if name != index {
nameToIndex[name] = index
}

for _, cmd := range s.Commands {
copyCmd, ok := cmd.(*instructions.CopyCommand)
if ok && copyCmd.From != "" {
from := strings.ToLower(copyCmd.From)
if val, ok := nameToIndex[from]; ok {
copyCmd.From = val
}
}
}
}
}

func GetDockerTargetStageIndex(dockerStages []instructions.Stage, dockerTargetStage string) (int, error) {
if dockerTargetStage == "" {
return len(dockerStages) - 1, nil
}

for i, s := range dockerStages {
if s.Name == dockerTargetStage {
return i, nil
}
}

return -1, fmt.Errorf("%s is not a valid target build stage", dockerTargetStage)
}
131 changes: 131 additions & 0 deletions pkg/build/stage/data_test.go
@@ -0,0 +1,131 @@
package stage

import (
"fmt"

. "github.com/onsi/gomega"

"github.com/werf/werf/pkg/config"
"github.com/werf/werf/pkg/util"
)

type TestDependencies struct {
Dependencies []*TestDependency
ExpectedDigest string
}

type TestDependency struct {
ImageName string

TargetEnvImageName string
TargetEnvImageRepo string
TargetEnvImageTag string
TargetEnvImageID string

TargetBuildArgImageName string
TargetBuildArgImageRepo string
TargetBuildArgImageTag string
TargetBuildArgImageID string

DockerImageRepo string
DockerImageTag string
DockerImageID string
}

func (dep *TestDependency) GetDockerImageName() string {
return fmt.Sprintf("%s:%s", dep.DockerImageRepo, dep.DockerImageTag)
}

func (dep *TestDependency) ToConfigDependency() *config.Dependency {
depCfg := &config.Dependency{ImageName: dep.ImageName}

if dep.TargetEnvImageName != "" {
depCfg.Imports = append(depCfg.Imports, &config.DependencyImport{
Type: config.ImageNameImport,
TargetEnv: dep.TargetEnvImageName,
})
}
if dep.TargetEnvImageRepo != "" {
depCfg.Imports = append(depCfg.Imports, &config.DependencyImport{
Type: config.ImageRepoImport,
TargetEnv: dep.TargetEnvImageRepo,
})
}
if dep.TargetEnvImageTag != "" {
depCfg.Imports = append(depCfg.Imports, &config.DependencyImport{
Type: config.ImageTagImport,
TargetEnv: dep.TargetEnvImageTag,
})
}
if dep.TargetEnvImageID != "" {
depCfg.Imports = append(depCfg.Imports, &config.DependencyImport{
Type: config.ImageIDImport,
TargetEnv: dep.TargetEnvImageID,
})
}

if dep.TargetBuildArgImageName != "" {
depCfg.Imports = append(depCfg.Imports, &config.DependencyImport{
Type: config.ImageNameImport,
TargetBuildArg: dep.TargetBuildArgImageName,
})
}
if dep.TargetBuildArgImageRepo != "" {
depCfg.Imports = append(depCfg.Imports, &config.DependencyImport{
Type: config.ImageRepoImport,
TargetBuildArg: dep.TargetBuildArgImageRepo,
})
}
if dep.TargetBuildArgImageTag != "" {
depCfg.Imports = append(depCfg.Imports, &config.DependencyImport{
Type: config.ImageTagImport,
TargetBuildArg: dep.TargetBuildArgImageTag,
})
}
if dep.TargetBuildArgImageID != "" {
depCfg.Imports = append(depCfg.Imports, &config.DependencyImport{
Type: config.ImageIDImport,
TargetBuildArg: dep.TargetBuildArgImageID,
})
}

return depCfg
}

func GetConfigDependencies(dependencies []*TestDependency) (res []*config.Dependency) {
for _, dep := range dependencies {
res = append(res, dep.ToConfigDependency())
}

return
}

func CheckImageDependenciesAfterPrepare(img *LegacyImageStub, dependencies []*TestDependency) {
for _, dep := range dependencies {
if dep.TargetEnvImageName != "" {
Expect(img._Container._ServiceCommitChangeOptions.Env[dep.TargetEnvImageName]).To(Equal(dep.GetDockerImageName()))
}
if dep.TargetEnvImageRepo != "" {
Expect(img._Container._ServiceCommitChangeOptions.Env[dep.TargetEnvImageRepo]).To(Equal(dep.DockerImageRepo))
}
if dep.TargetEnvImageTag != "" {
Expect(img._Container._ServiceCommitChangeOptions.Env[dep.TargetEnvImageTag]).To(Equal(dep.DockerImageTag))
}
if dep.TargetEnvImageID != "" {
Expect(img._Container._ServiceCommitChangeOptions.Env[dep.TargetEnvImageID]).To(Equal(dep.DockerImageID))
}

if dep.TargetBuildArgImageName != "" {
Expect(util.IsStringsContainValue(img._DockerfileImageBuilder.BuildDockerfileOptions.BuildArgs, fmt.Sprintf("%s=%s", dep.TargetBuildArgImageName, dep.GetDockerImageName())))
}
if dep.TargetBuildArgImageRepo != "" {
Expect(util.IsStringsContainValue(img._DockerfileImageBuilder.BuildDockerfileOptions.BuildArgs, fmt.Sprintf("%s=%s", dep.TargetBuildArgImageRepo, dep.DockerImageRepo)))
}
if dep.TargetBuildArgImageTag != "" {
Expect(util.IsStringsContainValue(img._DockerfileImageBuilder.BuildDockerfileOptions.BuildArgs, fmt.Sprintf("%s=%s", dep.TargetBuildArgImageTag, dep.DockerImageTag)))
}
if dep.TargetBuildArgImageID != "" {
Expect(util.IsStringsContainValue(img._DockerfileImageBuilder.BuildDockerfileOptions.BuildArgs, fmt.Sprintf("%s=%s", dep.TargetBuildArgImageID, dep.DockerImageID)))
}
}
}
2 changes: 0 additions & 2 deletions pkg/build/stage/dependencies.go
Expand Up @@ -136,8 +136,6 @@ func (s *DependenciesStage) PrepareImage(ctx context.Context, c Conveyor, _, img
depImageServiceOptions.AddEnv(map[string]string{
img.TargetEnv: depImageID,
})
default:
panic("unexpected configuration")
}
}
}
Expand Down

0 comments on commit f8b0204

Please sign in to comment.