From 73729924905ef03a199a5d2fb26cccded5f6c69a Mon Sep 17 00:00:00 2001 From: Timofey Kirillov Date: Wed, 19 Jan 2022 17:12:03 +0300 Subject: [PATCH] fix(build): virtual merge commits and inconsistent build cache An inconsistency in the werf/actions or github-actions lead to invalid --virtual-merge-into-commit param passed into werf. It leads to creating inconsistent stage in the repo by the werf. Solution: remove --virtual-merge-into-commit and --virtual-merge-from-commit params and detect from/into commits by itself reading virtual-merge commit metadata from the git. --- Makefile | 6 +- cmd/werf/build/main.go | 2 - cmd/werf/bundle/export/export.go | 4 +- cmd/werf/bundle/publish/publish.go | 4 +- cmd/werf/common/common.go | 14 +- cmd/werf/common/conveyor_options.go | 4 +- cmd/werf/compose/main.go | 2 - cmd/werf/converge/converge.go | 2 - cmd/werf/export/export.go | 2 - cmd/werf/helm/get_autogenerated_values.go | 2 - cmd/werf/render/render.go | 4 +- cmd/werf/run/run.go | 2 - cmd/werf/stage/image/main.go | 2 - go.mod | 3 +- go.sum | 5 + pkg/build/build_phase.go | 9 +- pkg/build/conveyor.go | 4 +- pkg/build/stage/conveyor.go | 4 +- pkg/build/stage/git_archive.go | 2 +- pkg/build/stage/git_mapping.go | 62 ++--- pkg/build/stage/git_mapping_test.go | 318 ++++++++++++++++++++++ pkg/build/stage/stage_suite_test.go | 13 + pkg/git_repo/git_repo.go | 1 + pkg/git_repo/local.go | 4 + pkg/git_repo/remote.go | 4 + pkg/git_repo/virtual_merge.go | 16 ++ 26 files changed, 410 insertions(+), 85 deletions(-) create mode 100644 pkg/build/stage/git_mapping_test.go create mode 100644 pkg/build/stage/stage_suite_test.go create mode 100644 pkg/git_repo/virtual_merge.go diff --git a/Makefile b/Makefile index cea06d1eb4..ac4b9ea7a7 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: all werf buildah-test unit-test fmt lint clean +.PHONY: all werf buildah-test unit-test fmt lint docs clean all: werf @@ -20,6 +20,10 @@ lint: golangci-lint run ./... --build-tags="dfrunmount dfssh containers_image_openpgp osusergo exclude_graphdriver_devicemapper netgo no_devmapper static_build" +docs: + ./docs/regen.sh + + clean: rm -f $$GOPATH/bin/werf rm -f $$GOPATH/buildah-test diff --git a/cmd/werf/build/main.go b/cmd/werf/build/main.go index da3fcfa35a..d8b80dcd25 100644 --- a/cmd/werf/build/main.go +++ b/cmd/werf/build/main.go @@ -107,8 +107,6 @@ If one or more IMAGE_NAME parameters specified, werf will build only these image common.SetupAddCustomTag(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit) common.SetupFollow(&commonCmdData, cmd) diff --git a/cmd/werf/bundle/export/export.go b/cmd/werf/bundle/export/export.go index 4f6adf88ec..505ff109f9 100644 --- a/cmd/werf/bundle/export/export.go +++ b/cmd/werf/bundle/export/export.go @@ -7,7 +7,7 @@ import ( "time" "github.com/spf13/cobra" - "helm.sh/helm/v3/cmd/helm" + helm_v3 "helm.sh/helm/v3/cmd/helm" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli/values" @@ -107,8 +107,6 @@ func NewCmd() *cobra.Command { common.SetupUseCustomTag(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit) diff --git a/cmd/werf/bundle/publish/publish.go b/cmd/werf/bundle/publish/publish.go index feec40583b..c7b1af1dec 100644 --- a/cmd/werf/bundle/publish/publish.go +++ b/cmd/werf/bundle/publish/publish.go @@ -10,7 +10,7 @@ import ( "github.com/Masterminds/semver" uuid "github.com/satori/go.uuid" "github.com/spf13/cobra" - "helm.sh/helm/v3/cmd/helm" + helm_v3 "helm.sh/helm/v3/cmd/helm" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/cli/values" @@ -117,8 +117,6 @@ Published into container registry bundle can be rolled out by the "werf bundle" common.SetupUseCustomTag(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit) diff --git a/cmd/werf/common/common.go b/cmd/werf/common/common.go index 14699cec9f..876e06f9fc 100644 --- a/cmd/werf/common/common.go +++ b/cmd/werf/common/common.go @@ -111,9 +111,7 @@ type CmdData struct { ReportPath *string ReportFormat *string - VirtualMerge *bool - VirtualMergeFromCommit *string - VirtualMergeIntoCommit *string + VirtualMerge *bool ScanContextNamespaceOnly *bool @@ -1504,16 +1502,6 @@ func SetupVirtualMerge(cmdData *CmdData, cmd *cobra.Command) { cmd.Flags().BoolVarP(cmdData.VirtualMerge, "virtual-merge", "", GetBoolEnvironmentDefaultFalse("WERF_VIRTUAL_MERGE"), "Enable virtual/ephemeral merge commit mode when building current application state ($WERF_VIRTUAL_MERGE by default)") } -func SetupVirtualMergeFromCommit(cmdData *CmdData, cmd *cobra.Command) { - cmdData.VirtualMergeFromCommit = new(string) - cmd.Flags().StringVarP(cmdData.VirtualMergeFromCommit, "virtual-merge-from-commit", "", os.Getenv("WERF_VIRTUAL_MERGE_FROM_COMMIT"), "Commit hash for virtual/ephemeral merge commit with new changes introduced in the pull request ($WERF_VIRTUAL_MERGE_FROM_COMMIT by default)") -} - -func SetupVirtualMergeIntoCommit(cmdData *CmdData, cmd *cobra.Command) { - cmdData.VirtualMergeIntoCommit = new(string) - cmd.Flags().StringVarP(cmdData.VirtualMergeIntoCommit, "virtual-merge-into-commit", "", os.Getenv("WERF_VIRTUAL_MERGE_INTO_COMMIT"), "Commit hash for virtual/ephemeral merge commit which is base for changes introduced in the pull request ($WERF_VIRTUAL_MERGE_INTO_COMMIT by default)") -} - func SetupPlatform(cmdData *CmdData, cmd *cobra.Command) { cmdData.Platform = new(string) diff --git a/cmd/werf/common/conveyor_options.go b/cmd/werf/common/conveyor_options.go index 8edbcd420d..a5ca6325c6 100644 --- a/cmd/werf/common/conveyor_options.go +++ b/cmd/werf/common/conveyor_options.go @@ -18,9 +18,7 @@ import ( func GetConveyorOptions(commonCmdData *CmdData) build.ConveyorOptions { return build.ConveyorOptions{ LocalGitRepoVirtualMergeOptions: stage.VirtualMergeOptions{ - VirtualMerge: *commonCmdData.VirtualMerge, - VirtualMergeFromCommit: *commonCmdData.VirtualMergeFromCommit, - VirtualMergeIntoCommit: *commonCmdData.VirtualMergeIntoCommit, + VirtualMerge: *commonCmdData.VirtualMerge, }, } } diff --git a/cmd/werf/compose/main.go b/cmd/werf/compose/main.go index 05372bdd91..1cd92157c3 100644 --- a/cmd/werf/compose/main.go +++ b/cmd/werf/compose/main.go @@ -236,8 +236,6 @@ services: common.SetupDryRun(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupDisableAutoHostCleanup(&commonCmdData, cmd) common.SetupAllowedDockerStorageVolumeUsage(&commonCmdData, cmd) diff --git a/cmd/werf/converge/converge.go b/cmd/werf/converge/converge.go index 5a90ead129..89291cba8d 100644 --- a/cmd/werf/converge/converge.go +++ b/cmd/werf/converge/converge.go @@ -139,8 +139,6 @@ werf converge --repo registry.mydomain.com/web --env production`, common.SetupUseCustomTag(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit) common.SetupSkipBuild(&commonCmdData, cmd) diff --git a/cmd/werf/export/export.go b/cmd/werf/export/export.go index 5b87ea4a33..7fda02e2e4 100644 --- a/cmd/werf/export/export.go +++ b/cmd/werf/export/export.go @@ -97,8 +97,6 @@ All meta-information related to werf is removed from the exported images, and th common.SetupDryRun(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupPlatform(&commonCmdData, cmd) diff --git a/cmd/werf/helm/get_autogenerated_values.go b/cmd/werf/helm/get_autogenerated_values.go index 882f461f4a..a16d0f9f05 100644 --- a/cmd/werf/helm/get_autogenerated_values.go +++ b/cmd/werf/helm/get_autogenerated_values.go @@ -72,8 +72,6 @@ These values includes project name, docker images ids and other`), common.SetupUseCustomTag(&getAutogeneratedValuedCmdData, cmd) common.SetupVirtualMerge(&getAutogeneratedValuedCmdData, cmd) - common.SetupVirtualMergeFromCommit(&getAutogeneratedValuedCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&getAutogeneratedValuedCmdData, cmd) common.SetupNamespace(&getAutogeneratedValuedCmdData, cmd) diff --git a/cmd/werf/render/render.go b/cmd/werf/render/render.go index 3db653a2df..c3c1dcf3b3 100644 --- a/cmd/werf/render/render.go +++ b/cmd/werf/render/render.go @@ -8,7 +8,7 @@ import ( "path/filepath" "github.com/spf13/cobra" - "helm.sh/helm/v3/cmd/helm" + helm_v3 "helm.sh/helm/v3/cmd/helm" "helm.sh/helm/v3/pkg/action" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" @@ -116,8 +116,6 @@ func NewCmd() *cobra.Command { common.SetupUseCustomTag(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupParallelOptions(&commonCmdData, cmd, common.DefaultBuildParallelTasksLimit) diff --git a/cmd/werf/run/run.go b/cmd/werf/run/run.go index 330f4ced3d..fe3018a783 100644 --- a/cmd/werf/run/run.go +++ b/cmd/werf/run/run.go @@ -147,8 +147,6 @@ func NewCmd() *cobra.Command { common.SetupDryRun(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupPlatform(&commonCmdData, cmd) diff --git a/cmd/werf/stage/image/main.go b/cmd/werf/stage/image/main.go index 4be08e106f..2972b4b95a 100644 --- a/cmd/werf/stage/image/main.go +++ b/cmd/werf/stage/image/main.go @@ -83,8 +83,6 @@ func NewCmd() *cobra.Command { common.SetupKubeContext(&commonCmdData, cmd) common.SetupVirtualMerge(&commonCmdData, cmd) - common.SetupVirtualMergeFromCommit(&commonCmdData, cmd) - common.SetupVirtualMergeIntoCommit(&commonCmdData, cmd) common.SetupPlatform(&commonCmdData, cmd) diff --git a/go.mod b/go.mod index f0b169e107..e857644ec5 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,8 @@ require ( github.com/mvdan/xurls v1.1.0 // indirect github.com/oleiade/reflections v1.0.1 // indirect github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.16.0 + github.com/onsi/ginkgo/v2 v2.0.0 // indirect + github.com/onsi/gomega v1.17.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 github.com/opencontainers/runc v1.0.3 // indirect diff --git a/go.sum b/go.sum index 30e1a5f8c2..e84088f8bf 100644 --- a/go.sum +++ b/go.sum @@ -979,6 +979,7 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -1494,6 +1495,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108 github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1506,6 +1509,8 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.17.0 h1:9Luw4uT5HTjHTN8+aNcSThgH1vdXnmdJ8xIfZ4wyTRE= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= diff --git a/pkg/build/build_phase.go b/pkg/build/build_phase.go index 6ccd67726e..04d0120292 100644 --- a/pkg/build/build_phase.go +++ b/pkg/build/build_phase.go @@ -19,6 +19,7 @@ import ( "github.com/werf/logboek/pkg/types" "github.com/werf/werf/pkg/build/stage" "github.com/werf/werf/pkg/container_runtime" + "github.com/werf/werf/pkg/git_repo" imagePkg "github.com/werf/werf/pkg/image" "github.com/werf/werf/pkg/stapel" "github.com/werf/werf/pkg/storage" @@ -272,7 +273,13 @@ func (phase *BuildPhase) publishImageMetadata(ctx context.Context, img *Image) e commits = append(commits, headCommit) if phase.Conveyor.GetLocalGitRepoVirtualMergeOptions().VirtualMerge { - fromCommit := phase.Conveyor.GetLocalGitRepoVirtualMergeOptions().VirtualMergeFromCommit + phase.Conveyor.giterminismManager.LocalGitRepo().GetMergeCommitParents(ctx, headCommit) + + fromCommit, _, err := git_repo.GetVirtualMergeParents(ctx, phase.Conveyor.giterminismManager.LocalGitRepo(), headCommit) + if err != nil { + return fmt.Errorf("unable to get virtual merge commit %q parents: %s", headCommit, err) + } + commits = append(commits, fromCommit) } diff --git a/pkg/build/conveyor.go b/pkg/build/conveyor.go index 51339a2a26..b02d8e1ca7 100644 --- a/pkg/build/conveyor.go +++ b/pkg/build/conveyor.go @@ -1077,7 +1077,7 @@ func gitRemoteArtifactInit(ctx context.Context, remoteGitMappingConfig *config.G gitMapping.Branch = remoteGitMappingConfig.Branch gitMapping.Name = remoteGitMappingConfig.Name - gitMapping.RemoteGitRepo = remoteGitRepo + gitMapping.SetGitRepo(remoteGitRepo) gitMappingTo, err := makeGitMappingTo(ctx, gitMapping, remoteGitMappingConfig.GitLocalExport.GitMappingTo(), c) if err != nil { @@ -1092,7 +1092,7 @@ func gitLocalPathInit(ctx context.Context, localGitMappingConfig *config.GitLoca gitMapping := baseGitMappingInit(localGitMappingConfig.GitLocalExport, imageName, c) gitMapping.Name = "own" - gitMapping.LocalGitRepo = c.giterminismManager.LocalGitRepo() + gitMapping.SetGitRepo(c.giterminismManager.LocalGitRepo()) gitMappingTo, err := makeGitMappingTo(ctx, gitMapping, localGitMappingConfig.GitLocalExport.GitMappingTo(), c) if err != nil { diff --git a/pkg/build/stage/conveyor.go b/pkg/build/stage/conveyor.go index 981c019889..de843618c7 100644 --- a/pkg/build/stage/conveyor.go +++ b/pkg/build/stage/conveyor.go @@ -29,7 +29,5 @@ type Conveyor interface { } type VirtualMergeOptions struct { - VirtualMerge bool - VirtualMergeFromCommit string - VirtualMergeIntoCommit string + VirtualMerge bool } diff --git a/pkg/build/stage/git_archive.go b/pkg/build/stage/git_archive.go index 3e2db76d85..5a58bc3c2e 100644 --- a/pkg/build/stage/git_archive.go +++ b/pkg/build/stage/git_archive.go @@ -50,7 +50,7 @@ func (s *GitArchiveStage) SelectSuitableStage(ctx context.Context, c Conveyor, s func (s *GitArchiveStage) GetDependencies(ctx context.Context, c Conveyor, _, _ container_runtime.LegacyImageInterface) (string, error) { var args []string for _, gitMapping := range s.gitMappings { - if gitMapping.LocalGitRepo != nil { + if gitMapping.IsLocal() { if err := c.GiterminismManager().Inspector().InspectBuildContextFiles(ctx, gitMapping.getPathMatcher()); err != nil { return "", err } diff --git a/pkg/build/stage/git_mapping.go b/pkg/build/stage/git_mapping.go index 66440a8d9a..de55c9ecd1 100644 --- a/pkg/build/stage/git_mapping.go +++ b/pkg/build/stage/git_mapping.go @@ -22,9 +22,6 @@ import ( ) type GitMapping struct { - LocalGitRepo *git_repo.Local - RemoteGitRepo *git_repo.Remote - Name string As string Branch string @@ -45,6 +42,7 @@ type GitMapping struct { BaseCommitByPrevBuiltImageName map[string]string + gitRepo git_repo.GitRepo mutexes map[string]*sync.Mutex mutex sync.Mutex } @@ -89,14 +87,12 @@ func (gm *GitMapping) getMutex(key string) *sync.Mutex { return m } -func (gm *GitMapping) GitRepo() git_repo.GitRepo { - if gm.LocalGitRepo != nil { - return gm.LocalGitRepo - } else if gm.RemoteGitRepo != nil { - return gm.RemoteGitRepo - } +func (gm *GitMapping) SetGitRepo(gitRepo git_repo.GitRepo) { + gm.gitRepo = gitRepo +} - panic("GitRepo not initialized") +func (gm *GitMapping) GitRepo() git_repo.GitRepo { + return gm.gitRepo } func (gm *GitMapping) makeArchiveOptions(ctx context.Context, commit string) (*git_repo.ArchiveOptions, error) { @@ -184,11 +180,7 @@ func (gm *GitMapping) getFileRenames(ctx context.Context, commit string) (map[st } func (gm *GitMapping) IsLocal() bool { - if gm.LocalGitRepo != nil { - return true - } else { - return false - } + return gm.GitRepo().IsLocal() } func (gm *GitMapping) getLatestCommit(ctx context.Context) (string, error) { @@ -280,31 +272,25 @@ func (gm *GitMapping) GetLatestCommitInfo(ctx context.Context, c Conveyor) (Imag res.Commit = commit } - if _, isLocal := gm.GitRepo().(*git_repo.Local); isLocal && c.GetLocalGitRepoVirtualMergeOptions().VirtualMerge { + if gm.GitRepo().IsLocal() && c.GetLocalGitRepoVirtualMergeOptions().VirtualMerge { res.VirtualMerge = true - res.VirtualMergeFromCommit = c.GetLocalGitRepoVirtualMergeOptions().VirtualMergeFromCommit - res.VirtualMergeIntoCommit = c.GetLocalGitRepoVirtualMergeOptions().VirtualMergeIntoCommit - - if res.VirtualMergeFromCommit == "" || res.VirtualMergeIntoCommit == "" { - if parents, err := gm.GitRepo().GetMergeCommitParents(ctx, res.Commit); err != nil { - return ImageCommitInfo{}, fmt.Errorf("unable to get virtual merge commit %s parents for git repo %s: %s", res.Commit, gm.GitRepo().GetName(), err) - } else if len(parents) == 2 { - if res.VirtualMergeIntoCommit == "" { - res.VirtualMergeIntoCommit = parents[0] - logboek.Context(ctx).Debug().LogF("Got virtual-merge-into-commit from parents of %s => %s\n", res.Commit, res.VirtualMergeIntoCommit) - } - if res.VirtualMergeFromCommit == "" { - res.VirtualMergeFromCommit = parents[1] - logboek.Context(ctx).Debug().LogF("Got virtual-merge-from-commit from parents of %s => %s\n", res.Commit, res.VirtualMergeFromCommit) - } - } + + fromCommit, intoCommit, err := git_repo.GetVirtualMergeParents(ctx, gm.GitRepo(), res.Commit) + if err != nil { + return ImageCommitInfo{}, fmt.Errorf("unable to get virtual merge commit %q parents: %s", res.Commit, err) } + res.VirtualMergeFromCommit = fromCommit + logboek.Context(ctx).Debug().LogF("Got virtual-merge-from-commit from parents of %s => %s\n", res.Commit, res.VirtualMergeFromCommit) + + res.VirtualMergeIntoCommit = intoCommit + logboek.Context(ctx).Debug().LogF("Got virtual-merge-into-commit from parents of %s => %s\n", res.Commit, res.VirtualMergeIntoCommit) + if res.VirtualMergeFromCommit == "" { - return ImageCommitInfo{}, fmt.Errorf("unable to detect --virtual-merge-from-commit for virtual merge commit %s", res.Commit) + return ImageCommitInfo{}, fmt.Errorf("unable to detect virtual-merge-from-commit for virtual merge commit %q", res.Commit) } if res.VirtualMergeIntoCommit == "" { - return ImageCommitInfo{}, fmt.Errorf("unable to detect --virtual-merge-into-commit for virtual merge commit %s", res.Commit) + return ImageCommitInfo{}, fmt.Errorf("unable to detect virtual-merge-into-commit for virtual merge commit %q", res.Commit) } } @@ -346,7 +332,7 @@ func (gm *GitMapping) GetBaseCommitForPrevBuiltImage(ctx context.Context, c Conv if prevBuiltImageCommitInfo.VirtualMerge { if latestCommit, err := gm.getLatestCommit(ctx); err != nil { return "", err - } else if _, isLocal := gm.GitRepo().(*git_repo.Local); isLocal && c.GetLocalGitRepoVirtualMergeOptions().VirtualMerge && latestCommit == prevBuiltImageCommitInfo.Commit { + } else if gm.GitRepo().IsLocal() && c.GetLocalGitRepoVirtualMergeOptions().VirtualMerge && latestCommit == prevBuiltImageCommitInfo.Commit { baseCommit = prevBuiltImageCommitInfo.Commit } else { if detachedMergeCommit, err := gm.GitRepo().CreateDetachedMergeCommit(ctx, prevBuiltImageCommitInfo.VirtualMergeFromCommit, prevBuiltImageCommitInfo.VirtualMergeIntoCommit); err != nil { @@ -365,8 +351,10 @@ func (gm *GitMapping) GetBaseCommitForPrevBuiltImage(ctx context.Context, c Conv } type ImageCommitInfo struct { - Commit string - VirtualMergeOptions + Commit string + VirtualMerge bool + VirtualMergeFromCommit string + VirtualMergeIntoCommit string } func makeInvalidImageError(label string) error { diff --git a/pkg/build/stage/git_mapping_test.go b/pkg/build/stage/git_mapping_test.go new file mode 100644 index 0000000000..17d85f64c7 --- /dev/null +++ b/pkg/build/stage/git_mapping_test.go @@ -0,0 +1,318 @@ +package stage_test + +import ( + "context" + "fmt" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/werf/werf/pkg/build/stage" + "github.com/werf/werf/pkg/container_runtime" + "github.com/werf/werf/pkg/git_repo" + "github.com/werf/werf/pkg/image" +) + +var _ = Describe("GitMapping", func() { + var gitMapping *stage.GitMapping + + BeforeEach(func() { + gitMapping = stage.NewGitMapping() + }) + + type BuiltImageCommitInfoCheckData struct { + CommitLabel string + VirtualMergeLabel string + VirtualMergeFromCommitLabel string + VirtualMergeIntoCommitLabel string + } + + DescribeTable("getting built image commit info from labels", + func(data BuiltImageCommitInfoCheckData, checkResult func(stage.ImageCommitInfo, BuiltImageCommitInfoCheckData)) { + gitRepo := NewGitRepoStub("own", true, "irrelevant-commit-id") + gitMapping.SetGitRepo(gitRepo) + + builtImageLabels := map[string]string{ + gitMapping.ImageGitCommitLabel(): data.CommitLabel, + gitMapping.VirtualMergeLabel(): data.VirtualMergeLabel, + gitMapping.VirtualMergeFromCommitLabel(): data.VirtualMergeFromCommitLabel, + gitMapping.VirtualMergeIntoCommitLabel(): data.VirtualMergeIntoCommitLabel, + } + + info, err := gitMapping.GetBuiltImageCommitInfo(builtImageLabels) + + Expect(err).To(Succeed()) + Expect(info.Commit).To(Equal(data.CommitLabel)) + + checkResult(info, data) + }, + + Entry("when using virtual merge commit", + BuiltImageCommitInfoCheckData{ + CommitLabel: "a4009816bf7d8f62b2d1cd3331d36f45b3df7d99", + VirtualMergeLabel: "true", + VirtualMergeFromCommitLabel: "c1159b9ce05143df3dd6450ee9d749c9642320d2", + VirtualMergeIntoCommitLabel: "cdb9a2e6ab3fec90f3ea12d4e4728ec4bf3f74dc", + }, + func(info stage.ImageCommitInfo, data BuiltImageCommitInfoCheckData) { + Expect(info.VirtualMerge).To(Equal(true)) + Expect(info.VirtualMergeFromCommit).To(Equal(data.VirtualMergeFromCommitLabel)) + Expect(info.VirtualMergeIntoCommit).To(Equal(data.VirtualMergeIntoCommitLabel)) + }), + + Entry("when not using virtual merge commit", + BuiltImageCommitInfoCheckData{ + CommitLabel: "08885f44226e48e19448e672b4db56b0f710a8b5", + VirtualMergeLabel: "false", + }, + func(info stage.ImageCommitInfo, data BuiltImageCommitInfoCheckData) { + Expect(info.VirtualMerge).To(Equal(false)) + }), + ) + + type LatestCommitInfoCheckData struct { + CurrentCommit string + IsCurrentCommitVirtualMerge bool + } + + DescribeTable("getting latest commit info", + func(data LatestCommitInfoCheckData, checkResultFunc func(stage.ImageCommitInfo, LatestCommitInfoCheckData)) { + ctx := context.Background() + c := NewConveyorStub(stage.VirtualMergeOptions{VirtualMerge: data.IsCurrentCommitVirtualMerge}) + + gitRepo := NewGitRepoStub("own", true, data.CurrentCommit) + gitMapping.SetGitRepo(gitRepo) + + info, err := gitMapping.GetLatestCommitInfo(ctx, c) + Expect(err).To(Succeed()) + Expect(info.Commit).To(Equal(data.CurrentCommit)) + + checkResultFunc(info, data) + }, + + Entry("when current commit is not virtual merge", + LatestCommitInfoCheckData{ + CurrentCommit: "ae6feb44da273003cfada392c33dfe33748a5e2f", + IsCurrentCommitVirtualMerge: false, + }, + func(info stage.ImageCommitInfo, data LatestCommitInfoCheckData) { + Expect(info.Commit).To(Equal(data.CurrentCommit)) + Expect(info.VirtualMerge).To(Equal(false)) + }, + ), + + Entry("when current commit is virtual merge", + LatestCommitInfoCheckData{ + CurrentCommit: "c69c9771d37c7cb8d8839c79079e83f59b29f343", + IsCurrentCommitVirtualMerge: true, + }, + func(info stage.ImageCommitInfo, data LatestCommitInfoCheckData) { + Expect(info.VirtualMerge).To(Equal(true)) + Expect(info.VirtualMergeFromCommit).To(Equal(constructMergeCommitParentFrom(data.CurrentCommit))) + Expect(info.VirtualMergeIntoCommit).To(Equal(constructMergeCommitParentInto(data.CurrentCommit))) + }, + ), + ) + + type BaseCommitForPrevBuiltImageCheckData struct { + BuiltCommitLabel string + BuiltVirtualMergeLabel string + BuiltVirtualMergeFromCommitLabel string + BuiltVirtualMergeIntoCommitLabel string + CurrentCommit string + IsCurrentCommitVirtualMerge bool + } + + DescribeTable("getting base commit from prev built image", + func(data BaseCommitForPrevBuiltImageCheckData, checkResultFunc func(string, BaseCommitForPrevBuiltImageCheckData)) { + ctx := context.Background() + c := NewConveyorStub(stage.VirtualMergeOptions{VirtualMerge: data.IsCurrentCommitVirtualMerge}) + + gitRepo := NewGitRepoStub("own", true, data.CurrentCommit) + gitMapping.SetGitRepo(gitRepo) + + prevBuiltImage := NewBuiltImageStub("stub", &image.StageDescription{ + Info: &image.Info{ + Labels: map[string]string{ + gitMapping.ImageGitCommitLabel(): data.BuiltCommitLabel, + gitMapping.VirtualMergeLabel(): data.BuiltVirtualMergeLabel, + gitMapping.VirtualMergeFromCommitLabel(): data.BuiltVirtualMergeFromCommitLabel, + gitMapping.VirtualMergeIntoCommitLabel(): data.BuiltVirtualMergeIntoCommitLabel, + }, + }, + }) + + baseCommit, err := gitMapping.GetBaseCommitForPrevBuiltImage(ctx, c, prevBuiltImage) + Expect(err).To(Succeed()) + + checkResultFunc(baseCommit, data) + }, + + Entry("when current commit the same as previously built and is not virtual merge", + BaseCommitForPrevBuiltImageCheckData{ + BuiltCommitLabel: "ae6feb44da273003cfada392c33dfe33748a5e2f", + BuiltVirtualMergeLabel: "false", + BuiltVirtualMergeFromCommitLabel: "", + BuiltVirtualMergeIntoCommitLabel: "", + CurrentCommit: "ae6feb44da273003cfada392c33dfe33748a5e2f", + IsCurrentCommitVirtualMerge: false, + }, + func(baseCommit string, data BaseCommitForPrevBuiltImageCheckData) { + Expect(baseCommit).To(Equal(data.CurrentCommit)) + }), + + Entry("when current commit the same as previously built and is virtual merge", + BaseCommitForPrevBuiltImageCheckData{ + BuiltCommitLabel: "a4009816bf7d8f62b2d1cd3331d36f45b3df7d99", + BuiltVirtualMergeLabel: "true", + BuiltVirtualMergeFromCommitLabel: "c1159b9ce05143df3dd6450ee9d749c9642320d2", + BuiltVirtualMergeIntoCommitLabel: "cdb9a2e6ab3fec90f3ea12d4e4728ec4bf3f74dc", + CurrentCommit: "a4009816bf7d8f62b2d1cd3331d36f45b3df7d99", + IsCurrentCommitVirtualMerge: true, + }, + func(baseCommit string, data BaseCommitForPrevBuiltImageCheckData) { + Expect(baseCommit).To(Equal(data.CurrentCommit)) + }), + + Entry("when current commit is virtual merge and previous commit is virtual merge", + BaseCommitForPrevBuiltImageCheckData{ + BuiltCommitLabel: "a4009816bf7d8f62b2d1cd3331d36f45b3df7d99", + BuiltVirtualMergeLabel: "true", + BuiltVirtualMergeFromCommitLabel: "c1159b9ce05143df3dd6450ee9d749c9642320d2", + BuiltVirtualMergeIntoCommitLabel: "cdb9a2e6ab3fec90f3ea12d4e4728ec4bf3f74dc", + CurrentCommit: "d2a341ee66189d0927c47a13541ae1926fad05bb", + IsCurrentCommitVirtualMerge: true, + }, + func(baseCommit string, data BaseCommitForPrevBuiltImageCheckData) { + Expect(baseCommit).To(Equal(constructVirtualMergeCommit(data.BuiltVirtualMergeFromCommitLabel, data.BuiltVirtualMergeIntoCommitLabel))) + }), + + Entry("when current commit is virtual merge and previous commit is not virtual merge", + BaseCommitForPrevBuiltImageCheckData{ + BuiltCommitLabel: "ae6feb44da273003cfada392c33dfe33748a5e2f", + BuiltVirtualMergeLabel: "false", + BuiltVirtualMergeFromCommitLabel: "", + BuiltVirtualMergeIntoCommitLabel: "", + CurrentCommit: "d2a341ee66189d0927c47a13541ae1926fad05bb", + IsCurrentCommitVirtualMerge: true, + }, + func(baseCommit string, data BaseCommitForPrevBuiltImageCheckData) { + Expect(baseCommit).To(Equal(data.BuiltCommitLabel)) + }), + + Entry("when current commit is not virtual merge and previous commit is virtual merge", + BaseCommitForPrevBuiltImageCheckData{ + BuiltCommitLabel: "a4009816bf7d8f62b2d1cd3331d36f45b3df7d99", + BuiltVirtualMergeLabel: "true", + BuiltVirtualMergeFromCommitLabel: "c1159b9ce05143df3dd6450ee9d749c9642320d2", + BuiltVirtualMergeIntoCommitLabel: "cdb9a2e6ab3fec90f3ea12d4e4728ec4bf3f74dc", + CurrentCommit: "913c4d1bac6ed0265e910616ad40f426d2e6e625", + IsCurrentCommitVirtualMerge: false, + }, + func(baseCommit string, data BaseCommitForPrevBuiltImageCheckData) { + Expect(baseCommit).To(Equal(constructVirtualMergeCommit(data.BuiltVirtualMergeFromCommitLabel, data.BuiltVirtualMergeIntoCommitLabel))) + }), + + Entry("when current commit is not virtual merge and previous commit is not virtual merge", + BaseCommitForPrevBuiltImageCheckData{ + BuiltCommitLabel: "ae6feb44da273003cfada392c33dfe33748a5e2f", + BuiltVirtualMergeLabel: "false", + BuiltVirtualMergeFromCommitLabel: "", + BuiltVirtualMergeIntoCommitLabel: "", + CurrentCommit: "a4009816bf7d8f62b2d1cd3331d36f45b3df7d99", + IsCurrentCommitVirtualMerge: true, + }, + func(baseCommit string, data BaseCommitForPrevBuiltImageCheckData) { + Expect(baseCommit).To(Equal(data.BuiltCommitLabel)) + }), + ) +}) + +type BuiltImageStub struct { + container_runtime.LegacyImageInterface + + name string + stageDescription *image.StageDescription +} + +func NewBuiltImageStub(name string, stageDescription *image.StageDescription) *BuiltImageStub { + return &BuiltImageStub{ + name: name, + stageDescription: stageDescription, + } +} + +func (img *BuiltImageStub) Name() string { + return img.name +} + +func (img *BuiltImageStub) GetStageDescription() *image.StageDescription { + return img.stageDescription +} + +type ConveyorStub struct { + stage.Conveyor + + VirtualMergeOptions stage.VirtualMergeOptions +} + +func NewConveyorStub(virtualMergeOptions stage.VirtualMergeOptions) *ConveyorStub { + return &ConveyorStub{ + VirtualMergeOptions: virtualMergeOptions, + } +} + +func (c *ConveyorStub) GetLocalGitRepoVirtualMergeOptions() stage.VirtualMergeOptions { + return c.VirtualMergeOptions +} + +type GitRepoStub struct { + git_repo.GitRepo + + isLocal bool + name string + headCommitHash string +} + +func NewGitRepoStub(name string, isLocal bool, headCommitHash string) *GitRepoStub { + return &GitRepoStub{ + name: name, + isLocal: isLocal, + headCommitHash: headCommitHash, + } +} + +func (gitRepo *GitRepoStub) GetName() string { + return gitRepo.name +} + +func (gitRepo *GitRepoStub) IsLocal() bool { + return gitRepo.isLocal +} + +func (gitRepo *GitRepoStub) HeadCommitHash(_ context.Context) (string, error) { + return gitRepo.headCommitHash, nil +} + +func (gitRepo *GitRepoStub) CreateDetachedMergeCommit(ctx context.Context, fromCommit, toCommit string) (string, error) { + return constructVirtualMergeCommit(fromCommit, toCommit), nil +} + +func (gitRepo *GitRepoStub) GetMergeCommitParents(ctx context.Context, commit string) ([]string, error) { + return []string{ + constructMergeCommitParentInto(commit), + constructMergeCommitParentFrom(commit), + }, nil +} + +func constructMergeCommitParentFrom(commit string) string { + return fmt.Sprintf("%s-from-commit", commit) +} + +func constructMergeCommitParentInto(commit string) string { + return fmt.Sprintf("%s-into-commit", commit) +} + +func constructVirtualMergeCommit(fromCommit, toCommit string) string { + return fmt.Sprintf("%s-%s", fromCommit, toCommit) +} diff --git a/pkg/build/stage/stage_suite_test.go b/pkg/build/stage/stage_suite_test.go new file mode 100644 index 0000000000..242de36534 --- /dev/null +++ b/pkg/build/stage/stage_suite_test.go @@ -0,0 +1,13 @@ +package stage_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestStage(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Stage Suite") +} diff --git a/pkg/git_repo/git_repo.go b/pkg/git_repo/git_repo.go index 12b1b35ef5..9f990a44ac 100644 --- a/pkg/git_repo/git_repo.go +++ b/pkg/git_repo/git_repo.go @@ -39,6 +39,7 @@ const ( type GitRepo interface { String() string GetName() string + IsLocal() bool CreateDetachedMergeCommit(ctx context.Context, fromCommit, toCommit string) (string, error) GetCommitTreeEntry(ctx context.Context, commit string, path string) (*ls_tree.LsTreeEntry, error) diff --git a/pkg/git_repo/local.go b/pkg/git_repo/local.go index f1e768bb10..8791025729 100644 --- a/pkg/git_repo/local.go +++ b/pkg/git_repo/local.go @@ -115,6 +115,10 @@ func newLocal(ctx context.Context, name, workTreeDir, gitDir string) (l *Local, return l, nil } +func (repo *Local) IsLocal() bool { + return true +} + func (repo *Local) PlainOpen() (*git.Repository, error) { repository, err := git.PlainOpenWithOptions(repo.WorkTreeDir, &git.PlainOpenOptions{EnableDotGitCommonDir: true}) if err != nil { diff --git a/pkg/git_repo/remote.go b/pkg/git_repo/remote.go index 8f980dd47a..3ee0564cb5 100644 --- a/pkg/git_repo/remote.go +++ b/pkg/git_repo/remote.go @@ -36,6 +36,10 @@ func OpenRemoteRepo(name, url string) (*Remote, error) { return repo, repo.ValidateEndpoint() } +func (repo *Remote) IsLocal() bool { + return false +} + func (repo *Remote) ValidateEndpoint() error { if ep, err := transport.NewEndpoint(repo.Url); err != nil { return fmt.Errorf("bad url %q: %s", repo.Url, err) diff --git a/pkg/git_repo/virtual_merge.go b/pkg/git_repo/virtual_merge.go new file mode 100644 index 0000000000..929be0e94c --- /dev/null +++ b/pkg/git_repo/virtual_merge.go @@ -0,0 +1,16 @@ +package git_repo + +import ( + "context" + "fmt" +) + +func GetVirtualMergeParents(ctx context.Context, gitRepo GitRepo, virtualMergeCommit string) (string, string, error) { + if parents, err := gitRepo.GetMergeCommitParents(ctx, virtualMergeCommit); err != nil { + return "", "", err + } else if len(parents) == 2 { + return parents[1], parents[0], nil + } else { + return "", "", fmt.Errorf("got unexpected parents: %v", parents) + } +}