From 34df9d279ef9ed6c3676588ac91fe526f919588f Mon Sep 17 00:00:00 2001 From: Ilya Lesikov Date: Mon, 7 Feb 2022 12:45:09 +0300 Subject: [PATCH] feat(dev-mode): less rebuilds due to better cache handling Single persistent dev-mode service branch instead of multiple. This branch now constantly merges user working branch in itself, which allows for better utilization of cached stages and less rebuilds. `--dev-branch-prefix` flag replaced with `--dev-branch` flag. Signed-off-by: Ilya Lesikov --- cmd/werf/common/common.go | 16 +++++----- pkg/git_repo/local.go | 35 +++++++++------------ pkg/true_git/repository.go | 9 ++++++ pkg/true_git/service_branch.go | 57 ++++++++++++++++++++-------------- 4 files changed, 65 insertions(+), 52 deletions(-) diff --git a/cmd/werf/common/common.go b/cmd/werf/common/common.go index 589a2d5fa0..8387132800 100644 --- a/cmd/werf/common/common.go +++ b/cmd/werf/common/common.go @@ -93,7 +93,7 @@ type CmdData struct { LooseGiterminism *bool Dev *bool DevIgnore *[]string - DevBranchPrefix *string + DevBranch *string IntrospectBeforeError *bool IntrospectAfterError *bool @@ -178,7 +178,7 @@ func SetupGiterminismOptions(cmdData *CmdData, cmd *cobra.Command) { setupLooseGiterminism(cmdData, cmd) setupDev(cmdData, cmd) setupDevIgnore(cmdData, cmd) - setupDevBranchPrefix(cmdData, cmd) + setupDevBranch(cmdData, cmd) } func setupLooseGiterminism(cmdData *CmdData, cmd *cobra.Command) { @@ -198,16 +198,16 @@ func setupDevIgnore(cmdData *CmdData, cmd *cobra.Command) { Also, can be specified with $WERF_DEV_IGNORE_* (e.g. $WERF_DEV_IGNORE_TESTS=*_test.go, $WERF_DEV_IGNORE_DOCS=path/to/docs)`) } -func setupDevBranchPrefix(cmdData *CmdData, cmd *cobra.Command) { - cmdData.DevBranchPrefix = new(string) +func setupDevBranch(cmdData *CmdData, cmd *cobra.Command) { + cmdData.DevBranch = new(string) - defaultValue := "werf-dev-" - envValue := os.Getenv("WERF_DEV_BRANCH_PREFIX") + defaultValue := "_werf-dev" + envValue := os.Getenv("WERF_DEV_BRANCH") if envValue != "" { defaultValue = envValue } - cmd.Flags().StringVarP(cmdData.DevBranchPrefix, "dev-branch-prefix", "", defaultValue, `Set dev git branch prefix (default $WERF_DEV_BRANCH_PREFIX or werf-dev-)`) + cmd.Flags().StringVarP(cmdData.DevBranch, "dev-branch", "", defaultValue, fmt.Sprintf("Set dev git branch name (default $WERF_DEV_BRANCH or %q)", defaultValue)) } func SetupHomeDir(cmdData *CmdData, cmd *cobra.Command) { @@ -1179,7 +1179,7 @@ func GetGiterminismManager(ctx context.Context, cmdData *CmdData) (giterminism_m var openLocalRepoOptions git_repo.OpenLocalRepoOptions if *cmdData.Dev { openLocalRepoOptions.WithServiceHeadCommit = true - openLocalRepoOptions.ServiceBranchOptions.Prefix = *cmdData.DevBranchPrefix + openLocalRepoOptions.ServiceBranchOptions.Name = *cmdData.DevBranch openLocalRepoOptions.ServiceBranchOptions.GlobExcludeList = GetDevIgnore(cmdData) } diff --git a/pkg/git_repo/local.go b/pkg/git_repo/local.go index 25762688d7..0a8de3046c 100644 --- a/pkg/git_repo/local.go +++ b/pkg/git_repo/local.go @@ -40,7 +40,7 @@ type OpenLocalRepoOptions struct { } type ServiceBranchOptions struct { - Prefix string + Name string GlobExcludeList []string } @@ -71,29 +71,22 @@ func OpenLocalRepo(ctx context.Context, name, workTreeDir string, opts OpenLocal defer werf.ReleaseHostLock(lock) } - gitStatusResult, err := l.status(ctx) + devHeadCommit, err := true_git.SyncSourceWorktreeWithServiceBranch( + context.Background(), + l.GitDir, + l.WorkTreeDir, + l.getRepoWorkTreeCacheDir(l.getRepoID()), + l.headCommitHash, + true_git.SyncSourceWorktreeWithServiceBranchOptions{ + ServiceBranch: opts.ServiceBranchOptions.Name, + GlobExcludeList: opts.ServiceBranchOptions.GlobExcludeList, + }, + ) if err != nil { - return nil, fmt.Errorf("unable to get git status: %s", err) + return l, err } - if len(gitStatusResult.PathListWithSubmodules()) != 0 { - devHeadCommit, err := true_git.SyncSourceWorktreeWithServiceBranch( - context.Background(), - l.GitDir, - l.WorkTreeDir, - l.getRepoWorkTreeCacheDir(l.getRepoID()), - l.headCommitHash, - true_git.SyncSourceWorktreeWithServiceBranchOptions{ - ServiceBranchPrefix: opts.ServiceBranchOptions.Prefix, - GlobExcludeList: opts.ServiceBranchOptions.GlobExcludeList, - }, - ) - if err != nil { - return l, err - } - - l.headCommitHash = devHeadCommit - } + l.headCommitHash = devHeadCommit } return l, nil diff --git a/pkg/true_git/repository.go b/pkg/true_git/repository.go index d8f4afba71..0144d2cc75 100644 --- a/pkg/true_git/repository.go +++ b/pkg/true_git/repository.go @@ -57,6 +57,15 @@ func Fetch(ctx context.Context, path string, options FetchOptions) error { return gitCmd.Run(ctx) } +func GetLastBranchCommitSHA(ctx context.Context, repoPath, branch string) (string, error) { + revParseCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: repoPath}, "rev-parse", branch) + if err := revParseCmd.Run(ctx); err != nil { + return "", fmt.Errorf("git rev parse branch command failed: %s", err) + } + + return strings.TrimSpace(revParseCmd.OutBuf.String()), nil +} + func IsShallowClone(ctx context.Context, path string) (bool, error) { if gitVersion.LessThan(semver.MustParse("2.15.0")) { exist, err := util.FileExists(filepath.Join(path, ".git", "shallow")) diff --git a/pkg/true_git/service_branch.go b/pkg/true_git/service_branch.go index fd446cb3db..c0abdd5de7 100644 --- a/pkg/true_git/service_branch.go +++ b/pkg/true_git/service_branch.go @@ -13,8 +13,8 @@ import ( ) type SyncSourceWorktreeWithServiceBranchOptions struct { - ServiceBranchPrefix string - GlobExcludeList []string + ServiceBranch string + GlobExcludeList []string } func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWorktreeDir, worktreeCacheDir, commit string, opts SyncSourceWorktreeWithServiceBranchOptions) (string, error) { @@ -39,10 +39,9 @@ func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWork return fmt.Errorf("unable to remove %s: %s", currentCommitPath, err) } - branchName := fmt.Sprintf("%s%s", opts.ServiceBranchPrefix, commit) - resultCommit, err = syncWorktreeWithServiceWorktreeBranch(ctx, sourceWorktreeDir, serviceWorktreeDir, commit, branchName, opts.GlobExcludeList) + resultCommit, err = syncWorktreeWithServiceWorktreeBranch(ctx, sourceWorktreeDir, serviceWorktreeDir, commit, opts.ServiceBranch, opts.GlobExcludeList) if err != nil { - return fmt.Errorf("unable to sync worktree with service branch %q: %s", branchName, err) + return fmt.Errorf("unable to sync worktree with service branch %q: %s", opts.ServiceBranch, err) } return nil @@ -54,14 +53,13 @@ func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWork } func syncWorktreeWithServiceWorktreeBranch(ctx context.Context, sourceWorktreeDir, serviceWorktreeDir, sourceCommit, branchName string, globExcludeList []string) (string, error) { - serviceBranchHeadCommit, err := getOrPrepareServiceBranchHeadCommit(ctx, serviceWorktreeDir, sourceCommit, branchName) - if err != nil { + if err := prepareAndCheckoutServiceBranch(ctx, serviceWorktreeDir, sourceCommit, branchName); err != nil { return "", fmt.Errorf("unable to get or prepare service branch head commit: %s", err) } - checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", branchName) - if err = checkoutCmd.Run(ctx); err != nil { - return "", fmt.Errorf("git checkout command failed: %s", err) + serviceBranchHeadCommit, err := GetLastBranchCommitSHA(ctx, serviceWorktreeDir, branchName) + if err != nil { + return "", fmt.Errorf("unable to get service worktree commit SHA: %s", err) } revertedChangesExist, err := revertExcludedChangesInServiceWorktreeIndex(ctx, sourceWorktreeDir, serviceWorktreeDir, sourceCommit, serviceBranchHeadCommit, globExcludeList) @@ -90,31 +88,44 @@ func syncWorktreeWithServiceWorktreeBranch(ctx context.Context, sourceWorktreeDi return newCommit, nil } -func getOrPrepareServiceBranchHeadCommit(ctx context.Context, serviceWorktreeDir string, sourceCommit string, branchName string) (string, error) { +func prepareAndCheckoutServiceBranch(ctx context.Context, serviceWorktreeDir string, sourceCommit string, branchName string) error { branchListCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "branch", "--list", branchName) if err := branchListCmd.Run(ctx); err != nil { - return "", fmt.Errorf("git branch list command failed: %s", err) + return fmt.Errorf("git branch list command failed: %s", err) } - var isServiceBranchExist bool - isServiceBranchExist = branchListCmd.OutBuf.Len() != 0 - - if !isServiceBranchExist { + if branchListCmd.OutBuf.Len() == 0 { checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", "-b", branchName, sourceCommit) if err := checkoutCmd.Run(ctx); err != nil { - return "", fmt.Errorf("git checkout command failed: %s", err) + return fmt.Errorf("git checkout command failed: %s", err) } - return sourceCommit, nil + return nil } - revParseCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "rev-parse", branchName) - if err := revParseCmd.Run(ctx); err != nil { - return "", fmt.Errorf("git rev parse branch command failed: %s", err) + checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", branchName) + if err := checkoutCmd.Run(ctx); err != nil { + return fmt.Errorf("git checkout command failed: %s", err) + } + + isSourceCommitInServiceBranch, err := IsAncestor(ctx, sourceCommit, branchName, serviceWorktreeDir) + if err != nil { + return fmt.Errorf("unable to detect whether sourceCommit %q is in service branch: %s", sourceCommit, err) + } + if isSourceCommitInServiceBranch { + return nil + } + + mergeCmd := NewGitCmd( + ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, + "-c", "user.email=werf@werf.io", "-c", "user.name=werf", + "merge", "--no-edit", "--no-ff", "--allow-unrelated-histories", "-s", "recursive", "-X", "theirs", sourceCommit, + ) + if err = mergeCmd.Run(ctx); err != nil { + return fmt.Errorf("git merge of source commit %q into service branch failed: %s", sourceCommit, err) } - serviceBranchHeadCommit := strings.TrimSpace(revParseCmd.OutBuf.String()) - return serviceBranchHeadCommit, nil + return nil } func revertExcludedChangesInServiceWorktreeIndex(ctx context.Context, sourceWorktreeDir string, serviceWorktreeDir string, sourceCommit string, serviceBranchHeadCommit string, globExcludeList []string) (bool, error) {