Skip to content

Commit

Permalink
feat(dev-mode): less rebuilds due to better cache handling
Browse files Browse the repository at this point in the history 
Single persistent dev-mode service branch instead of multiple. This
branch now constantly merges user working branch in itself, which allows
for better utilization of cached stages and less rebuilds.

`--dev-branch-prefix` flag replaced with `--dev-branch` flag.

Signed-off-by: Ilya Lesikov <ilya@lesikov.com>
  • Loading branch information
@ilya-lesikov @alexey-igrychev
ilya-lesikov authored and alexey-igrychev committed Feb 7, 2022
1 parent 3a87b24 commit 34df9d2
Show file tree
Hide file tree
Showing 4 changed files with 65 additions and 52 deletions.
16 changes: 8 additions & 8 deletions cmd/werf/common/common.go
View file
Expand Up @@ -93,7 +93,7 @@ type CmdData struct {
LooseGiterminism *bool
Dev *bool
DevIgnore *[]string
DevBranchPrefix *string
DevBranch *string

IntrospectBeforeError *bool
IntrospectAfterError *bool
Expand Down Expand Up @@ -178,7 +178,7 @@ func SetupGiterminismOptions(cmdData *CmdData, cmd *cobra.Command) {
setupLooseGiterminism(cmdData, cmd)
setupDev(cmdData, cmd)
setupDevIgnore(cmdData, cmd)
setupDevBranchPrefix(cmdData, cmd)
setupDevBranch(cmdData, cmd)
}

func setupLooseGiterminism(cmdData *CmdData, cmd *cobra.Command) {
Expand All @@ -198,16 +198,16 @@ func setupDevIgnore(cmdData *CmdData, cmd *cobra.Command) {
Also, can be specified with $WERF_DEV_IGNORE_* (e.g. $WERF_DEV_IGNORE_TESTS=*_test.go, $WERF_DEV_IGNORE_DOCS=path/to/docs)`)
}

func setupDevBranchPrefix(cmdData *CmdData, cmd *cobra.Command) {
cmdData.DevBranchPrefix = new(string)
func setupDevBranch(cmdData *CmdData, cmd *cobra.Command) {
cmdData.DevBranch = new(string)

defaultValue := "werf-dev-"
envValue := os.Getenv("WERF_DEV_BRANCH_PREFIX")
defaultValue := "_werf-dev"
envValue := os.Getenv("WERF_DEV_BRANCH")
if envValue != "" {
defaultValue = envValue
}

cmd.Flags().StringVarP(cmdData.DevBranchPrefix, "dev-branch-prefix", "", defaultValue, `Set dev git branch prefix (default $WERF_DEV_BRANCH_PREFIX or werf-dev-)`)
cmd.Flags().StringVarP(cmdData.DevBranch, "dev-branch", "", defaultValue, fmt.Sprintf("Set dev git branch name (default $WERF_DEV_BRANCH or %q)", defaultValue))
}

func SetupHomeDir(cmdData *CmdData, cmd *cobra.Command) {
Expand Down Expand Up @@ -1179,7 +1179,7 @@ func GetGiterminismManager(ctx context.Context, cmdData *CmdData) (giterminism_m
var openLocalRepoOptions git_repo.OpenLocalRepoOptions
if *cmdData.Dev {
openLocalRepoOptions.WithServiceHeadCommit = true
openLocalRepoOptions.ServiceBranchOptions.Prefix = *cmdData.DevBranchPrefix
openLocalRepoOptions.ServiceBranchOptions.Name = *cmdData.DevBranch
openLocalRepoOptions.ServiceBranchOptions.GlobExcludeList = GetDevIgnore(cmdData)
}

Expand Down
35 changes: 14 additions & 21 deletions pkg/git_repo/local.go
View file
Expand Up @@ -40,7 +40,7 @@ type OpenLocalRepoOptions struct {
}

type ServiceBranchOptions struct {
Prefix string
Name string
GlobExcludeList []string
}

Expand Down Expand Up @@ -71,29 +71,22 @@ func OpenLocalRepo(ctx context.Context, name, workTreeDir string, opts OpenLocal
defer werf.ReleaseHostLock(lock)
}

gitStatusResult, err := l.status(ctx)
devHeadCommit, err := true_git.SyncSourceWorktreeWithServiceBranch(
context.Background(),
l.GitDir,
l.WorkTreeDir,
l.getRepoWorkTreeCacheDir(l.getRepoID()),
l.headCommitHash,
true_git.SyncSourceWorktreeWithServiceBranchOptions{
ServiceBranch: opts.ServiceBranchOptions.Name,
GlobExcludeList: opts.ServiceBranchOptions.GlobExcludeList,
},
)
if err != nil {
return nil, fmt.Errorf("unable to get git status: %s", err)
return l, err
}

if len(gitStatusResult.PathListWithSubmodules()) != 0 {
devHeadCommit, err := true_git.SyncSourceWorktreeWithServiceBranch(
context.Background(),
l.GitDir,
l.WorkTreeDir,
l.getRepoWorkTreeCacheDir(l.getRepoID()),
l.headCommitHash,
true_git.SyncSourceWorktreeWithServiceBranchOptions{
ServiceBranchPrefix: opts.ServiceBranchOptions.Prefix,
GlobExcludeList: opts.ServiceBranchOptions.GlobExcludeList,
},
)
if err != nil {
return l, err
}

l.headCommitHash = devHeadCommit
}
l.headCommitHash = devHeadCommit
}

return l, nil
Expand Down
9 changes: 9 additions & 0 deletions pkg/true_git/repository.go
View file
Expand Up @@ -57,6 +57,15 @@ func Fetch(ctx context.Context, path string, options FetchOptions) error {
return gitCmd.Run(ctx)
}

func GetLastBranchCommitSHA(ctx context.Context, repoPath, branch string) (string, error) {
revParseCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: repoPath}, "rev-parse", branch)
if err := revParseCmd.Run(ctx); err != nil {
return "", fmt.Errorf("git rev parse branch command failed: %s", err)
}

return strings.TrimSpace(revParseCmd.OutBuf.String()), nil
}

func IsShallowClone(ctx context.Context, path string) (bool, error) {
if gitVersion.LessThan(semver.MustParse("2.15.0")) {
exist, err := util.FileExists(filepath.Join(path, ".git", "shallow"))
Expand Down
57 changes: 34 additions & 23 deletions pkg/true_git/service_branch.go
View file
Expand Up @@ -13,8 +13,8 @@ import (
)

type SyncSourceWorktreeWithServiceBranchOptions struct {
ServiceBranchPrefix string
GlobExcludeList []string
ServiceBranch string
GlobExcludeList []string
}

func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWorktreeDir, worktreeCacheDir, commit string, opts SyncSourceWorktreeWithServiceBranchOptions) (string, error) {
Expand All @@ -39,10 +39,9 @@ func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWork
return fmt.Errorf("unable to remove %s: %s", currentCommitPath, err)
}

branchName := fmt.Sprintf("%s%s", opts.ServiceBranchPrefix, commit)
resultCommit, err = syncWorktreeWithServiceWorktreeBranch(ctx, sourceWorktreeDir, serviceWorktreeDir, commit, branchName, opts.GlobExcludeList)
resultCommit, err = syncWorktreeWithServiceWorktreeBranch(ctx, sourceWorktreeDir, serviceWorktreeDir, commit, opts.ServiceBranch, opts.GlobExcludeList)
if err != nil {
return fmt.Errorf("unable to sync worktree with service branch %q: %s", branchName, err)
return fmt.Errorf("unable to sync worktree with service branch %q: %s", opts.ServiceBranch, err)
}

return nil
Expand All @@ -54,14 +53,13 @@ func SyncSourceWorktreeWithServiceBranch(ctx context.Context, gitDir, sourceWork
}

func syncWorktreeWithServiceWorktreeBranch(ctx context.Context, sourceWorktreeDir, serviceWorktreeDir, sourceCommit, branchName string, globExcludeList []string) (string, error) {
serviceBranchHeadCommit, err := getOrPrepareServiceBranchHeadCommit(ctx, serviceWorktreeDir, sourceCommit, branchName)
if err != nil {
if err := prepareAndCheckoutServiceBranch(ctx, serviceWorktreeDir, sourceCommit, branchName); err != nil {
return "", fmt.Errorf("unable to get or prepare service branch head commit: %s", err)
}

checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", branchName)
if err = checkoutCmd.Run(ctx); err != nil {
return "", fmt.Errorf("git checkout command failed: %s", err)
serviceBranchHeadCommit, err := GetLastBranchCommitSHA(ctx, serviceWorktreeDir, branchName)
if err != nil {
return "", fmt.Errorf("unable to get service worktree commit SHA: %s", err)
}

revertedChangesExist, err := revertExcludedChangesInServiceWorktreeIndex(ctx, sourceWorktreeDir, serviceWorktreeDir, sourceCommit, serviceBranchHeadCommit, globExcludeList)
Expand Down Expand Up @@ -90,31 +88,44 @@ func syncWorktreeWithServiceWorktreeBranch(ctx context.Context, sourceWorktreeDi
return newCommit, nil
}

func getOrPrepareServiceBranchHeadCommit(ctx context.Context, serviceWorktreeDir string, sourceCommit string, branchName string) (string, error) {
func prepareAndCheckoutServiceBranch(ctx context.Context, serviceWorktreeDir string, sourceCommit string, branchName string) error {
branchListCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "branch", "--list", branchName)
if err := branchListCmd.Run(ctx); err != nil {
return "", fmt.Errorf("git branch list command failed: %s", err)
return fmt.Errorf("git branch list command failed: %s", err)
}

var isServiceBranchExist bool
isServiceBranchExist = branchListCmd.OutBuf.Len() != 0

if !isServiceBranchExist {
if branchListCmd.OutBuf.Len() == 0 {
checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", "-b", branchName, sourceCommit)
if err := checkoutCmd.Run(ctx); err != nil {
return "", fmt.Errorf("git checkout command failed: %s", err)
return fmt.Errorf("git checkout command failed: %s", err)
}

return sourceCommit, nil
return nil
}

revParseCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "rev-parse", branchName)
if err := revParseCmd.Run(ctx); err != nil {
return "", fmt.Errorf("git rev parse branch command failed: %s", err)
checkoutCmd := NewGitCmd(ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir}, "checkout", branchName)
if err := checkoutCmd.Run(ctx); err != nil {
return fmt.Errorf("git checkout command failed: %s", err)
}

isSourceCommitInServiceBranch, err := IsAncestor(ctx, sourceCommit, branchName, serviceWorktreeDir)
if err != nil {
return fmt.Errorf("unable to detect whether sourceCommit %q is in service branch: %s", sourceCommit, err)
}
if isSourceCommitInServiceBranch {
return nil
}

mergeCmd := NewGitCmd(
ctx, &GitCmdOptions{RepoDir: serviceWorktreeDir},
"-c", "user.email=werf@werf.io", "-c", "user.name=werf",
"merge", "--no-edit", "--no-ff", "--allow-unrelated-histories", "-s", "recursive", "-X", "theirs", sourceCommit,
)
if err = mergeCmd.Run(ctx); err != nil {
return fmt.Errorf("git merge of source commit %q into service branch failed: %s", sourceCommit, err)
}

serviceBranchHeadCommit := strings.TrimSpace(revParseCmd.OutBuf.String())
return serviceBranchHeadCommit, nil
return nil
}

func revertExcludedChangesInServiceWorktreeIndex(ctx context.Context, sourceWorktreeDir string, serviceWorktreeDir string, sourceCommit string, serviceBranchHeadCommit string, globExcludeList []string) (bool, error) {
Expand Down

0 comments on commit 34df9d2

Please sign in to comment.