Merge pull request #4389 from werf/refactor-buildah-backend-switch-to…

…-copyrec

refactor(buildah_backend): switch to copy-recursive module

go.mod

@@ -55,7 +55,7 @@ require (
 	github.com/mvdan/xurls v1.1.0 // indirect
 	github.com/oleiade/reflections v1.0.1 // indirect
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/ginkgo/v2 v2.0.0
+	github.com/onsi/ginkgo/v2 v2.1.3
 	github.com/onsi/gomega v1.18.1
 	github.com/opencontainers/go-digest v1.0.0
 	github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84
@@ -72,6 +72,7 @@ require (
 	github.com/spf13/pflag v1.0.5
 	github.com/theupdateframework/notary v0.6.1 // indirect
 	github.com/tonistiigi/go-rosetta v0.0.0-20200727161949-f79598599c5d // indirect
+	github.com/werf/copy-recurse v0.2.2
 	github.com/werf/kubedog v0.6.4-0.20220222141823-4ca722ade0ef
 	github.com/werf/lockgate v0.0.0-20200729113342-ec2c142f71ea
 	github.com/werf/logboek v0.5.4

go.sum

@@ -1631,8 +1631,9 @@ github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9k
 github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.0.0 h1:CcuG/HvWNkkaqCUpJifQY8z7qEMBJya6aLPx6ftGyjQ=
 github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
+github.com/onsi/ginkgo/v2 v2.1.3 h1:e/3Cwtogj0HA+25nMP1jCMDIf8RtRYbGwGGuBIFztkc=
+github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c=
 github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -2058,10 +2059,10 @@ github.com/vmware/govmomi v0.20.3/go.mod h1:URlwyTFZX72RmxtxuaFL2Uj3fD1JTvZdx59b
 github.com/weppos/publicsuffix-go v0.4.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
 github.com/weppos/publicsuffix-go v0.5.0 h1:rutRtjBJViU/YjcI5d80t4JAVvDltS6bciJg2K1HrLU=
 github.com/weppos/publicsuffix-go v0.5.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
-github.com/werf/3p-helm/v3 v3.0.0-20220408112844-a661f7cd4c64 h1:SuB96B8WTpKdBjHSc3uyCJiX3YcQJEc9HyZD4kzURzo=
-github.com/werf/3p-helm/v3 v3.0.0-20220408112844-a661f7cd4c64/go.mod h1:Nm0Z2ciZFFvR9cRKpiRE2SMhJTgqY0b+ezT2cDcyqNw=
 github.com/werf/3p-helm/v3 v3.0.0-20220419131239-a2df9b725de3 h1:H3C8D/30GQ71laOCsLGJrIHZMAa4Joab48hkOfTH8KA=
 github.com/werf/3p-helm/v3 v3.0.0-20220419131239-a2df9b725de3/go.mod h1:Nm0Z2ciZFFvR9cRKpiRE2SMhJTgqY0b+ezT2cDcyqNw=
+github.com/werf/copy-recurse v0.2.2 h1:OpBB+Ezsv7j+iQR02p7zUQXSefZ7UaKBtQPMg2dxi7M=
+github.com/werf/copy-recurse v0.2.2/go.mod h1:KVHSQ90p19xflWW0B7BJhLBwmSbEtuxIaBnjlUYRPhk=
 github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f h1:81YscYTF9mmTf0ULOsCmm42YWQp+qWDzWi1HjWniZrg=
 github.com/werf/helm v0.0.0-20210202111118-81e74d46da0f/go.mod h1:OMONwLWU9zEENgaVjWEX+M+xik2QakejzKHG1+6mnUo=
 github.com/werf/kubedog v0.6.4-0.20220222141823-4ca722ade0ef h1:jidfI8MH4qRvWHlxGw06VKWiKRdBIfGFcjQ3pGwsquc=

pkg/container_backend/buildah_backend.go

@@ -7,7 +7,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/fs"
 	"os"
 	"path/filepath"
 	"sort"
@@ -16,8 +15,8 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/otiai10/copy"
 
+	"github.com/werf/copy-recurse"
 	"github.com/werf/logboek"
 	"github.com/werf/werf/pkg/buildah"
 	"github.com/werf/werf/pkg/image"
@@ -336,12 +335,29 @@ func (runtime *BuildahBackend) applyDependenciesImports(ctx context.Context, con
 				ExcludeGlobs: imp.ExcludePaths,
 			})
 
-			if err := copyFromTo(ctx, uid, gid, absFrom, absTo, pathMatcher); err != nil {
-				return fmt.Errorf("error copying dependency import files from %q to %q: %w", absFrom, absTo, err)
+			copyRec, err := copyrec.New(absFrom, absTo, copyrec.Options{
+				MatchDir: func(path string) (copyrec.DirAction, error) {
+					switch {
+					case pathMatcher.IsPathMatched(path):
+						return copyrec.DirMatch, nil
+					case pathMatcher.ShouldGoThrough(path):
+						return copyrec.DirFallThrough, nil
+					default:
+						return copyrec.DirSkip, nil
+					}
+				},
+				MatchFile: func(path string) (bool, error) {
+					return pathMatcher.IsPathMatched(path), err
+				},
+				UID: uid,
+				GID: gid,
+			})
+			if err != nil {
+				return fmt.Errorf("error creating recursive copy command: %w", err)
 			}
 
-			if err := updateOwner(absTo, uid, gid); err != nil && !errors.Is(err, fs.ErrNotExist) {
-				return fmt.Errorf("error updating file ownersnip: %w", err)
+			if err := copyRec.Run(ctx); err != nil {
+				return fmt.Errorf("error copying dependency import files from %q to %q: %w", absFrom, absTo, err)
 			}
 		}
 	}
@@ -602,110 +618,6 @@ func makeBuildahMounts(volumes []string) ([]specs.Mount, error) {
 	return mounts, nil
 }
 
-func copyFromTo(ctx context.Context, uid *uint32, gid *uint32, absFrom, absTo string, pathMatcher path_matcher.PathMatcher) error {
-	if err := walkPath(absFrom, func(rootPathIsFile bool, relSrc string, dirEntry *fs.DirEntry, err error) error {
-		if rootPathIsFile {
-			logboek.Context(ctx).Debug().LogF("Copying dependency import %q to %q\n", absFrom, absTo)
-			if err := copy.Copy(absFrom, absTo); err != nil {
-				return fmt.Errorf("error copying file %q to %q: %w", absFrom, absTo, err)
-			}
-			return nil
-		}
-
-		if err != nil {
-			return fmt.Errorf("error while walking dir %q: %w", absFrom, err)
-		}
-
-		absSrc := filepath.Join(absFrom, relSrc)
-		absDst := filepath.Join(absTo, relSrc)
-
-		srcFileInfo, err := (*dirEntry).Info()
-		if errors.Is(err, fs.ErrNotExist) {
-			return fmt.Errorf("file %q was renamed or removed while we were trying to get info about it", absSrc)
-		} else if err != nil {
-			return fmt.Errorf("error getting info about file %q", absSrc)
-		}
-
-		if srcFileInfo.IsDir() {
-			if !pathMatcher.IsDirOrSubmodulePathMatched(relSrc) {
-				return fs.SkipDir
-			}
-
-			if pathMatcher.ShouldGoThrough(relSrc) {
-				if err := os.MkdirAll(relSrc, srcFileInfo.Mode()); err != nil {
-					return fmt.Errorf("error creating directory %q: %w", absSrc, err)
-				}
-				return nil
-			}
-		} else if !pathMatcher.IsPathMatched(relSrc) {
-			return nil
-		}
-
-		logboek.Context(ctx).Debug().LogF("Copying dependency import %q to %q\n", absSrc, absDst)
-		if err := copy.Copy(absSrc, absDst); err != nil {
-			return fmt.Errorf("error copying file %q to %q: %w", absSrc, absDst, err)
-		}
-
-		if err := updateOwner(absDst, uid, gid); err != nil {
-			return fmt.Errorf("error updating file ownership: %w", err)
-		}
-
-		if srcFileInfo.IsDir() {
-			return fs.SkipDir
-		}
-
-		return nil
-	}); err != nil {
-		return fmt.Errorf("error walking dependency import source root: %w", err)
-	}
-	return nil
-}
-
-func walkPath(path string, fn func(rootPathIsFile bool, entryRelPath string, dirEntry *fs.DirEntry, err error) error) error {
-	fileInfo, err := os.Lstat(path)
-	if err != nil {
-		return fmt.Errorf("error getting file info for path %q: %w", path, err)
-	}
-
-	if !fileInfo.IsDir() {
-		return fn(true, "", nil, nil)
-	} else {
-		rootFs := os.DirFS(path)
-		if err := fs.WalkDir(rootFs, ".", func(relSrc string, entry fs.DirEntry, err error) error {
-			return fn(false, relSrc, &entry, err)
-		}); err != nil {
-			return fmt.Errorf("error walking directory %q: %w", rootFs, err)
-		}
-		return nil
-	}
-}
-
-func updateOwner(path string, uid *uint32, gid *uint32) error {
-	if uid == nil && gid == nil {
-		return nil
-	}
-
-	if err := walkPath(path, func(rootPathIsFile bool, entryRelPath string, dirEntry *fs.DirEntry, err error) error {
-		if rootPathIsFile {
-			return os.Lchown(path, uint32PtrUIDOrGIDToInt(uid), uint32PtrUIDOrGIDToInt(gid))
-		}
-
-		return os.Lchown(filepath.Join(path, entryRelPath), uint32PtrUIDOrGIDToInt(uid), uint32PtrUIDOrGIDToInt(gid))
-	}); err != nil {
-		return fmt.Errorf("error walking path %q: %w", path, err)
-	}
-
-	return nil
-}
-
-func uint32PtrUIDOrGIDToInt(uidOrGid *uint32) int {
-	if uidOrGid == nil {
-		return -1
-	}
-
-	return int(*uidOrGid)
-}
-
 func getUIDAndGID(userNameOrUID, groupNameOrGID, fsRoot string) (*uint32, *uint32, error) {
 	uid, err := getUID(userNameOrUID, fsRoot)
 	if err != nil {