feat(buildah): native OCI rootless mode; vfs storage driver; bugfixes

Done:

* Added native OCI rootless Buildah mode. Provides more isolation and security
  in comparison to chroot mode. Works only outside of containers.
* $WERF_BUILDAH_MODE instead of $WERF_CONTAINER_RUNTIME_BUILDAN for
  Buidah mode/isolation configuration.
* VFS storage driver now available in addition to OverlayFS for both native and docker-with-fuse
  Buildah modes. Configurable via $WERF_BUILDAH_STORAGE_DRIVER.
* Lots of bugfixes. Minor refactoring. Tests.

Work in progress:

Trying to get rid of a need in external configuration, external dependencies
and work environment preparations to run `werf` in Buildah mode, especially
when running `werf` in containers. As of now no /etc/containers/* configs
needed anymore, lots of configuration built-in, some autodetection/autoconfiguration
implemented.

cmd/werf/common/container_runtime.go

@@ -7,25 +7,78 @@ import (
 	"strings"
 
 	"github.com/werf/werf/pkg/buildah"
+	"github.com/werf/werf/pkg/buildah/types"
 	"github.com/werf/werf/pkg/container_runtime"
 	"github.com/werf/werf/pkg/docker"
+	"github.com/werf/werf/pkg/util"
 )
 
 func ContainerRuntimeProcessStartupHook() (bool, error) {
-	buildahMode := GetContainerRuntimeBuildahMode()
+	buildahMode, _, err := GetBuildahMode()
+	if err != nil {
+		return false, fmt.Errorf("unable to determine buildah mode: %s", err)
+	}
 
 	switch {
-	case buildahMode != "":
-		return buildah.ProcessStartupHook(buildahMode)
+	case *buildahMode != buildah.ModeDisabled:
+		return buildah.ProcessStartupHook(*buildahMode)
 	case strings.HasPrefix(os.Args[0], "buildah-") || strings.HasPrefix(os.Args[0], "chrootuser-") || strings.HasPrefix(os.Args[0], "storage-"):
-		return buildah.ProcessStartupHook("native-rootless")
+		return buildah.ProcessStartupHook(buildah.ModeNativeRootless)
 	}
 
 	return false, nil
 }
 
-func GetContainerRuntimeBuildahMode() buildah.Mode {
-	return buildah.Mode(os.Getenv("WERF_CONTAINER_RUNTIME_BUILDAH"))
+func GetBuildahMode() (*buildah.Mode, *types.Isolation, error) {
+	var (
+		mode      buildah.Mode
+		isolation types.Isolation
+	)
+
+	modeRaw := os.Getenv("WERF_BUILDAH_MODE")
+	switch modeRaw {
+	case "native-rootless":
+		if isInContainer, err := util.IsInContainer(); err != nil {
+			return nil, nil, fmt.Errorf("unable to determine if is in container: %s", err)
+		} else if isInContainer {
+			return nil, nil, fmt.Errorf("native rootless mode is not available in containers: %s", err)
+		}
+		mode = buildah.ModeNativeRootless
+		isolation = types.IsolationOCIRootless
+	case "native-chroot":
+		mode = buildah.ModeNativeRootless
+		isolation = types.IsolationChroot
+	case "docker-with-fuse":
+		mode = buildah.ModeDockerWithFuse
+		isolation = types.IsolationChroot
+	case "default", "auto":
+		mode = buildah.ModeAuto
+		var err error
+		isolation, err = buildah.GetDefaultIsolation()
+		if err != nil {
+			return nil, nil, fmt.Errorf("unable to determine default isolation: %s", err)
+		}
+	case "docker", "":
+		mode = buildah.ModeDisabled
+	default:
+		return nil, nil, fmt.Errorf("unexpected mode specified: %s", modeRaw)
+	}
+
+	return &mode, &isolation, nil
+}
+
+func GetBuildahStorageDriver() (*buildah.StorageDriver, error) {
+	storageDriverRaw := os.Getenv("WERF_BUILDAH_STORAGE_DRIVER")
+	var storageDriver buildah.StorageDriver
+	switch storageDriverRaw {
+	case string(buildah.StorageDriverOverlay), string(buildah.StorageDriverVFS):
+		storageDriver = buildah.StorageDriver(storageDriverRaw)
+	case "default", "auto", "":
+		storageDriver = buildah.DefaultStorageDriver
+	default:
+		return nil, fmt.Errorf("unexpected driver specified: %s", storageDriverRaw)
+	}
+	return &storageDriver, nil
 }
 
 func wrapContainerRuntime(containerRuntime container_runtime.ContainerRuntime) container_runtime.ContainerRuntime {
@@ -36,9 +89,13 @@ func wrapContainerRuntime(containerRuntime container_runtime.ContainerRuntime) c
 }
 
 func InitProcessContainerRuntime(ctx context.Context, cmdData *CmdData) (container_runtime.ContainerRuntime, context.Context, error) {
-	buildahMode := GetContainerRuntimeBuildahMode()
-	if buildahMode != "" {
-		resolvedMode := buildah.ResolveMode(buildahMode)
+	buildahMode, buildahIsolation, err := GetBuildahMode()
+	if err != nil {
+		return nil, ctx, fmt.Errorf("unable to determine buildah mode: %s", err)
+	}
+
+	if *buildahMode != buildah.ModeDisabled {
+		resolvedMode := buildah.ResolveMode(*buildahMode)
 		if resolvedMode == buildah.ModeDockerWithFuse {
 			newCtx, err := InitProcessDocker(ctx, cmdData)
 			if err != nil {
@@ -47,10 +104,17 @@ func InitProcessContainerRuntime(ctx context.Context, cmdData *CmdData) (contain
 			ctx = newCtx
 		}
 
+		storageDriver, err := GetBuildahStorageDriver()
+		if err != nil {
+			return nil, ctx, fmt.Errorf("unable to determine buildah container runtime storage driver: %s", err)
+		}
+
 		insecure := *cmdData.InsecureRegistry || *cmdData.SkipTlsVerifyRegistry
 		b, err := buildah.NewBuildah(resolvedMode, buildah.BuildahOpts{
 			CommonBuildahOpts: buildah.CommonBuildahOpts{
-				Insecure: insecure,
+				Insecure:      insecure,
+				Isolation:     buildahIsolation,
+				StorageDriver: storageDriver,
 			},
 		})
 		if err != nil {

pkg/build/conveyor.go

@@ -399,7 +399,7 @@ func (c *Conveyor) checkContainerRuntimeSupported(ctx context.Context) error {
 
 Please select only dockerfile images or delete all non-dockerfile images from your werf.yaml.
 
-Or disable buildah runtime by unsetting WERF_CONTAINER_RUNTIME_BUILDAH environment variable.`, strings.Join(nonDockerfileImages, ", "))
+Or disable buildah runtime by unsetting WERF_BUILDAH_MODE environment variable.`, strings.Join(nonDockerfileImages, ", "))
 	}
 
 	return nil

pkg/buildah/base.go

@@ -7,28 +7,45 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/werf/werf/pkg/buildah/types"
 	"github.com/werf/werf/pkg/util"
 )
 
 type BaseBuildah struct {
-	TmpDir   string
-	Insecure bool
+	Isolation           types.Isolation
+	TmpDir              string
+	InstanceTmpDir      string
+	SignaturePolicyPath string
+	Insecure            bool
 }
 
 type BaseBuildahOpts struct {
-	Insecure bool
+	Isolation types.Isolation
+	Insecure  bool
 }
 
 func NewBaseBuildah(tmpDir string, opts BaseBuildahOpts) (*BaseBuildah, error) {
 	b := &BaseBuildah{
-		TmpDir:   tmpDir,
-		Insecure: opts.Insecure,
+		Isolation: opts.Isolation,
+		TmpDir:    tmpDir,
+		Insecure:  opts.Insecure,
 	}
 
 	if err := os.MkdirAll(b.TmpDir, os.ModePerm); err != nil {
 		return nil, fmt.Errorf("unable to create dir %q: %s", b.TmpDir, err)
 	}
 
+	var err error
+	b.InstanceTmpDir, err = ioutil.TempDir(b.TmpDir, "instance")
+	if err != nil {
+		return nil, fmt.Errorf("unable to create instance tmp dir: %s", err)
+	}
+
+	b.SignaturePolicyPath = filepath.Join(b.InstanceTmpDir, "policy.json")
+	if err := ioutil.WriteFile(b.SignaturePolicyPath, []byte(DefaultSignaturePolicy), os.ModePerm); err != nil {
+		return nil, fmt.Errorf("unable to write file %q: %s", b.SignaturePolicyPath, err)
+	}
+
 	return b, nil
 }
 

pkg/buildah/buildah.go

@@ -5,19 +5,24 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"runtime"
 
 	"github.com/opencontainers/runtime-spec/specs-go"
 
 	"github.com/werf/werf/pkg/buildah/types"
+	"github.com/werf/werf/pkg/util"
 	"github.com/werf/werf/pkg/werf"
 )
 
 const (
 	DefaultShmSize              = "65536k"
+	DefaultSignaturePolicy      = `{"default": [{"type": "insecureAcceptAnything"}], "transports": {"docker-daemon": {"": [{"type": "insecureAcceptAnything"}]}}}`
 	BuildahImage                = "ghcr.io/werf/buildah:v1.22.3-1"
 	BuildahStorageContainerName = "werf-buildah-storage"
+
+	DefaultStorageDriver StorageDriver = StorageDriverOverlay
 )
 
 type CommonOpts struct {
@@ -74,6 +79,7 @@ type Mode string
 
 const (
 	ModeAuto           Mode = "auto"
+	ModeDisabled       Mode = "disabled"
 	ModeNativeRootless Mode = "native-rootless"
 	ModeDockerWithFuse Mode = "docker-with-fuse"
 )
@@ -89,9 +95,18 @@ func ProcessStartupHook(mode Mode) (bool, error) {
 	}
 }
 
+type StorageDriver string
+
+const (
+	StorageDriverOverlay StorageDriver = "overlay"
+	StorageDriverVFS     StorageDriver = "vfs"
+)
+
 type CommonBuildahOpts struct {
-	TmpDir   string
-	Insecure bool
+	Isolation     *types.Isolation
+	StorageDriver *StorageDriver
+	TmpDir        string
+	Insecure      bool
 }
 
 type NativeRootlessModeOpts struct{}
@@ -105,6 +120,19 @@ type BuildahOpts struct {
 }
 
 func NewBuildah(mode Mode, opts BuildahOpts) (b Buildah, err error) {
+	if opts.CommonBuildahOpts.Isolation == nil {
+		defIsolation, err := GetDefaultIsolation()
+		if err != nil {
+			return b, fmt.Errorf("unable to determine default isolation: %s", err)
+		}
+		opts.CommonBuildahOpts.Isolation = &defIsolation
+	}
+
+	if opts.CommonBuildahOpts.StorageDriver == nil {
+		defStorageDriver := DefaultStorageDriver
+		opts.CommonBuildahOpts.StorageDriver = &defStorageDriver
+	}
+
 	if opts.CommonBuildahOpts.TmpDir == "" {
 		opts.CommonBuildahOpts.TmpDir = filepath.Join(werf.GetHomeDir(), "buildah", "tmp")
 	}
@@ -146,6 +174,33 @@ func ResolveMode(mode Mode) Mode {
 	}
 }
 
+func GetOverlayOptions() ([]string, error) {
+	fuseOverlayBinPath, err := exec.LookPath("fuse-overlayfs")
+	if err != nil {
+		return nil, fmt.Errorf("\"fuse-overlayfs\" binary not found in PATH: %s", err)
+	}
+
+	result := []string{fmt.Sprintf("overlay.mount_program=%s", fuseOverlayBinPath)}
+
+	if isInContainer, err := util.IsInContainer(); err != nil {
+		return nil, fmt.Errorf("unable to determine whether we are in the container: %s", err)
+	} else if isInContainer {
+		result = append(result, fmt.Sprintf("overlay.mountopt=%s", "nodev,fsync=0"))
+	}
+
+	return result, nil
+}
+
+func GetDefaultIsolation() (types.Isolation, error) {
+	if isInContainer, err := util.IsInContainer(); err != nil {
+		return 0, fmt.Errorf("unable to determine if is in container: %s", err)
+	} else if isInContainer {
+		return types.IsolationChroot, nil
+	} else {
+		return types.IsolationOCIRootless, nil
+	}
+}
+
 func debug() bool {
 	return os.Getenv("WERF_BUILDAH_DEBUG") == "1"
 }