From 58e92a28b65f2492b5388582740bcb18f3e26068 Mon Sep 17 00:00:00 2001 From: Ilya Lesikov Date: Tue, 7 Dec 2021 18:28:55 +0300 Subject: [PATCH] feat(buildah): native OCI rootless mode; vfs storage driver; bugfixes Done: * Added native OCI rootless Buildah mode. Provides more isolation and security in comparison to chroot mode. Works only outside of containers. * $WERF_BUILDAH_MODE instead of $WERF_CONTAINER_RUNTIME_BUILDAN for Buidah mode/isolation configuration. * VFS storage driver now available in addition to OverlayFS for both native and docker-with-fuse Buildah modes. Configurable via $WERF_BUILDAH_STORAGE_DRIVER. * Lots of bugfixes. Minor refactoring. Tests. Work in progress: Trying to get rid of a need in external configuration, external dependencies and work environment preparations to run `werf` in Buildah mode, especially when running `werf` in containers. As of now no /etc/containers/* configs needed anymore, lots of configuration built-in, some autodetection/autoconfiguration implemented. --- cmd/werf/common/container_runtime.go | 84 ++++++++++-- pkg/build/conveyor.go | 2 +- pkg/buildah/base.go | 27 +++- pkg/buildah/buildah.go | 59 ++++++++- pkg/buildah/docker_with_fuse_buildah.go | 90 ++++++++++--- pkg/buildah/native_rootless_buildah_linux.go | 120 ++++++++++++++---- pkg/buildah/types/isolation.go | 22 ++++ pkg/buildah/types/storageoptions.go | 31 +++++ pkg/buildah/types/storageoptions_linux.go | 8 ++ pkg/util/linux_container.go | 16 +++ test/e2e/build/build_test.go | 74 ++++++----- test/pkg/contruntime/base.go | 11 +- test/pkg/contruntime/basebuildah.go | 11 ++ test/pkg/contruntime/docker.go | 22 +++- test/pkg/contruntime/dockerwithfusebuildah.go | 44 ++++--- test/pkg/contruntime/interface.go | 21 ++- test/pkg/contruntime/nativerootlessbuildah.go | 41 ++++-- 17 files changed, 548 insertions(+), 135 deletions(-) create mode 100644 pkg/buildah/types/isolation.go create mode 100644 pkg/buildah/types/storageoptions.go create mode 100644 pkg/buildah/types/storageoptions_linux.go create mode 100644 test/pkg/contruntime/basebuildah.go diff --git a/cmd/werf/common/container_runtime.go b/cmd/werf/common/container_runtime.go index 4b3fcb605c..9f7933de60 100644 --- a/cmd/werf/common/container_runtime.go +++ b/cmd/werf/common/container_runtime.go @@ -7,25 +7,78 @@ import ( "strings" "github.com/werf/werf/pkg/buildah" + "github.com/werf/werf/pkg/buildah/types" "github.com/werf/werf/pkg/container_runtime" "github.com/werf/werf/pkg/docker" + "github.com/werf/werf/pkg/util" ) func ContainerRuntimeProcessStartupHook() (bool, error) { - buildahMode := GetContainerRuntimeBuildahMode() + buildahMode, _, err := GetBuildahMode() + if err != nil { + return false, fmt.Errorf("unable to determine buildah mode: %s", err) + } switch { - case buildahMode != "": - return buildah.ProcessStartupHook(buildahMode) + case *buildahMode != buildah.ModeDisabled: + return buildah.ProcessStartupHook(*buildahMode) case strings.HasPrefix(os.Args[0], "buildah-") || strings.HasPrefix(os.Args[0], "chrootuser-") || strings.HasPrefix(os.Args[0], "storage-"): - return buildah.ProcessStartupHook("native-rootless") + return buildah.ProcessStartupHook(buildah.ModeNativeRootless) } return false, nil } -func GetContainerRuntimeBuildahMode() buildah.Mode { - return buildah.Mode(os.Getenv("WERF_CONTAINER_RUNTIME_BUILDAH")) +func GetBuildahMode() (*buildah.Mode, *types.Isolation, error) { + var ( + mode buildah.Mode + isolation types.Isolation + ) + + modeRaw := os.Getenv("WERF_BUILDAH_MODE") + switch modeRaw { + case "native-rootless": + if isInContainer, err := util.IsInContainer(); err != nil { + return nil, nil, fmt.Errorf("unable to determine if is in container: %s", err) + } else if isInContainer { + return nil, nil, fmt.Errorf("native rootless mode is not available in containers: %s", err) + } + mode = buildah.ModeNativeRootless + isolation = types.IsolationOCIRootless + case "native-chroot": + mode = buildah.ModeNativeRootless + isolation = types.IsolationChroot + case "docker-with-fuse": + mode = buildah.ModeDockerWithFuse + isolation = types.IsolationChroot + case "default", "auto": + mode = buildah.ModeAuto + var err error + isolation, err = buildah.GetDefaultIsolation() + if err != nil { + return nil, nil, fmt.Errorf("unable to determine default isolation: %s", err) + } + case "docker", "": + mode = buildah.ModeDisabled + default: + return nil, nil, fmt.Errorf("unexpected mode specified: %s", modeRaw) + } + + return &mode, &isolation, nil +} + +func GetBuildahStorageDriver() (*buildah.StorageDriver, error) { + storageDriverRaw := os.Getenv("WERF_BUILDAH_STORAGE_DRIVER") + var storageDriver buildah.StorageDriver + switch storageDriverRaw { + case string(buildah.StorageDriverOverlay), string(buildah.StorageDriverVFS): + storageDriver = buildah.StorageDriver(storageDriverRaw) + case "default", "auto", "": + storageDriver = buildah.DefaultStorageDriver + default: + return nil, fmt.Errorf("unexpected driver specified: %s", storageDriverRaw) + } + return &storageDriver, nil } func wrapContainerRuntime(containerRuntime container_runtime.ContainerRuntime) container_runtime.ContainerRuntime { @@ -36,9 +89,13 @@ func wrapContainerRuntime(containerRuntime container_runtime.ContainerRuntime) c } func InitProcessContainerRuntime(ctx context.Context, cmdData *CmdData) (container_runtime.ContainerRuntime, context.Context, error) { - buildahMode := GetContainerRuntimeBuildahMode() - if buildahMode != "" { - resolvedMode := buildah.ResolveMode(buildahMode) + buildahMode, buildahIsolation, err := GetBuildahMode() + if err != nil { + return nil, ctx, fmt.Errorf("unable to determine buildah mode: %s", err) + } + + if *buildahMode != buildah.ModeDisabled { + resolvedMode := buildah.ResolveMode(*buildahMode) if resolvedMode == buildah.ModeDockerWithFuse { newCtx, err := InitProcessDocker(ctx, cmdData) if err != nil { @@ -47,10 +104,17 @@ func InitProcessContainerRuntime(ctx context.Context, cmdData *CmdData) (contain ctx = newCtx } + storageDriver, err := GetBuildahStorageDriver() + if err != nil { + return nil, ctx, fmt.Errorf("unable to determine buildah container runtime storage driver: %s", err) + } + insecure := *cmdData.InsecureRegistry || *cmdData.SkipTlsVerifyRegistry b, err := buildah.NewBuildah(resolvedMode, buildah.BuildahOpts{ CommonBuildahOpts: buildah.CommonBuildahOpts{ - Insecure: insecure, + Insecure: insecure, + Isolation: buildahIsolation, + StorageDriver: storageDriver, }, }) if err != nil { diff --git a/pkg/build/conveyor.go b/pkg/build/conveyor.go index c01658afe8..9102270d55 100644 --- a/pkg/build/conveyor.go +++ b/pkg/build/conveyor.go @@ -399,7 +399,7 @@ func (c *Conveyor) checkContainerRuntimeSupported(ctx context.Context) error { Please select only dockerfile images or delete all non-dockerfile images from your werf.yaml. -Or disable buildah runtime by unsetting WERF_CONTAINER_RUNTIME_BUILDAH environment variable.`, strings.Join(nonDockerfileImages, ", ")) +Or disable buildah runtime by unsetting WERF_BUILDAH_MODE environment variable.`, strings.Join(nonDockerfileImages, ", ")) } return nil diff --git a/pkg/buildah/base.go b/pkg/buildah/base.go index 14de98d235..377ac2cbd5 100644 --- a/pkg/buildah/base.go +++ b/pkg/buildah/base.go @@ -7,28 +7,45 @@ import ( "os" "path/filepath" + "github.com/werf/werf/pkg/buildah/types" "github.com/werf/werf/pkg/util" ) type BaseBuildah struct { - TmpDir string - Insecure bool + Isolation types.Isolation + TmpDir string + InstanceTmpDir string + SignaturePolicyPath string + Insecure bool } type BaseBuildahOpts struct { - Insecure bool + Isolation types.Isolation + Insecure bool } func NewBaseBuildah(tmpDir string, opts BaseBuildahOpts) (*BaseBuildah, error) { b := &BaseBuildah{ - TmpDir: tmpDir, - Insecure: opts.Insecure, + Isolation: opts.Isolation, + TmpDir: tmpDir, + Insecure: opts.Insecure, } if err := os.MkdirAll(b.TmpDir, os.ModePerm); err != nil { return nil, fmt.Errorf("unable to create dir %q: %s", b.TmpDir, err) } + var err error + b.InstanceTmpDir, err = ioutil.TempDir(b.TmpDir, "instance") + if err != nil { + return nil, fmt.Errorf("unable to create instance tmp dir: %s", err) + } + + b.SignaturePolicyPath = filepath.Join(b.InstanceTmpDir, "policy.json") + if err := ioutil.WriteFile(b.SignaturePolicyPath, []byte(DefaultSignaturePolicy), os.ModePerm); err != nil { + return nil, fmt.Errorf("unable to write file %q: %s", b.SignaturePolicyPath, err) + } + return b, nil } diff --git a/pkg/buildah/buildah.go b/pkg/buildah/buildah.go index a42e5271d4..1750bec83d 100644 --- a/pkg/buildah/buildah.go +++ b/pkg/buildah/buildah.go @@ -5,19 +5,24 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "runtime" "github.com/opencontainers/runtime-spec/specs-go" "github.com/werf/werf/pkg/buildah/types" + "github.com/werf/werf/pkg/util" "github.com/werf/werf/pkg/werf" ) const ( DefaultShmSize = "65536k" + DefaultSignaturePolicy = `{"default": [{"type": "insecureAcceptAnything"}], "transports": {"docker-daemon": {"": [{"type": "insecureAcceptAnything"}]}}}` BuildahImage = "ghcr.io/werf/buildah:v1.22.3-1" BuildahStorageContainerName = "werf-buildah-storage" + + DefaultStorageDriver StorageDriver = StorageDriverOverlay ) type CommonOpts struct { @@ -74,6 +79,7 @@ type Mode string const ( ModeAuto Mode = "auto" + ModeDisabled Mode = "disabled" ModeNativeRootless Mode = "native-rootless" ModeDockerWithFuse Mode = "docker-with-fuse" ) @@ -89,9 +95,18 @@ func ProcessStartupHook(mode Mode) (bool, error) { } } +type StorageDriver string + +const ( + StorageDriverOverlay StorageDriver = "overlay" + StorageDriverVFS StorageDriver = "vfs" +) + type CommonBuildahOpts struct { - TmpDir string - Insecure bool + Isolation *types.Isolation + StorageDriver *StorageDriver + TmpDir string + Insecure bool } type NativeRootlessModeOpts struct{} @@ -105,6 +120,19 @@ type BuildahOpts struct { } func NewBuildah(mode Mode, opts BuildahOpts) (b Buildah, err error) { + if opts.CommonBuildahOpts.Isolation == nil { + defIsolation, err := GetDefaultIsolation() + if err != nil { + return b, fmt.Errorf("unable to determine default isolation: %s", err) + } + opts.CommonBuildahOpts.Isolation = &defIsolation + } + + if opts.CommonBuildahOpts.StorageDriver == nil { + defStorageDriver := DefaultStorageDriver + opts.CommonBuildahOpts.StorageDriver = &defStorageDriver + } + if opts.CommonBuildahOpts.TmpDir == "" { opts.CommonBuildahOpts.TmpDir = filepath.Join(werf.GetHomeDir(), "buildah", "tmp") } @@ -146,6 +174,33 @@ func ResolveMode(mode Mode) Mode { } } +func GetOverlayOptions() ([]string, error) { + fuseOverlayBinPath, err := exec.LookPath("fuse-overlayfs") + if err != nil { + return nil, fmt.Errorf("\"fuse-overlayfs\" binary not found in PATH: %s", err) + } + + result := []string{fmt.Sprintf("overlay.mount_program=%s", fuseOverlayBinPath)} + + if isInContainer, err := util.IsInContainer(); err != nil { + return nil, fmt.Errorf("unable to determine whether we are in the container: %s", err) + } else if isInContainer { + result = append(result, fmt.Sprintf("overlay.mountopt=%s", "nodev,fsync=0")) + } + + return result, nil +} + +func GetDefaultIsolation() (types.Isolation, error) { + if isInContainer, err := util.IsInContainer(); err != nil { + return 0, fmt.Errorf("unable to determine if is in container: %s", err) + } else if isInContainer { + return types.IsolationChroot, nil + } else { + return types.IsolationOCIRootless, nil + } +} + func debug() bool { return os.Getenv("WERF_BUILDAH_DEBUG") == "1" } diff --git a/pkg/buildah/docker_with_fuse_buildah.go b/pkg/buildah/docker_with_fuse_buildah.go index 13c15e79b8..cea4ea04ed 100644 --- a/pkg/buildah/docker_with_fuse_buildah.go +++ b/pkg/buildah/docker_with_fuse_buildah.go @@ -21,17 +21,35 @@ import ( type DockerWithFuseBuildah struct { BaseBuildah + + commonBuildahCliArgs []string +} + +type StoreOptions struct { + GraphDriverName string + GraphDriverOptions []string } func NewDockerWithFuseBuildah(commonOpts CommonBuildahOpts, opts DockerWithFuseModeOpts) (*DockerWithFuseBuildah, error) { b := &DockerWithFuseBuildah{} - baseBuildah, err := NewBaseBuildah(commonOpts.TmpDir, BaseBuildahOpts{Insecure: commonOpts.Insecure}) + baseBuildah, err := NewBaseBuildah(commonOpts.TmpDir, BaseBuildahOpts{ + Isolation: *commonOpts.Isolation, + Insecure: commonOpts.Insecure, + }) if err != nil { return nil, fmt.Errorf("unable to create BaseBuildah: %s", err) } b.BaseBuildah = *baseBuildah + // TODO: remove this string and mount the previously generated policy.json file inside of a docker-with-fuse container + b.SignaturePolicyPath = "/etc/containers/policy.json" + + b.commonBuildahCliArgs, err = GetCommonBuildahCliArgs(*commonOpts.StorageDriver) + if err != nil { + return nil, fmt.Errorf("unable to get common Buildah cli args: %s", err) + } + return b, nil } @@ -41,9 +59,11 @@ func (b *DockerWithFuseBuildah) Tag(ctx context.Context, ref, newRef string, opt } func (b *DockerWithFuseBuildah) Push(ctx context.Context, ref string, opts PushOpts) error { - _, _, err := b.runBuildah(ctx, []string{}, []string{ - "push", fmt.Sprintf("--tls-verify=%s", strconv.FormatBool(!b.Insecure)), ref, fmt.Sprintf("docker://%s", ref), - }, opts.LogWriter) + args := []string{ + "push", fmt.Sprintf("--tls-verify=%s", strconv.FormatBool(!b.Insecure)), + "--signature-policy", b.SignaturePolicyPath, ref, fmt.Sprintf("docker://%s", ref), + } + _, _, err := b.runBuildah(ctx, []string{}, args, opts.LogWriter) return err } @@ -70,23 +90,22 @@ func (b *DockerWithFuseBuildah) BuildFromDockerfile(ctx context.Context, dockerf } }() - var buildArgs []string + buildArgs := []string{ + "bud", "--isolation", b.Isolation.String(), "--format", "docker", + fmt.Sprintf("--tls-verify=%s", strconv.FormatBool(!b.Insecure)), "--signature-policy", b.SignaturePolicyPath, + } for k, v := range opts.BuildArgs { buildArgs = append(buildArgs, "--build-arg", fmt.Sprintf("%s=%s", k, v)) } + buildArgs = append(buildArgs, "-f", "/.werf/buildah/tmp/Dockerfile") // NOTE: it is principal to use cli option --tls-verify=true|false form with equality sign, instead of separate arguments (--tls-verify true|false), because --tls-verify is by itself a boolean argument - budArgs := []string{"bud", "--format", "docker", fmt.Sprintf("--tls-verify=%s", strconv.FormatBool(!b.Insecure))} - budArgs = append(budArgs, buildArgs...) - budArgs = append(budArgs, "-f", "/.werf/buildah/tmp/Dockerfile") - output, _, err := b.runBuildah( - ctx, - []string{ + ctx, []string{ "--volume", fmt.Sprintf("%s:/.werf/buildah/tmp", sessionTmpDir), "--workdir", "/.werf/buildah/tmp/context", }, - budArgs, opts.LogWriter, + buildArgs, opts.LogWriter, ) if err != nil { return "", err @@ -104,13 +123,14 @@ func (b *DockerWithFuseBuildah) RunCommand(ctx context.Context, container string func (b *DockerWithFuseBuildah) FromCommand(ctx context.Context, container string, image string, opts FromCommandOpts) (string, error) { _, _, err := b.runBuildah(ctx, []string{}, []string{ - "from", fmt.Sprintf("--tls-verify=%s", strconv.FormatBool(!b.Insecure)), "--name", container, image, + "from", fmt.Sprintf("--tls-verify=%s", strconv.FormatBool(!b.Insecure)), "--name", container, + "--signature-policy", b.SignaturePolicyPath, "--isolation", b.Isolation.String(), "--format", "docker", image, }, opts.LogWriter) // FIXME: return container name return "", err } -// TODO(ilya-lesikov): make it more generic to handle not only images +// TODO: make it more generic to handle not only images func (b *DockerWithFuseBuildah) Inspect(ctx context.Context, ref string) (*types.BuilderInfo, error) { stdout, stderr, err := b.runBuildah(ctx, []string{}, []string{"inspect", "--type", "image", ref}, nil) if err != nil { @@ -129,7 +149,11 @@ func (b *DockerWithFuseBuildah) Inspect(ctx context.Context, ref string) (*types } func (b *DockerWithFuseBuildah) Pull(ctx context.Context, ref string, opts PullOpts) error { - _, _, err := b.runBuildah(ctx, []string{}, []string{"pull", fmt.Sprintf("--tls-verify=%s", strconv.FormatBool(!b.Insecure)), ref}, opts.LogWriter) + args := []string{ + "pull", fmt.Sprintf("--tls-verify=%s", strconv.FormatBool(!b.Insecure)), + "--signature-policy", b.SignaturePolicyPath, ref, + } + _, _, err := b.runBuildah(ctx, []string{}, args, opts.LogWriter) return err } @@ -166,6 +190,7 @@ func (b *DockerWithFuseBuildah) runBuildah(ctx context.Context, dockerArgs []str args := []string{"--rm"} args = append(args, dockerArgs...) args = append(args, BuildahWithFuseDockerArgs(BuildahStorageContainerName, docker.DockerConfigDir)...) + args = append(args, b.commonBuildahCliArgs...) args = append(args, buildahArgs...) if debug() { @@ -222,6 +247,41 @@ func BuildahWithFuseDockerArgs(storageContainerName, dockerConfigDir string) []s } } +func GetCommonBuildahCliArgs(driver StorageDriver) ([]string, error) { + var result []string + + cliStoreOpts, err := newBuildahCliStoreOptions(driver) + if err != nil { + return result, fmt.Errorf("unable to get buildah cli store options: %s", err) + } + + if cliStoreOpts.GraphDriverName != "" { + result = append(result, "--storage-driver", cliStoreOpts.GraphDriverName) + } + + if len(cliStoreOpts.GraphDriverOptions) > 0 { + result = append(result, "--storage-opt", strings.Join(cliStoreOpts.GraphDriverOptions, ",")) + } + + return result, nil +} + +func newBuildahCliStoreOptions(driver StorageDriver) (*StoreOptions, error) { + var graphDriverOptions []string + if driver == StorageDriverOverlay { + overlayOpts, err := GetOverlayOptions() + if err != nil { + return nil, fmt.Errorf("unable to get overlay options: %s", err) + } + graphDriverOptions = append(graphDriverOptions, overlayOpts...) + } + + return &StoreOptions{ + GraphDriverName: string(driver), + GraphDriverOptions: graphDriverOptions, + }, nil +} + func scanLines(data string) []string { var lines []string diff --git a/pkg/buildah/native_rootless_buildah_linux.go b/pkg/buildah/native_rootless_buildah_linux.go index c02044c93c..cb09c7b7a0 100644 --- a/pkg/buildah/native_rootless_buildah_linux.go +++ b/pkg/buildah/native_rootless_buildah_linux.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os" + "path/filepath" "strings" "time" @@ -17,10 +18,11 @@ import ( "github.com/containers/buildah/imagebuildah" "github.com/containers/common/libimage" "github.com/containers/image/v5/manifest" - is "github.com/containers/image/v5/storage" + imgstor "github.com/containers/image/v5/storage" "github.com/containers/image/v5/transports/alltransports" imgtypes "github.com/containers/image/v5/types" "github.com/containers/storage" + "github.com/containers/storage/pkg/homedir" "github.com/containers/storage/pkg/reexec" "github.com/containers/storage/pkg/unshare" "github.com/hashicorp/go-multierror" @@ -59,23 +61,33 @@ type NativeRootlessBuildah struct { func NewNativeRootlessBuildah(commonOpts CommonBuildahOpts, opts NativeRootlessModeOpts) (*NativeRootlessBuildah, error) { b := &NativeRootlessBuildah{} - baseBuildah, err := NewBaseBuildah(commonOpts.TmpDir, BaseBuildahOpts{Insecure: commonOpts.Insecure}) + baseBuildah, err := NewBaseBuildah(commonOpts.TmpDir, BaseBuildahOpts{ + Isolation: *commonOpts.Isolation, + Insecure: commonOpts.Insecure, + }) if err != nil { return nil, fmt.Errorf("unable to create BaseBuildah: %s", err) } b.BaseBuildah = *baseBuildah - storeOpts, err := storage.DefaultStoreOptions(unshare.IsRootless(), unshare.GetRootlessUID()) + storeOpts, err := NewNativeStoreOptions(unshare.GetRootlessUID(), *commonOpts.StorageDriver) if err != nil { - return nil, fmt.Errorf("unable to set default storage opts: %s", err) + return nil, fmt.Errorf("unable to initialize storage opts: %s", err) } - b.Store, err = storage.GetStore(storeOpts) + + b.Store, err = storage.GetStore(storage.StoreOptions(*storeOpts)) if err != nil { return nil, fmt.Errorf("unable to get storage: %s", err) } - is.Transport.SetStore(b.Store) - runtime, err := libimage.RuntimeFromStore(b.Store, &libimage.RuntimeOptions{}) + imgstor.Transport.SetStore(b.Store) + runtime, err := libimage.RuntimeFromStore(b.Store, &libimage.RuntimeOptions{ + SystemContext: &imgtypes.SystemContext{ + OCIInsecureSkipTLSVerify: b.Insecure, + DockerInsecureSkipTLSVerify: imgtypes.NewOptionalBool(b.Insecure), + DockerDaemonInsecureSkipTLSVerify: b.Insecure, + }, + }) if err != nil { return nil, fmt.Errorf("error getting runtime from store: %s", err) } @@ -114,11 +126,12 @@ func (b *NativeRootlessBuildah) Tag(_ context.Context, ref, newRef string, opts func (b *NativeRootlessBuildah) Push(ctx context.Context, ref string, opts PushOpts) error { pushOpts := buildah.PushOptions{ - Compression: define.Gzip, - Store: b.Store, - ManifestType: manifest.DockerV2Schema2MediaType, - MaxRetries: MaxPullPushRetries, - RetryDelay: PullPushRetryDelay, + Compression: define.Gzip, + Store: b.Store, + ManifestType: manifest.DockerV2Schema2MediaType, + MaxRetries: MaxPullPushRetries, + RetryDelay: PullPushRetryDelay, + SignaturePolicyPath: b.SignaturePolicyPath, SystemContext: &imgtypes.SystemContext{ OCIInsecureSkipTLSVerify: b.Insecure, DockerInsecureSkipTLSVerify: imgtypes.NewOptionalBool(b.Insecure), @@ -144,11 +157,14 @@ func (b *NativeRootlessBuildah) Push(ctx context.Context, ref string, opts PushO func (b *NativeRootlessBuildah) BuildFromDockerfile(ctx context.Context, dockerfile []byte, opts BuildFromDockerfileOpts) (string, error) { buildOpts := define.BuildOptions{ - Isolation: define.IsolationChroot, + Isolation: define.Isolation(b.Isolation), + // FIXME(ilya-lesikov): + // IDMappingOptions: nil, OutputFormat: buildah.Dockerv2ImageManifest, CommonBuildOpts: &define.CommonBuildOptions{ ShmSize: DefaultShmSize, }, + SignaturePolicyPath: b.SignaturePolicyPath, SystemContext: &imgtypes.SystemContext{ OCIInsecureSkipTLSVerify: b.Insecure, DockerInsecureSkipTLSVerify: imgtypes.NewOptionalBool(b.Insecure), @@ -204,8 +220,9 @@ func (b *NativeRootlessBuildah) Umount(ctx context.Context, container string, op func (b *NativeRootlessBuildah) RunCommand(ctx context.Context, container string, command []string, opts RunCommandOpts) error { runOpts := buildah.RunOptions{ - Args: opts.Args, - Mounts: opts.Mounts, + Args: opts.Args, + Isolation: define.Isolation(b.Isolation), + Mounts: opts.Mounts, } stderr := &bytes.Buffer{} @@ -242,10 +259,11 @@ func (b *NativeRootlessBuildah) FromCommand(ctx context.Context, container strin func (b *NativeRootlessBuildah) Pull(ctx context.Context, ref string, opts PullOpts) error { pullOpts := buildah.PullOptions{ - Store: b.Store, - MaxRetries: MaxPullPushRetries, - RetryDelay: PullPushRetryDelay, - PullPolicy: define.PullIfNewer, + Store: b.Store, + MaxRetries: MaxPullPushRetries, + RetryDelay: PullPushRetryDelay, + PullPolicy: define.PullIfNewer, + SignaturePolicyPath: b.SignaturePolicyPath, SystemContext: &imgtypes.SystemContext{ OCIInsecureSkipTLSVerify: b.Insecure, DockerInsecureSkipTLSVerify: imgtypes.NewOptionalBool(b.Insecure), @@ -266,8 +284,8 @@ func (b *NativeRootlessBuildah) Pull(ctx context.Context, ref string, opts PullO func (b *NativeRootlessBuildah) Rmi(ctx context.Context, ref string, opts RmiOpts) error { _, rmiErrors := b.Runtime.RemoveImages(ctx, []string{ref}, &libimage.RemoveImagesOptions{ - // REVIEW(ilya-lesikov): readonly=false is default, is it ok? Filters: []string{"readonly=false", "intermediate=false", "dangling=true"}, + Force: opts.Force, }) var multiErr *multierror.Error @@ -279,7 +297,7 @@ func (b *NativeRootlessBuildah) getImage(ref string) (*libimage.Image, error) { ManifestList: true, }) if err != nil { - fmt.Errorf("error looking up image %q: %s", ref, err) + return nil, fmt.Errorf("error looking up image %q: %s", ref, err) } return image, nil @@ -288,7 +306,13 @@ func (b *NativeRootlessBuildah) getImage(ref string) (*libimage.Image, error) { // getImageBuilder returns nil, nil if image not found. func (b *NativeRootlessBuildah) getImageBuilder(ctx context.Context, imgName string) (builder *buildah.Builder, err error) { builder, err = buildah.ImportBuilderFromImage(ctx, b.Store, buildah.ImportFromImageOptions{ - Image: imgName, + Image: imgName, + SignaturePolicyPath: b.SignaturePolicyPath, + SystemContext: &imgtypes.SystemContext{ + OCIInsecureSkipTLSVerify: b.Insecure, + DockerInsecureSkipTLSVerify: imgtypes.NewOptionalBool(b.Insecure), + DockerDaemonInsecureSkipTLSVerify: b.Insecure, + }, }) switch { case err != nil && strings.HasSuffix(err.Error(), storage.ErrImageUnknown.Error()): @@ -318,3 +342,55 @@ func (b *NativeRootlessBuildah) openContainerBuilder(ctx context.Context, contai return builder, err } + +func NewNativeStoreOptions(rootlessUID int, driver StorageDriver) (*types.StoreOptions, error) { + var ( + runRoot string + err error + ) + + if rootlessUID == 0 { + runRoot = "/run/containers/storage" + } else { + runRoot, err = storage.GetRootlessRuntimeDir(rootlessUID) + if err != nil { + return nil, fmt.Errorf("unable to get runtime dir: %s", err) + } + } + + home, err := homedir.GetDataHome() + if err != nil { + return nil, fmt.Errorf("unable to get HOME data dir: %s", err) + } + + rootlessStoragePath := filepath.Join(home, "containers", "storage") + + var graphRoot string + if rootlessUID == 0 { + graphRoot = "/var/lib/containers/storage" + } else { + graphRoot = rootlessStoragePath + } + + var graphDriverOptions []string + if driver == StorageDriverOverlay { + overlayOpts, err := GetOverlayOptions() + if err != nil { + return nil, fmt.Errorf("unable to get overlay options: %s", err) + } + graphDriverOptions = append(graphDriverOptions, overlayOpts...) + } + + return &types.StoreOptions{ + RunRoot: runRoot, + GraphRoot: graphRoot, + RootlessStoragePath: rootlessStoragePath, + GraphDriverName: string(driver), + GraphDriverOptions: graphDriverOptions, + // FIXME(ilya-lesikov): + // AutoNsMinSize: 2000, + // AutoNsMaxSize: 50000, + // UIDMap: nil, + // GIDMap: nil, + }, nil +} diff --git a/pkg/buildah/types/isolation.go b/pkg/buildah/types/isolation.go new file mode 100644 index 0000000000..101e822b7e --- /dev/null +++ b/pkg/buildah/types/isolation.go @@ -0,0 +1,22 @@ +package types + +import ( + "fmt" +) + +type Isolation int + +const ( + IsolationChroot Isolation = 2 + IsolationOCIRootless Isolation = 3 +) + +func (i Isolation) String() string { + switch i { + case IsolationChroot: + return "chroot" + case IsolationOCIRootless: + return "rootless" + } + return fmt.Sprintf("unrecognized isolation type %d", i) +} diff --git a/pkg/buildah/types/storageoptions.go b/pkg/buildah/types/storageoptions.go new file mode 100644 index 0000000000..0f7a656e40 --- /dev/null +++ b/pkg/buildah/types/storageoptions.go @@ -0,0 +1,31 @@ +// This is from "github.com/containers/storage". +//go:build !linux +// +build !linux + +package types + +// IDMap contains a single entry for user namespace range remapping. An array +// of IDMap entries represents the structure that will be provided to the Linux +// kernel for creating a user namespace. +type IDMap struct { + ContainerID int `json:"container_id"` + HostID int `json:"host_id"` + Size int `json:"size"` +} + +// StoreOptions is used for passing initialization options to GetStore(), for +// initializing a Store object and the underlying storage that it controls. +type StoreOptions struct { + RunRoot string `json:"runroot,omitempty"` + GraphRoot string `json:"root,omitempty"` + RootlessStoragePath string `toml:"rootless_storage_path"` + GraphDriverName string `json:"driver,omitempty"` + GraphDriverOptions []string `json:"driver-options,omitempty"` + UIDMap []IDMap `json:"uidmap,omitempty"` + GIDMap []IDMap `json:"gidmap,omitempty"` + RootAutoNsUser string `json:"root_auto_ns_user,omitempty"` + AutoNsMinSize uint32 `json:"auto_userns_min_size,omitempty"` + AutoNsMaxSize uint32 `json:"auto_userns_max_size,omitempty"` + PullOptions map[string]string `toml:"pull_options"` + DisableVolatile bool `json:"disable-volatile,omitempty"` +} diff --git a/pkg/buildah/types/storageoptions_linux.go b/pkg/buildah/types/storageoptions_linux.go new file mode 100644 index 0000000000..12b8160ada --- /dev/null +++ b/pkg/buildah/types/storageoptions_linux.go @@ -0,0 +1,8 @@ +//go:build linux +// +build linux + +package types + +import "github.com/containers/storage" + +type StoreOptions storage.StoreOptions diff --git a/pkg/util/linux_container.go b/pkg/util/linux_container.go index 88b6cd667a..5d6abb69ce 100644 --- a/pkg/util/linux_container.go +++ b/pkg/util/linux_container.go @@ -35,3 +35,19 @@ func ToLinuxContainerPath(path string) string { ), ) } + +func IsInContainer() (bool, error) { + if dockerEnvExist, err := RegularFileExists("/.dockerenv"); err != nil { + return false, fmt.Errorf("unable to check for /.dockerenv existence: %s", err) + } else if dockerEnvExist { + return true, nil + } + + if containerEnvExist, err := RegularFileExists("/run/.containerenv"); err != nil { + return false, fmt.Errorf("unable to check for /run/.containerenv existence: %s", err) + } else if containerEnvExist { + return true, nil + } + + return false, nil +} diff --git a/test/e2e/build/build_test.go b/test/e2e/build/build_test.go index 9e106272f8..1b0ed554c2 100644 --- a/test/e2e/build/build_test.go +++ b/test/e2e/build/build_test.go @@ -14,10 +14,10 @@ import ( var _ = Describe("Build", func() { DescribeTable("should succeed and produce expected image", - func(withLocalRepo bool, containerRuntime string) { + func(withLocalRepo bool, buildahMode string) { By("initializing") - setupEnv(withLocalRepo, containerRuntime) - contRuntime, err := contruntime.NewContainerRuntime(containerRuntime) + setupEnv(withLocalRepo, buildahMode) + contRuntime, err := contruntime.NewContainerRuntime(buildahMode) if err == contruntime.ErrRuntimeUnavailable { Skip(err.Error()) } else if err != nil { @@ -49,9 +49,8 @@ var _ = Describe("Build", func() { config := contRuntime.GetImageInspectConfig(buildReport.Images["dockerfile"].DockerImageName) By("state0: checking built images metadata") - // FIXME(ilya-lesikov): CHANGED_ARG not changed on Native Buildah, needs investigation, then uncomment - // Expect(config.Env).To(ContainElement("COMPOSED_ENV=env-was_changed")) - // Expect(config.Labels).To(HaveKeyWithValue("COMPOSED_LABEL", "label-was_changed")) + Expect(config.Env).To(ContainElement("COMPOSED_ENV=env-was_changed")) + Expect(config.Labels).To(HaveKeyWithValue("COMPOSED_LABEL", "label-was_changed")) Expect(config.Shell).To(ContainElements("/bin/sh", "-c")) Expect(config.User).To(Equal("0:0")) Expect(config.WorkingDir).To(Equal("/")) @@ -101,13 +100,11 @@ var _ = Describe("Build", func() { )) By(`state1: getting built "dockerfile" image metadata`) - // FIXME(ilya-lesikov): CHANGED_ARG not changed on Native Buildah, needs investigation, then uncomment - // config := contRuntime.GetImageInspectConfig(buildReport.Images["dockerfile"].DockerImageName) + config := contRuntime.GetImageInspectConfig(buildReport.Images["dockerfile"].DockerImageName) By("state1: checking built images metadata") - // FIXME(ilya-lesikov): CHANGED_ARG not changed on Native Buildah, needs investigation, then uncomment - // Expect(config.Env).To(ContainElement("COMPOSED_ENV=env-was_changed-state1")) - // Expect(config.Labels).To(HaveKeyWithValue("COMPOSED_LABEL", "label-was_changed-state1")) + Expect(config.Env).To(ContainElement("COMPOSED_ENV=env-was_changed-state1")) + Expect(config.Labels).To(HaveKeyWithValue("COMPOSED_LABEL", "label-was_changed-state1")) By("state1: checking built images content") contRuntime.ExpectCmdsToSucceed( @@ -127,39 +124,48 @@ var _ = Describe("Build", func() { ) } }, - Entry("without repo using Docker", false, "docker"), - Entry("with local repo using Docker", true, "docker"), - Entry("with local repo using Native Rootless Buildah", true, "native-rootless-buildah"), - Entry("with local repo using Docker-With-Fuse Buildah", true, "docker-with-fuse-buildah"), + Entry( + "without repo using Docker", + false, "docker", + ), + Entry( + "with local repo using Docker", + true, "docker", + ), + Entry( + "with local repo using Native Buildah with rootless isolation", + true, "native-rootless", + ), + Entry( + "with local repo using Native Buildah with chroot isolation", + true, "native-chroot", + ), + Entry( + "with local repo using Docker-With-Fuse Buildah with chroot isolation", + true, "docker-with-fuse", + ), // TODO: uncomment when buildah allows building without --repo flag // Entry("without repo using Native Rootless Buildah", false, contruntime.NativeRootlessBuildah), // Entry("without repo using Docker-With-Fuse Buildah", false, contruntime.DockerWithFuseBuildah), ) }) -func setupEnv(withLocalRepo bool, containerRuntime string) { - switch containerRuntime { - case "docker": - if withLocalRepo { - SuiteData.Stubs.SetEnv("WERF_REPO", strings.Join([]string{SuiteData.RegistryLocalAddress, SuiteData.ProjectName}, "/")) - } - SuiteData.Stubs.UnsetEnv("WERF_CONTAINER_RUNTIME_BUILDAH") - case "native-rootless-buildah": - if withLocalRepo { - SuiteData.Stubs.SetEnv("WERF_REPO", strings.Join([]string{SuiteData.RegistryInternalAddress, SuiteData.ProjectName}, "/")) - } - SuiteData.Stubs.SetEnv("WERF_CONTAINER_RUNTIME_BUILDAH", "native-rootless") - case "docker-with-fuse-buildah": - if withLocalRepo { - SuiteData.Stubs.SetEnv("WERF_REPO", strings.Join([]string{SuiteData.RegistryInternalAddress, SuiteData.ProjectName}, "/")) - } - SuiteData.Stubs.SetEnv("WERF_CONTAINER_RUNTIME_BUILDAH", "docker-with-fuse") - default: - panic("unexpected containerRuntime") +func setupEnv(withLocalRepo bool, buildahMode string) { + SuiteData.Stubs.SetEnv("WERF_BUILDAH_MODE", buildahMode) + + if withLocalRepo && buildahMode == "docker" { + SuiteData.Stubs.SetEnv("WERF_REPO", strings.Join([]string{SuiteData.RegistryLocalAddress, SuiteData.ProjectName}, "/")) + } else if withLocalRepo { + SuiteData.Stubs.SetEnv("WERF_REPO", strings.Join([]string{SuiteData.RegistryInternalAddress, SuiteData.ProjectName}, "/")) + } else { + SuiteData.Stubs.UnsetEnv("WERF_REPO") } if withLocalRepo { SuiteData.Stubs.SetEnv("WERF_INSECURE_REGISTRY", "1") SuiteData.Stubs.SetEnv("WERF_SKIP_TLS_VERIFY_REGISTRY", "1") + } else { + SuiteData.Stubs.UnsetEnv("WERF_INSECURE_REGISTRY") + SuiteData.Stubs.UnsetEnv("WERF_SKIP_TLS_VERIFY_REGISTRY") } } diff --git a/test/pkg/contruntime/base.go b/test/pkg/contruntime/base.go index d28cdcd1af..ac8c03d8ce 100644 --- a/test/pkg/contruntime/base.go +++ b/test/pkg/contruntime/base.go @@ -3,17 +3,14 @@ package contruntime import ( "github.com/google/uuid" - "github.com/werf/werf/test/pkg/thirdparty/contruntime/manifest" + "github.com/werf/werf/pkg/buildah/types" ) -type BuildahInspect struct { - Docker struct { - Config manifest.Schema2Config `json:"config"` - } `json:"Docker"` +type BaseContainerRuntime struct { + CommonCliArgs []string + Isolation types.Isolation } -type BaseContainerRuntime struct{} - func expectCmdsToSucceed(r ContainerRuntime, image string, cmds ...string) { containerName := uuid.New().String() r.RunSleepingContainer(containerName, image) diff --git a/test/pkg/contruntime/basebuildah.go b/test/pkg/contruntime/basebuildah.go new file mode 100644 index 0000000000..7261e4575a --- /dev/null +++ b/test/pkg/contruntime/basebuildah.go @@ -0,0 +1,11 @@ +package contruntime + +import ( + "github.com/werf/werf/test/pkg/thirdparty/contruntime/manifest" +) + +type BuildahInspect struct { + Docker struct { + Config manifest.Schema2Config `json:"config"` + } `json:"Docker"` +} diff --git a/test/pkg/contruntime/docker.go b/test/pkg/contruntime/docker.go index 9123e1b32d..462cb36c0c 100644 --- a/test/pkg/contruntime/docker.go +++ b/test/pkg/contruntime/docker.go @@ -22,27 +22,35 @@ func (r *DockerRuntime) ExpectCmdsToSucceed(image string, cmds ...string) { } func (r *DockerRuntime) RunSleepingContainer(containerName, image string) { - utils.RunSucceedCommand("/", - "docker", "run", "--rm", "-d", "--entrypoint=", "--name", containerName, image, "tail", "-f", "/dev/null", - ) + args := r.CommonCliArgs + args = append(args, "run", "--rm", "-d", "--entrypoint=", "--name", containerName, image, "tail", "-f", "/dev/null") + utils.RunSucceedCommand("/", "docker", args...) } func (r *DockerRuntime) Exec(containerName string, cmds ...string) { for _, cmd := range cmds { - utils.RunSucceedCommand("/", "docker", "exec", containerName, "sh", "-ec", cmd) + args := r.CommonCliArgs + args = append(args, "exec", containerName, "sh", "-ec", cmd) + utils.RunSucceedCommand("/", "docker", args...) } } func (r *DockerRuntime) Rm(containerName string) { - utils.RunSucceedCommand("/", "docker", "rm", "-fv", containerName) + args := r.CommonCliArgs + args = append(args, "rm", "-fv", containerName) + utils.RunSucceedCommand("/", "docker", args...) } func (r *DockerRuntime) Pull(image string) { - utils.RunSucceedCommand("/", "docker", "pull", image) + args := r.CommonCliArgs + args = append(args, "pull", image) + utils.RunSucceedCommand("/", "docker", args...) } func (r *DockerRuntime) GetImageInspectConfig(image string) (config manifest.Schema2Config) { - configRaw, err := utils.RunCommand("/", "docker", "image", "inspect", "-f", "{{ json .Config }}", image) + args := r.CommonCliArgs + args = append(args, "image", "inspect", "-f", "{{ json .Config }}", image) + configRaw, err := utils.RunCommand("/", "docker", args...) Expect(err).NotTo(HaveOccurred()) Expect(json.Unmarshal(configRaw, &config)).To(Succeed()) diff --git a/test/pkg/contruntime/dockerwithfusebuildah.go b/test/pkg/contruntime/dockerwithfusebuildah.go index 8ef578f6c9..30efd1c613 100644 --- a/test/pkg/contruntime/dockerwithfusebuildah.go +++ b/test/pkg/contruntime/dockerwithfusebuildah.go @@ -9,11 +9,27 @@ import ( "github.com/werf/werf/integration/pkg/utils" "github.com/werf/werf/pkg/buildah" + "github.com/werf/werf/pkg/buildah/types" "github.com/werf/werf/test/pkg/thirdparty/contruntime/manifest" ) -func NewDockerWithFuseBuildahRuntime() ContainerRuntime { - return &DockerWithFuseBuildahRuntime{} +func NewDockerWithFuseBuildahRuntime(isolation types.Isolation, storageDriver buildah.StorageDriver) ContainerRuntime { + home, err := os.UserHomeDir() + Expect(err).NotTo(HaveOccurred()) + + commonCliArgs := append([]string{"run", "--rm"}, buildah.BuildahWithFuseDockerArgs(buildah.BuildahStorageContainerName, filepath.Join(home, ".docker"))...) + + commonBuildahCliArgs, err := buildah.GetCommonBuildahCliArgs(storageDriver) + Expect(err).NotTo(HaveOccurred()) + + commonCliArgs = append(commonCliArgs, commonBuildahCliArgs...) + + return &DockerWithFuseBuildahRuntime{ + BaseContainerRuntime: BaseContainerRuntime{ + CommonCliArgs: commonCliArgs, + Isolation: isolation, + }, + } } type DockerWithFuseBuildahRuntime struct { @@ -25,31 +41,36 @@ func (r *DockerWithFuseBuildahRuntime) ExpectCmdsToSucceed(image string, cmds .. } func (r *DockerWithFuseBuildahRuntime) RunSleepingContainer(containerName, image string) { - args := append(buildahDockerWithFuseDockerArgs(), "from", "--tls-verify=false", "--format", "docker", "--name", containerName, image) + args := r.CommonCliArgs + args = append(args, "from", "--tls-verify=false", "--isolation", r.Isolation.String(), "--format", "docker", "--name", containerName, image) utils.RunSucceedCommand("/", "docker", args...) } func (r *DockerWithFuseBuildahRuntime) Exec(containerName string, cmds ...string) { for _, cmd := range cmds { - args := append(buildahDockerWithFuseDockerArgs(), "run", containerName, "--", "sh", "-ec", cmd) + args := r.CommonCliArgs + args = append(args, "run", "--isolation", r.Isolation.String(), containerName, "--", "sh", "-ec", cmd) utils.RunSucceedCommand("/", "docker", args...) } } func (r *DockerWithFuseBuildahRuntime) Rm(containerName string) { - args := append(buildahDockerWithFuseDockerArgs(), "rm", containerName) + args := r.CommonCliArgs + args = append(args, "rm", containerName) utils.RunSucceedCommand("/", "docker", args...) } func (r *DockerWithFuseBuildahRuntime) Pull(image string) { - args := append(buildahDockerWithFuseDockerArgs(), "pull", "--tls-verify=false", image) + args := r.CommonCliArgs + args = append(args, "pull", "--tls-verify=false", image) utils.RunSucceedCommand("/", "docker", args...) } func (r *DockerWithFuseBuildahRuntime) GetImageInspectConfig(image string) (config manifest.Schema2Config) { r.Pull(image) - args := append(buildahDockerWithFuseDockerArgs(), "inspect", "--type", "image", image) + args := r.CommonCliArgs + args = append(args, "inspect", "--type", "image", image) inspectRaw, err := utils.RunCommand("/", "docker", args...) Expect(err).NotTo(HaveOccurred()) @@ -57,12 +78,3 @@ func (r *DockerWithFuseBuildahRuntime) GetImageInspectConfig(image string) (conf Expect(json.Unmarshal(inspectRaw, &inspect)).To(Succeed()) return inspect.Docker.Config } - -func buildahDockerWithFuseDockerArgs() []string { - home, err := os.UserHomeDir() - Expect(err).NotTo(HaveOccurred()) - - args := []string{"run", "--rm"} - - return append(args, buildah.BuildahWithFuseDockerArgs(buildah.BuildahStorageContainerName, filepath.Join(home, ".docker"))...) -} diff --git a/test/pkg/contruntime/interface.go b/test/pkg/contruntime/interface.go index acc8d8b9b3..a0c5c2e511 100644 --- a/test/pkg/contruntime/interface.go +++ b/test/pkg/contruntime/interface.go @@ -5,24 +5,31 @@ import ( "fmt" "runtime" + "github.com/werf/werf/pkg/buildah" + bdTypes "github.com/werf/werf/pkg/buildah/types" "github.com/werf/werf/test/pkg/thirdparty/contruntime/manifest" ) var ErrRuntimeUnavailable = errors.New("requested runtime unavailable") -func NewContainerRuntime(name string) (ContainerRuntime, error) { - switch name { +func NewContainerRuntime(buildahMode string) (ContainerRuntime, error) { + switch buildahMode { case "docker": return NewDockerRuntime(), nil - case "native-rootless-buildah": + case "native-rootless": if runtime.GOOS != "linux" { return nil, ErrRuntimeUnavailable } - return NewNativeRootlessBuildahRuntime(), nil - case "docker-with-fuse-buildah": - return NewDockerWithFuseBuildahRuntime(), nil + return NewNativeRootlessBuildahRuntime(bdTypes.IsolationOCIRootless, buildah.DefaultStorageDriver), nil + case "native-chroot": + if runtime.GOOS != "linux" { + return nil, ErrRuntimeUnavailable + } + return NewNativeRootlessBuildahRuntime(bdTypes.IsolationChroot, buildah.DefaultStorageDriver), nil + case "docker-with-fuse": + return NewDockerWithFuseBuildahRuntime(bdTypes.IsolationChroot, buildah.DefaultStorageDriver), nil default: - panic(fmt.Sprint("unexpected name for container runtime: ", name)) + panic(fmt.Sprintf("unexpected buildah mode: %s", buildahMode)) } } diff --git a/test/pkg/contruntime/nativerootlessbuildah.go b/test/pkg/contruntime/nativerootlessbuildah.go index 52b9f94fdb..970e09f9d3 100644 --- a/test/pkg/contruntime/nativerootlessbuildah.go +++ b/test/pkg/contruntime/nativerootlessbuildah.go @@ -6,11 +6,25 @@ import ( . "github.com/onsi/gomega" "github.com/werf/werf/integration/pkg/utils" + "github.com/werf/werf/pkg/buildah" + "github.com/werf/werf/pkg/buildah/types" "github.com/werf/werf/test/pkg/thirdparty/contruntime/manifest" ) -func NewNativeRootlessBuildahRuntime() ContainerRuntime { - return &NativeRootlessBuildahRuntime{} +func NewNativeRootlessBuildahRuntime(isolation types.Isolation, storageDriver buildah.StorageDriver) ContainerRuntime { + var commonCliArgs []string + + commonBuildahCliArgs, err := buildah.GetCommonBuildahCliArgs(storageDriver) + Expect(err).NotTo(HaveOccurred()) + + commonCliArgs = append(commonCliArgs, commonBuildahCliArgs...) + + return &NativeRootlessBuildahRuntime{ + BaseContainerRuntime: BaseContainerRuntime{ + CommonCliArgs: commonCliArgs, + Isolation: isolation, + }, + } } type NativeRootlessBuildahRuntime struct { @@ -22,30 +36,39 @@ func (r *NativeRootlessBuildahRuntime) ExpectCmdsToSucceed(image string, cmds .. } func (r *NativeRootlessBuildahRuntime) RunSleepingContainer(containerName, image string) { - utils.RunSucceedCommand("/", - "buildah", "from", "--tls-verify=false", "--format", "docker", "--name", containerName, image, - ) + args := r.CommonCliArgs + args = append(args, "from", "--tls-verify=false", "--isolation", r.Isolation.String(), "--format", "docker", "--name", containerName, image) + utils.RunSucceedCommand("/", "buildah", args...) } func (r *NativeRootlessBuildahRuntime) Exec(containerName string, cmds ...string) { for _, cmd := range cmds { - utils.RunSucceedCommand("/", "buildah", "run", containerName, "--", "sh", "-ec", cmd) + args := r.CommonCliArgs + args = append(args, "run", "--isolation", r.Isolation.String(), containerName, "--", "sh", "-ec", cmd) + utils.RunSucceedCommand("/", "buildah", args...) } } func (r *NativeRootlessBuildahRuntime) Rm(containerName string) { - utils.RunSucceedCommand("/", "buildah", "rm", containerName) + args := r.CommonCliArgs + args = append(args, "rm", containerName) + utils.RunSucceedCommand("/", "buildah", args...) } func (r *NativeRootlessBuildahRuntime) Pull(image string) { - utils.RunSucceedCommand("/", "buildah", "pull", "--tls-verify=false", image) + args := r.CommonCliArgs + args = append(args, "pull", "--tls-verify=false", image) + utils.RunSucceedCommand("/", "buildah", args...) } func (r *NativeRootlessBuildahRuntime) GetImageInspectConfig(image string) (config manifest.Schema2Config) { r.Pull(image) - inspectRaw, err := utils.RunCommand("/", "buildah", "inspect", "--type", "image", image) + args := r.CommonCliArgs + args = append(args, "inspect", "--type", "image", image) + inspectRaw, err := utils.RunCommand("/", "buildah", args...) Expect(err).NotTo(HaveOccurred()) + var inspect BuildahInspect Expect(json.Unmarshal(inspectRaw, &inspect)).To(Succeed())