diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a666c4df8e..92509435f0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -4,9 +4,15 @@ on: - pull_request env: - GO_VERSION: "1.21.x" + GO_VERSION: "oldstable" + + GO_BUILD_CMD: "go build \"-ldflags=-s -w\" -trimpath" + GO_BUILD_TEST_CMD: "go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional" + GOTESTSUM_VERSION: "latest" - GOTESTCMD: "gotestsum --format standard-verbose --debug --" + + GOTESTSUM_CMD: "gotestsum --format standard-verbose --debug --" + GOTESTSUM_CMD_RAW: "gotestsum --format standard-verbose --debug --raw-command -- go tool test2json -t" jobs: lint: @@ -280,19 +286,19 @@ jobs: run: go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }} - name: Test standard security policy - run: ${{ env.GOTESTCMD }} -timeout=30m -gcflags=all=-d=checkptr ./pkg/securitypolicy/... + run: ${{ env.GOTESTSUM_CMD }} -timeout=30m -gcflags=all=-d=checkptr ./pkg/securitypolicy/... - name: Test rego security policy - run: ${{ env.GOTESTCMD }} -tags=rego -timeout=30m -gcflags=all=-d=checkptr ./pkg/securitypolicy/... + run: ${{ env.GOTESTSUM_CMD }} -tags=rego -timeout=30m -gcflags=all=-d=checkptr ./pkg/securitypolicy/... - name: Test rego policy interpreter - run: ${{ env.GOTESTCMD }} -gcflags=all=-d=checkptr ./internal/regopolicyinterpreter/... + run: ${{ env.GOTESTSUM_CMD }} -gcflags=all=-d=checkptr ./internal/regopolicyinterpreter/... - name: Run guest code unit tests - run: ${{ env.GOTESTCMD }} -gcflags=all=-d=checkptr ./internal/guest/... + run: ${{ env.GOTESTSUM_CMD }} -gcflags=all=-d=checkptr ./internal/guest/... - name: Build gcs Testing Binary - run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./gcs + run: ${{ env.GO_BUILD_TEST_CMD }} ./gcs working-directory: test test-windows: @@ -321,34 +327,45 @@ jobs: # run tests - name: Test repo - run: ${{ env.GOTESTCMD }} -gcflags=all=-d=checkptr -tags admin ./... + run: ${{ env.GOTESTSUM_CMD }} -gcflags=all=-d=checkptr -tags admin -timeout=20m ./... - name: Run non-functional tests - run: ${{ env.GOTESTCMD }} -mod=mod -gcflags=all=-d=checkptr ./internal/... ./pkg/... + run: ${{ env.GOTESTSUM_CMD }} -mod=mod -gcflags=all=-d=checkptr ./internal/... ./pkg/... working-directory: test - - name: Run containerd-shim-runhcs-v1 tests + - name: Build and run containerd-shim-runhcs-v1 tests shell: powershell run: | - powershell { - cd '../..' - go build -trimpath -o './test/containerd-shim-runhcs-v1' ./cmd/containerd-shim-runhcs-v1 + pwsh { + cd '..' + ${{ env.GO_BUILD_CMD }} -o ./test ./cmd/containerd-shim-runhcs-v1 2>&1 + } + if ( $LASTEXITCODE ) { + Write-Output '::error::Could not build containerd-shim-runhcs-v1.exe' + exit $LASTEXITCODE } - ${{ env.GOTESTCMD }} -mod=mod -tags functional -gcflags=all=-d=checkptr ./... - working-directory: test/containerd-shim-runhcs-v1 + + ${{ env.GO_BUILD_TEST_CMD }} ./containerd-shim-runhcs-v1 + if ( $LASTEXITCODE ) { + Write-Output '::error::Could not build containerd-shim-runhcs-v1.test.exe' + exit $LASTEXITCODE + } + + ${{ env.GOTESTSUM_CMD_RAW }} ./containerd-shim-runhcs-v1.test.exe '-test.v' + working-directory: test # build testing binaries - name: Build cri-containerd Testing Binary - run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./cri-containerd + run: ${{ env.GO_BUILD_TEST_CMD }} ./cri-containerd working-directory: test - name: Build functional Testing Binary - run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./functional + run: ${{ env.GO_BUILD_TEST_CMD }} ./functional working-directory: test - name: Build runhcs Testing Binary - run: go test -mod=mod -gcflags=all=-d=checkptr -c -tags functional ./runhcs + run: ${{ env.GO_BUILD_TEST_CMD }} ./runhcs working-directory: test - name: Build logging-driver Binary - run: go build -mod=mod -o sample-logging-driver.exe ./cri-containerd/helpers/log.go + run: ${{ env.GO_BUILD_CMD }} -mod=mod -o sample-logging-driver.exe ./cri-containerd/helpers/log.go working-directory: test - uses: actions/upload-artifact@v4 @@ -485,7 +502,7 @@ jobs: working-directory: src/github.com/Microsoft/hcsshim shell: powershell run: | - go build -mod vendor -o "${{ github.workspace }}/src/github.com/containerd/containerd/bin/containerd-shim-runhcs-v1.exe" .\cmd\containerd-shim-runhcs-v1 + ${{ env.GO_BUILD_CMD }} -mod vendor -o "${{ github.workspace }}/src/github.com/containerd/containerd/bin/containerd-shim-runhcs-v1.exe" .\cmd\containerd-shim-runhcs-v1 - name: Install gotestsum run: go install gotest.tools/gotestsum@${{ env.GOTESTSUM_VERSION }} @@ -603,22 +620,35 @@ jobs: $LASTEXITCODE = 0 } - - run: go build ./cmd/containerd-shim-runhcs-v1 - - run: go build ./cmd/runhcs - - run: go build ./cmd/tar2ext4 - - run: go build ./cmd/wclayer - - run: go build ./cmd/device-util - - run: go build ./cmd/ncproxy - - run: go build ./cmd/dmverity-vhd - - run: go build ./cmd/dmverity-vhd + - run: ${{ env.GO_BUILD_CMD }} ./cmd/containerd-shim-runhcs-v1 + name: Build containerd-shim-runhcs-v1.exe + - run: ${{ env.GO_BUILD_CMD }} ./cmd/runhcs + name: Build runhcs.exe + - run: ${{ env.GO_BUILD_CMD }} ./cmd/tar2ext4 + name: Build tar2ext4.exe + - run: ${{ env.GO_BUILD_CMD }} ./cmd/wclayer + name: Build wclayer.exe + - run: ${{ env.GO_BUILD_CMD }} ./cmd/device-util + name: Build device-util.exe + - run: ${{ env.GO_BUILD_CMD }} ./cmd/ncproxy + name: Build ncproxy.exe + - run: ${{ env.GO_BUILD_CMD }} ./cmd/dmverity-vhd + name: Build dmverity-vhd.exe + - run: ${{ env.GO_BUILD_CMD }} ./cmd/dmverity-vhd + name: Build dmverity-vhd env: GOOS: linux GOARCH: amd64 - - run: go build ./internal/tools/grantvmgroupaccess - - run: go build ./internal/tools/networkagent - - run: go build ./internal/tools/securitypolicy - - run: go build ./internal/tools/uvmboot - - run: go build ./internal/tools/zapdir + - run: ${{ env.GO_BUILD_CMD }} ./internal/tools/grantvmgroupaccess + name: Build grantvmgroupaccess.exe + - run: ${{ env.GO_BUILD_CMD }} ./internal/tools/networkagent + name: Build networkagent.exe + - run: ${{ env.GO_BUILD_CMD }} ./internal/tools/securitypolicy + name: Build securitypolicy.exe + - run: ${{ env.GO_BUILD_CMD }} ./internal/tools/uvmboot + name: Build uvmboot.exe + - run: ${{ env.GO_BUILD_CMD }} ./internal/tools/zapdir + name: Build zapdir.exe - uses: actions/upload-artifact@v4 if: ${{ github.event_name == 'pull_request' }} diff --git a/internal/cmd/cmd.go b/internal/cmd/cmd.go index 4568a99c1f..cb7c6f4bee 100644 --- a/internal/cmd/cmd.go +++ b/internal/cmd/cmd.go @@ -212,7 +212,6 @@ func (c *Cmd) Start() error { // Start relaying process IO. stdin, stdout, stderr := p.Stdio() if c.Stdin != nil { - c.Log.Info("coping stdin") // Do not make stdin part of the error group because there is no way for // us or the caller to reliably unblock the c.Stdin read when the // process exits. diff --git a/internal/log/context.go b/internal/log/context.go index 58b257f3ad..4399cec6f8 100644 --- a/internal/log/context.go +++ b/internal/log/context.go @@ -19,13 +19,13 @@ var ( // Instead, use `L.With*` or `L.Dup()`. Or `G(context.Background())`. L = logrus.NewEntry(logrus.StandardLogger()) - // G is an alias for GetEntry + // G is an alias for GetEntry. G = GetEntry - // S is an alias for SetEntry + // S is an alias for SetEntry. S = SetEntry - // U is an alias for UpdateContext + // U is an alias for UpdateContext. U = UpdateContext ) @@ -82,7 +82,7 @@ func UpdateContext(ctx context.Context) context.Context { // WithContext returns a context that contains the provided log entry. // The entry can be extracted with `GetEntry` (`G`) // -// The entry in the context is a copy of `entry` (generated by `entry.WithContext`) +// The entry in the context is a copy of `entry` (generated by `entry.WithContext`). func WithContext(ctx context.Context, entry *logrus.Entry) (context.Context, *logrus.Entry) { // regardless of the order, entry.Context != GetEntry(ctx) // here, the returned entry will reference the supplied context diff --git a/internal/sync/once.go b/internal/sync/once.go new file mode 100644 index 0000000000..34cae5f277 --- /dev/null +++ b/internal/sync/once.go @@ -0,0 +1,41 @@ +package sync + +import ( + "context" + "sync" +) + +// TODO (go1.21): use pkg.go.dev/sync#OnceValues + +// OnceValue is a wrapper around [sync.Once] that runs f only once and +// returns both a value (of type T) and an error. +func OnceValue[T any](f func() (T, error)) func() (T, error) { + var ( + once sync.Once + v T + err error + ) + + return func() (T, error) { + once.Do(func() { + v, err = f() + }) + return v, err + } +} + +// NewOnce is similar to [OnceValue], but allows passing a context to f. +func OnceValueCtx[T any](f func(ctx context.Context) (T, error)) func(context.Context) (T, error) { + var ( + once sync.Once + v T + err error + ) + + return func(ctx context.Context) (T, error) { + once.Do(func() { + v, err = f(ctx) + }) + return v, err + } +} diff --git a/pkg/cimfs/cim_test.go b/pkg/cimfs/cim_test.go index aa78ff12e5..c1e2bc4028 100644 --- a/pkg/cimfs/cim_test.go +++ b/pkg/cimfs/cim_test.go @@ -146,5 +146,4 @@ func TestCimReadWrite(t *testing.T) { } } } - } diff --git a/test/functional/lcow_bench_test.go b/test/functional/lcow_bench_test.go index 9e2e3c538f..752f002acd 100644 --- a/test/functional/lcow_bench_test.go +++ b/test/functional/lcow_bench_test.go @@ -11,14 +11,14 @@ import ( "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" - "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) func BenchmarkLCOW_UVM(b *testing.B) { - requireFeatures(b, featureLCOW) + requireFeatures(b, featureLCOW, featureUVM) require.Build(b, osversion.RS5) - pCtx := context.Background() + pCtx := util.Context(context.Background(), b) b.Run("Create", func(b *testing.B) { b.StopTimer() @@ -26,11 +26,11 @@ func BenchmarkLCOW_UVM(b *testing.B) { for i := 0; i < b.N; i++ { ctx, cancel := context.WithTimeout(pCtx, benchmarkIterationTimeout) - opts := defaultLCOWOptions(b) + opts := defaultLCOWOptions(ctx, b) opts.ID += util.RandNameSuffix(i) b.StartTimer() - _, cleanup := uvm.CreateLCOW(ctx, b, opts) + _, cleanup := testuvm.CreateLCOW(ctx, b, opts) b.StopTimer() cleanup(ctx) @@ -44,9 +44,9 @@ func BenchmarkLCOW_UVM(b *testing.B) { for i := 0; i < b.N; i++ { ctx, cancel := context.WithTimeout(pCtx, benchmarkIterationTimeout) - opts := defaultLCOWOptions(b) + opts := defaultLCOWOptions(ctx, b) opts.ID += util.RandNameSuffix(i) - vm, cleanup := uvm.CreateLCOW(ctx, b, opts) + vm, cleanup := testuvm.CreateLCOW(ctx, b, opts) b.StartTimer() if err := vm.Start(ctx); err != nil { @@ -65,13 +65,13 @@ func BenchmarkLCOW_UVM(b *testing.B) { for i := 0; i < b.N; i++ { ctx, cancel := context.WithTimeout(pCtx, benchmarkIterationTimeout) - opts := defaultLCOWOptions(b) + opts := defaultLCOWOptions(ctx, b) opts.ID += util.RandNameSuffix(i) - vm, cleanup := uvm.CreateLCOW(ctx, b, opts) - uvm.Start(ctx, b, vm) + vm, cleanup := testuvm.CreateLCOW(ctx, b, opts) + testuvm.Start(ctx, b, vm) b.StartTimer() - uvm.Kill(ctx, b, vm) + testuvm.Kill(ctx, b, vm) if err := vm.WaitCtx(ctx); err != nil { b.Fatalf("could not kill uvm %q: %v", vm.ID(), err) } @@ -88,13 +88,13 @@ func BenchmarkLCOW_UVM(b *testing.B) { for i := 0; i < b.N; i++ { ctx, cancel := context.WithTimeout(pCtx, benchmarkIterationTimeout) - opts := defaultLCOWOptions(b) + opts := defaultLCOWOptions(ctx, b) opts.ID += util.RandNameSuffix(i) - vm, cleanup := uvm.CreateLCOW(ctx, b, opts) - uvm.Start(ctx, b, vm) + vm, cleanup := testuvm.CreateLCOW(ctx, b, opts) + testuvm.Start(ctx, b, vm) b.StartTimer() - if err := vm.Close(); err != nil { + if err := vm.CloseCtx(ctx); err != nil { b.Fatalf("could not kill uvm %q: %v", vm.ID(), err) } b.StopTimer() diff --git a/test/functional/lcow_container_bench_test.go b/test/functional/lcow_container_bench_test.go index 2fc0ceedff..9c7bca3454 100644 --- a/test/functional/lcow_container_bench_test.go +++ b/test/functional/lcow_container_bench_test.go @@ -11,7 +11,7 @@ import ( "testing" ctrdoci "github.com/containerd/containerd/oci" - cri_util "github.com/containerd/containerd/pkg/cri/util" + criutil "github.com/containerd/containerd/pkg/cri/util" "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/hcsoci" @@ -21,19 +21,19 @@ import ( "github.com/Microsoft/hcsshim/osversion" testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" - "github.com/Microsoft/hcsshim/test/internal/container" + testcontainer "github.com/Microsoft/hcsshim/test/internal/container" testlayers "github.com/Microsoft/hcsshim/test/internal/layers" - "github.com/Microsoft/hcsshim/test/internal/oci" + testoci "github.com/Microsoft/hcsshim/test/internal/oci" "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) func BenchmarkLCOW_Container(b *testing.B) { - requireFeatures(b, featureLCOW, featureContainer) + requireFeatures(b, featureLCOW, featureUVM, featureContainer) require.Build(b, osversion.RS5) - pCtx := namespacedContext() + pCtx := util.Context(namespacedContext(context.Background()), b) ls := linuxImageLayers(pCtx, b) // Create a new uVM per benchmark in case any left over state lingers @@ -65,19 +65,18 @@ func BenchmarkLCOW_Container(b *testing.B) { vmCleanup(ctx) } // recreate the uVM - opts := defaultLCOWOptions(b) - opts.ID += util.RandNameSuffix(i) + opts := defaultLCOWOptions(ctx, b) vm, vmCleanup = testuvm.CreateLCOW(ctx, b, opts) testuvm.Start(ctx, b, vm) cache = testlayers.CacheFile(ctx, b, "") } - id := cri_util.GenerateID() + id := criutil.GenerateID() scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache) - spec := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, + spec := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, ctrdoci.WithProcessArgs("/bin/sh", "-c", "true"), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) co := &hcsoci.CreateOptions{ ID: id, @@ -107,11 +106,11 @@ func BenchmarkLCOW_Container(b *testing.B) { // container creation launches go rountines on the guest that do // not finish until the init process has terminated. // so start the container, then clean everything up - init := container.Start(ctx, b, c, nil) + init := testcontainer.Start(ctx, b, c, nil) testcmd.WaitExitCode(ctx, b, init, 0) - container.Kill(ctx, b, c) - container.Wait(ctx, b, c) + testcontainer.Kill(ctx, b, c) + testcontainer.Wait(ctx, b, c) if err := resources.ReleaseResources(ctx, r, vm, true); err != nil { b.Errorf("failed to release container resources: %v", err) } @@ -145,21 +144,20 @@ func BenchmarkLCOW_Container(b *testing.B) { vmCleanup(ctx) } // recreate the uVM - opts := defaultLCOWOptions(b) - opts.ID += util.RandNameSuffix(i) + opts := defaultLCOWOptions(ctx, b) vm, vmCleanup = testuvm.CreateLCOW(ctx, b, opts) testuvm.Start(ctx, b, vm) cache = testlayers.CacheFile(ctx, b, "") } - id := cri_util.GenerateID() + id := criutil.GenerateID() scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache) - spec := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, + spec := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, ctrdoci.WithProcessArgs("/bin/sh", "-c", "true"), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) + c, _, cleanup := testcontainer.Create(ctx, b, vm, spec, id, hcsOwner) b.StartTimer() if err := c.Start(ctx); err != nil { @@ -171,8 +169,8 @@ func BenchmarkLCOW_Container(b *testing.B) { testcmd.Start(ctx, b, init) testcmd.WaitExitCode(ctx, b, init, 0) - container.Kill(ctx, b, c) - container.Wait(ctx, b, c) + testcontainer.Kill(ctx, b, c) + testcontainer.Wait(ctx, b, c) cleanup() cancel() } @@ -200,21 +198,20 @@ func BenchmarkLCOW_Container(b *testing.B) { vmCleanup(ctx) } // recreate the uVM - opts := defaultLCOWOptions(b) - opts.ID += util.RandNameSuffix(i) + opts := defaultLCOWOptions(ctx, b) vm, vmCleanup = testuvm.CreateLCOW(ctx, b, opts) testuvm.Start(ctx, b, vm) cache = testlayers.CacheFile(ctx, b, "") } - id := cri_util.GenerateID() + id := criutil.GenerateID() scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache) - spec := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + spec := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs), + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) + c, _, cleanup := testcontainer.Create(ctx, b, vm, spec, id, hcsOwner) if err := c.Start(ctx); err != nil { b.Fatalf("could not start %q: %v", c.ID(), err) } @@ -229,8 +226,8 @@ func BenchmarkLCOW_Container(b *testing.B) { testcmd.Kill(ctx, b, init) testcmd.WaitExitCode(ctx, b, init, testcmd.ForcedKilledExitCode) - container.Kill(ctx, b, c) - container.Wait(ctx, b, c) + testcontainer.Kill(ctx, b, c) + testcontainer.Wait(ctx, b, c) cleanup() cancel() } @@ -258,22 +255,21 @@ func BenchmarkLCOW_Container(b *testing.B) { vmCleanup(ctx) } // recreate the uVM - opts := defaultLCOWOptions(b) - opts.ID += util.RandNameSuffix(i) + opts := defaultLCOWOptions(ctx, b) vm, vmCleanup = testuvm.CreateLCOW(ctx, b, opts) testuvm.Start(ctx, b, vm) cache = testlayers.CacheFile(ctx, b, "") } - id := cri_util.GenerateID() + id := criutil.GenerateID() scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache) - spec := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + spec := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs), + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) - init := container.Start(ctx, b, c, nil) + c, _, cleanup := testcontainer.Create(ctx, b, vm, spec, id, hcsOwner) + init := testcontainer.Start(ctx, b, c, nil) b.StartTimer() if ok, err := init.Process.Kill(ctx); !ok { @@ -293,8 +289,8 @@ func BenchmarkLCOW_Container(b *testing.B) { } b.StopTimer() - container.Kill(ctx, b, c) - container.Wait(ctx, b, c) + testcontainer.Kill(ctx, b, c) + testcontainer.Wait(ctx, b, c) cleanup() cancel() } @@ -322,27 +318,26 @@ func BenchmarkLCOW_Container(b *testing.B) { vmCleanup(ctx) } // recreate the uVM - opts := defaultLCOWOptions(b) - opts.ID += util.RandNameSuffix(i) + opts := defaultLCOWOptions(ctx, b) vm, vmCleanup = testuvm.CreateLCOW(ctx, b, opts) testuvm.Start(ctx, b, vm) cache = testlayers.CacheFile(ctx, b, "") } - id := cri_util.GenerateID() + id := criutil.GenerateID() scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache) - spec := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + spec := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs), + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) - init := container.Start(ctx, b, c, nil) + c, _, cleanup := testcontainer.Create(ctx, b, vm, spec, id, hcsOwner) + init := testcontainer.Start(ctx, b, c, nil) - ps := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, + ps := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, ctrdoci.WithDefaultPathEnv, - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs))..., + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs))..., ).Process exec := testcmd.Create(ctx, b, c, ps, nil) @@ -358,8 +353,8 @@ func BenchmarkLCOW_Container(b *testing.B) { testcmd.Kill(ctx, b, init) testcmd.WaitExitCode(ctx, b, init, testcmd.ForcedKilledExitCode) - container.Kill(ctx, b, c) - container.Wait(ctx, b, c) + testcontainer.Kill(ctx, b, c) + testcontainer.Wait(ctx, b, c) cleanup() cancel() } @@ -387,25 +382,24 @@ func BenchmarkLCOW_Container(b *testing.B) { vmCleanup(ctx) } // recreate the uVM - opts := defaultLCOWOptions(b) - opts.ID += util.RandNameSuffix(i) + opts := defaultLCOWOptions(ctx, b) vm, vmCleanup = testuvm.CreateLCOW(ctx, b, opts) testuvm.Start(ctx, b, vm) cache = testlayers.CacheFile(ctx, b, "") } - id := cri_util.GenerateID() + id := criutil.GenerateID() scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache) - spec := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + spec := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs), + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) - init := container.Start(ctx, b, c, nil) + c, _, cleanup := testcontainer.Create(ctx, b, vm, spec, id, hcsOwner) + init := testcontainer.Start(ctx, b, c, nil) - ps := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, + ps := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, ctrdoci.WithDefaultPathEnv, ctrdoci.WithProcessArgs("/bin/sh", "-c", "true"))..., ).Process @@ -423,8 +417,8 @@ func BenchmarkLCOW_Container(b *testing.B) { testcmd.Kill(ctx, b, init) testcmd.WaitExitCode(ctx, b, init, testcmd.ForcedKilledExitCode) - container.Kill(ctx, b, c) - container.Wait(ctx, b, c) + testcontainer.Kill(ctx, b, c) + testcontainer.Wait(ctx, b, c) cleanup() cancel() } @@ -452,21 +446,20 @@ func BenchmarkLCOW_Container(b *testing.B) { vmCleanup(ctx) } // recreate the uVM - opts := defaultLCOWOptions(b) - opts.ID += util.RandNameSuffix(i) + opts := defaultLCOWOptions(ctx, b) vm, vmCleanup = testuvm.CreateLCOW(ctx, b, opts) testuvm.Start(ctx, b, vm) cache = testlayers.CacheFile(ctx, b, "") } - id := cri_util.GenerateID() + id := criutil.GenerateID() scratch, _ := testlayers.ScratchSpace(ctx, b, vm, "", "", cache) - spec := oci.CreateLinuxSpec(ctx, b, id, - oci.DefaultLinuxSpecOpts(id, + spec := testoci.CreateLinuxSpec(ctx, b, id, + testoci.DefaultLinuxSpecOpts(id, ctrdoci.WithProcessArgs("/bin/sh", "-c", "true"), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, b, vm, spec, id, hcsOwner) + c, _, cleanup := testcontainer.Create(ctx, b, vm, spec, id, hcsOwner) // (c container).Wait() waits until the gc receives a notification message from // the guest (via the bridge) that the container exited. @@ -476,12 +469,12 @@ func BenchmarkLCOW_Container(b *testing.B) { // (hcsv2/process.go:(*containerProcess).Wait). // // So ... to test container kill and wait times, we need to first start and wait on the init process - init := container.Start(ctx, b, c, nil) + init := testcontainer.Start(ctx, b, c, nil) testcmd.WaitExitCode(ctx, b, init, 0) b.StartTimer() - container.Kill(ctx, b, c) - container.Wait(ctx, b, c) + testcontainer.Kill(ctx, b, c) + testcontainer.Wait(ctx, b, c) b.StopTimer() cleanup() diff --git a/test/functional/lcow_container_test.go b/test/functional/lcow_container_test.go index c354064438..d5e94fde03 100644 --- a/test/functional/lcow_container_test.go +++ b/test/functional/lcow_container_test.go @@ -4,6 +4,7 @@ package functional import ( + "context" "strings" "testing" @@ -11,42 +12,42 @@ import ( "github.com/Microsoft/hcsshim/osversion" - "github.com/Microsoft/hcsshim/test/internal/cmd" - "github.com/Microsoft/hcsshim/test/internal/container" - "github.com/Microsoft/hcsshim/test/internal/layers" - "github.com/Microsoft/hcsshim/test/internal/oci" + testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" + testcontainer "github.com/Microsoft/hcsshim/test/internal/container" + testlayers "github.com/Microsoft/hcsshim/test/internal/layers" + testoci "github.com/Microsoft/hcsshim/test/internal/oci" "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" - "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) func TestLCOW_ContainerLifecycle(t *testing.T) { - requireFeatures(t, featureLCOW, featureContainer) + requireFeatures(t, featureLCOW, featureUVM, featureContainer) require.Build(t, osversion.RS5) - ctx := namespacedContext() + ctx := util.Context(namespacedContext(context.Background()), t) ls := linuxImageLayers(ctx, t) - opts := defaultLCOWOptions(t) + opts := defaultLCOWOptions(ctx, t) opts.ID += util.RandNameSuffix() - vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) - scratch, _ := layers.ScratchSpace(ctx, t, vm, "", "", "") + scratch, _ := testlayers.ScratchSpace(ctx, t, vm, "", "", "") - spec := oci.CreateLinuxSpec(ctx, t, t.Name()+util.RandNameSuffix(), - oci.DefaultLinuxSpecOpts("", - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + spec := testoci.CreateLinuxSpec(ctx, t, t.Name()+util.RandNameSuffix(), + testoci.DefaultLinuxSpecOpts("", + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs), + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, t, vm, spec, t.Name(), hcsOwner) + c, _, cleanup := testcontainer.Create(ctx, t, vm, spec, t.Name(), hcsOwner) t.Cleanup(cleanup) - init := container.Start(ctx, t, c, nil) + init := testcontainer.Start(ctx, t, c, nil) t.Cleanup(func() { - container.Kill(ctx, t, c) - container.Wait(ctx, t, c) + testcontainer.Kill(ctx, t, c) + testcontainer.Wait(ctx, t, c) }) - cmd.Kill(ctx, t, init) - cmd.WaitExitCode(ctx, t, init, cmd.ForcedKilledExitCode) + testcmd.Kill(ctx, t, init) + testcmd.WaitExitCode(ctx, t, init, testcmd.ForcedKilledExitCode) } var ioTests = []struct { @@ -74,40 +75,40 @@ var ioTests = []struct { } func TestLCOW_ContainerIO(t *testing.T) { - requireFeatures(t, featureLCOW, featureContainer) + requireFeatures(t, featureLCOW, featureUVM, featureContainer) require.Build(t, osversion.RS5) - ctx := namespacedContext() + ctx := util.Context(namespacedContext(context.Background()), t) ls := linuxImageLayers(ctx, t) - opts := defaultLCOWOptions(t) + opts := defaultLCOWOptions(ctx, t) opts.ID += util.RandNameSuffix() - cache := layers.CacheFile(ctx, t, "") - vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + cache := testlayers.CacheFile(ctx, t, "") + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) for _, tt := range ioTests { t.Run(tt.name, func(t *testing.T) { id := strings.ReplaceAll(t.Name(), "/", "") + util.RandNameSuffix() - scratch, _ := layers.ScratchSpace(ctx, t, vm, "", "", cache) - spec := oci.CreateLinuxSpec(ctx, t, id, - oci.DefaultLinuxSpecOpts(id, + scratch, _ := testlayers.ScratchSpace(ctx, t, vm, "", "", cache) + spec := testoci.CreateLinuxSpec(ctx, t, id, + testoci.DefaultLinuxSpecOpts(id, ctrdoci.WithProcessArgs(tt.args...), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, t, vm, spec, id, hcsOwner) + c, _, cleanup := testcontainer.Create(ctx, t, vm, spec, id, hcsOwner) t.Cleanup(cleanup) - io := cmd.NewBufferedIO() + io := testcmd.NewBufferedIO() if tt.in != "" { - io = cmd.NewBufferedIOFromString(tt.in) + io = testcmd.NewBufferedIOFromString(tt.in) } - init := container.Start(ctx, t, c, io) + init := testcontainer.Start(ctx, t, c, io) t.Cleanup(func() { - container.Kill(ctx, t, c) - container.Wait(ctx, t, c) + testcontainer.Kill(ctx, t, c) + testcontainer.Wait(ctx, t, c) }) - if e := cmd.Wait(ctx, t, init); e != 0 { + if e := testcmd.Wait(ctx, t, init); e != 0 { t.Fatalf("got exit code %d, wanted %d", e, 0) } @@ -117,48 +118,48 @@ func TestLCOW_ContainerIO(t *testing.T) { } func TestLCOW_ContainerExec(t *testing.T) { - requireFeatures(t, featureLCOW, featureContainer) + requireFeatures(t, featureLCOW, featureUVM, featureContainer) require.Build(t, osversion.RS5) - ctx := namespacedContext() + ctx := util.Context(namespacedContext(context.Background()), t) ls := linuxImageLayers(ctx, t) - opts := defaultLCOWOptions(t) + opts := defaultLCOWOptions(ctx, t) opts.ID += util.RandNameSuffix() - vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) id := strings.ReplaceAll(t.Name(), "/", "") + util.RandNameSuffix() - scratch, _ := layers.ScratchSpace(ctx, t, vm, "", "", "") - spec := oci.CreateLinuxSpec(ctx, t, id, - oci.DefaultLinuxSpecOpts(id, - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) + scratch, _ := testlayers.ScratchSpace(ctx, t, vm, "", "", "") + spec := testoci.CreateLinuxSpec(ctx, t, id, + testoci.DefaultLinuxSpecOpts(id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs), + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) - c, _, cleanup := container.Create(ctx, t, vm, spec, id, hcsOwner) + c, _, cleanup := testcontainer.Create(ctx, t, vm, spec, id, hcsOwner) t.Cleanup(cleanup) - init := container.Start(ctx, t, c, nil) + init := testcontainer.Start(ctx, t, c, nil) t.Cleanup(func() { - cmd.Kill(ctx, t, init) - cmd.Wait(ctx, t, init) - container.Kill(ctx, t, c) - container.Wait(ctx, t, c) + testcmd.Kill(ctx, t, init) + testcmd.Wait(ctx, t, init) + testcontainer.Kill(ctx, t, c) + testcontainer.Wait(ctx, t, c) }) for _, tt := range ioTests { t.Run(tt.name, func(t *testing.T) { - ps := oci.CreateLinuxSpec(ctx, t, id, - oci.DefaultLinuxSpecOpts(id, + ps := testoci.CreateLinuxSpec(ctx, t, id, + testoci.DefaultLinuxSpecOpts(id, // oci.WithTTY, ctrdoci.WithDefaultPathEnv, ctrdoci.WithProcessArgs(tt.args...))..., ).Process - io := cmd.NewBufferedIO() + io := testcmd.NewBufferedIO() if tt.in != "" { - io = cmd.NewBufferedIOFromString(tt.in) + io = testcmd.NewBufferedIOFromString(tt.in) } - p := cmd.Create(ctx, t, c, ps, io) - cmd.Start(ctx, t, p) + p := testcmd.Create(ctx, t, c, ps, io) + testcmd.Start(ctx, t, p) - if e := cmd.Wait(ctx, t, p); e != 0 { + if e := testcmd.Wait(ctx, t, p); e != 0 { t.Fatalf("got exit code %d, wanted %d", e, 0) } diff --git a/test/functional/lcow_networking_test.go b/test/functional/lcow_networking_test.go index 63a61210da..5126e3e1d0 100644 --- a/test/functional/lcow_networking_test.go +++ b/test/functional/lcow_networking_test.go @@ -4,6 +4,7 @@ package functional import ( + "context" "fmt" "strings" "testing" @@ -13,16 +14,17 @@ import ( "github.com/Microsoft/hcsshim/hcn" "github.com/Microsoft/hcsshim/osversion" - "github.com/Microsoft/hcsshim/test/internal/cmd" - "github.com/Microsoft/hcsshim/test/internal/container" - "github.com/Microsoft/hcsshim/test/internal/layers" - "github.com/Microsoft/hcsshim/test/internal/oci" + testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" + testcontainer "github.com/Microsoft/hcsshim/test/internal/container" + testlayers "github.com/Microsoft/hcsshim/test/internal/layers" + testoci "github.com/Microsoft/hcsshim/test/internal/oci" + "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" - "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) func TestLCOW_IPv6_Assignment(t *testing.T) { - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) require.Build(t, osversion.RS5) ns, err := newNetworkNamespace() @@ -121,10 +123,10 @@ func TestLCOW_IPv6_Assignment(t *testing.T) { t.Fatalf("network attachment: %v", err) } - ctx := namespacedContext() + ctx := util.Context(namespacedContext(context.Background()), t) ls := linuxImageLayers(ctx, t) - opts := defaultLCOWOptions(t) - vm := uvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + opts := defaultLCOWOptions(ctx, t) + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) if err := vm.CreateAndAssignNetworkSetup(ctx, "", ""); err != nil { t.Fatalf("setting up network: %v", err) @@ -134,35 +136,35 @@ func TestLCOW_IPv6_Assignment(t *testing.T) { } cID := strings.ReplaceAll(t.Name(), "/", "") - scratch, _ := layers.ScratchSpace(ctx, t, vm, "", "", "") - spec := oci.CreateLinuxSpec(ctx, t, cID, - oci.DefaultLinuxSpecOpts(ns.Id, - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), - oci.WithWindowsNetworkNamespace(ns.Id), - oci.WithWindowsLayerFolders(append(ls, scratch)))...) - - c, _, cleanup := container.Create(ctx, t, vm, spec, cID, hcsOwner) + scratch, _ := testlayers.ScratchSpace(ctx, t, vm, "", "", "") + spec := testoci.CreateLinuxSpec(ctx, t, cID, + testoci.DefaultLinuxSpecOpts(ns.Id, + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs), + ctrdoci.WithWindowsNetworkNamespace(ns.Id), + testoci.WithWindowsLayerFolders(append(ls, scratch)))...) + + c, _, cleanup := testcontainer.Create(ctx, t, vm, spec, cID, hcsOwner) t.Logf("created container %s", cID) t.Cleanup(cleanup) - init := container.Start(ctx, t, c, nil) + init := testcontainer.Start(ctx, t, c, nil) t.Cleanup(func() { - cmd.Kill(ctx, t, init) - cmd.Wait(ctx, t, init) - container.Kill(ctx, t, c) - container.Wait(ctx, t, c) + testcmd.Kill(ctx, t, init) + testcmd.Wait(ctx, t, init) + testcontainer.Kill(ctx, t, c) + testcontainer.Wait(ctx, t, c) }) - ps := oci.CreateLinuxSpec(ctx, t, cID, - oci.DefaultLinuxSpecOpts(ns.Id, + ps := testoci.CreateLinuxSpec(ctx, t, cID, + testoci.DefaultLinuxSpecOpts(ns.Id, ctrdoci.WithDefaultPathEnv, ctrdoci.WithProcessArgs("/bin/sh", "-c", `ip -o address show dev eth0 scope global`), )..., ).Process - io := cmd.NewBufferedIO() - p := cmd.Create(ctx, t, c, ps, io) - cmd.Start(ctx, t, p) + io := testcmd.NewBufferedIO() + p := testcmd.Create(ctx, t, c, ps, io) + testcmd.Start(ctx, t, p) - e := cmd.Wait(ctx, t, p) + e := testcmd.Wait(ctx, t, p) out, err := io.Output() t.Logf("cmd output:\n%s", out) if e != 0 || err != nil { diff --git a/test/functional/lcow_policy_test.go b/test/functional/lcow_policy_test.go index 20a214bb32..53e65cc8cb 100644 --- a/test/functional/lcow_policy_test.go +++ b/test/functional/lcow_policy_test.go @@ -11,19 +11,20 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/pkg/securitypolicy" - "github.com/Microsoft/hcsshim/test/internal/cmd" - "github.com/Microsoft/hcsshim/test/internal/container" - "github.com/Microsoft/hcsshim/test/internal/layers" - "github.com/Microsoft/hcsshim/test/internal/oci" + + testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" + testcontainer "github.com/Microsoft/hcsshim/test/internal/container" + testlayers "github.com/Microsoft/hcsshim/test/internal/layers" + testoci "github.com/Microsoft/hcsshim/test/internal/oci" "github.com/Microsoft/hcsshim/test/internal/util" - "github.com/Microsoft/hcsshim/test/pkg/images" + testimages "github.com/Microsoft/hcsshim/test/pkg/images" policytest "github.com/Microsoft/hcsshim/test/pkg/securitypolicy" - uvmtest "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) func setupScratchTemplate(ctx context.Context, tb testing.TB) string { tb.Helper() - opts := defaultLCOWOptions(tb) + opts := defaultLCOWOptions(ctx, tb) vm, err := uvm.CreateLCOW(ctx, opts) if err != nil { tb.Fatalf("failed to create scratch formatting uVM: %s", err) @@ -31,27 +32,27 @@ func setupScratchTemplate(ctx context.Context, tb testing.TB) string { if err := vm.Start(ctx); err != nil { tb.Fatalf("failed to start scratch formatting uVM: %s", err) } - defer vm.Close() - scratch, _ := layers.ScratchSpace(ctx, tb, vm, "", "", "") + defer testuvm.Close(ctx, tb, vm) + scratch, _ := testlayers.ScratchSpace(ctx, tb, vm, "", "", "") return scratch } -func Test_GetProperties_WithPolicy(t *testing.T) { - requireFeatures(t, featureLCOWIntegrity) +func TestGetProperties_WithPolicy(t *testing.T) { + requireFeatures(t, featureLCOW, featureUVM, featureLCOWIntegrity) - ctx := namespacedContext() + ctx := util.Context(namespacedContext(context.Background()), t) scratchPath := setupScratchTemplate(ctx, t) ls := linuxImageLayers(ctx, t) for _, allowProperties := range []bool{true, false} { t.Run(fmt.Sprintf("AllowPropertiesAccess_%t", allowProperties), func(t *testing.T) { - opts := defaultLCOWOptions(t) + opts := defaultLCOWOptions(ctx, t) policy := policytest.PolicyFromImageWithOpts( t, - images.ImageLinuxAlpineLatest, + testimages.ImageLinuxAlpineLatest, "rego", []securitypolicy.ContainerConfigOpt{ - securitypolicy.WithCommand([]string{"/bin/sh", "-c", oci.TailNullArgs}), + securitypolicy.WithCommand([]string{"/bin/sh", "-c", testoci.TailNullArgs}), }, []securitypolicy.PolicyConfigOpt{ securitypolicy.WithAllowPropertiesAccess(allowProperties), @@ -62,25 +63,25 @@ func Test_GetProperties_WithPolicy(t *testing.T) { opts.SecurityPolicy = policy cleanName := util.CleanName(t.Name()) - vm := uvmtest.CreateAndStartLCOWFromOpts(ctx, t, opts) - spec := oci.CreateLinuxSpec( + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + spec := testoci.CreateLinuxSpec( ctx, t, cleanName, - oci.DefaultLinuxSpecOpts( + testoci.DefaultLinuxSpecOpts( "", - ctrdoci.WithProcessArgs("/bin/sh", "-c", oci.TailNullArgs), - oci.WithWindowsLayerFolders(append(ls, scratchPath)), + ctrdoci.WithProcessArgs("/bin/sh", "-c", testoci.TailNullArgs), + testoci.WithWindowsLayerFolders(append(ls, scratchPath)), )..., ) - c, _, cleanup := container.Create(ctx, t, vm, spec, cleanName, hcsOwner) + c, _, cleanup := testcontainer.Create(ctx, t, vm, spec, cleanName, hcsOwner) t.Cleanup(cleanup) - init := container.Start(ctx, t, c, nil) + init := testcontainer.Start(ctx, t, c, nil) t.Cleanup(func() { - container.Kill(ctx, t, c) - container.Wait(ctx, t, c) + testcontainer.Kill(ctx, t, c) + testcontainer.Wait(ctx, t, c) }) _, err := c.Properties(ctx) @@ -98,8 +99,8 @@ func Test_GetProperties_WithPolicy(t *testing.T) { } } - cmd.Kill(ctx, t, init) - cmd.WaitExitCode(ctx, t, init, cmd.ForcedKilledExitCode) + testcmd.Kill(ctx, t, init) + testcmd.WaitExitCode(ctx, t, init, testcmd.ForcedKilledExitCode) }) } } diff --git a/test/functional/lcow_test.go b/test/functional/lcow_test.go index c9ead0c580..a634241b13 100644 --- a/test/functional/lcow_test.go +++ b/test/functional/lcow_test.go @@ -26,33 +26,32 @@ import ( testutilities "github.com/Microsoft/hcsshim/test/internal" testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" + "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) -// test if closing a waiting (but not starting) uVM succeeds +// test if closing a waiting (but not starting) uVM succeeds. func TestLCOW_UVMCreateClose(t *testing.T) { - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) require.Build(t, osversion.RS5) - ctx := context.Background() - vm, err := uvm.CreateLCOW(ctx, defaultLCOWOptions(t)) - if err != nil { - t.Fatalf("could not create LCOW UVM: %v", err) - } + ctx := util.Context(context.Background(), t) + vm, cleanup := testuvm.CreateLCOW(ctx, t, defaultLCOWOptions(ctx, t)) - if err := vm.CloseCtx(ctx); err != nil { - t.Fatalf("could not close uvm %q: %s", vm.ID(), err) - } + testuvm.Close(ctx, t, vm) + + // also run cleanup to make sure that works fine too + cleanup(ctx) } -// test if waiting after creating (but not starting) an LCOW uVM returns +// test if waiting after creating (but not starting) an LCOW uVM returns. func TestLCOW_UVMCreateWait(t *testing.T) { - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) require.Build(t, osversion.RS5) - pCtx := context.Background() - vm, cleanup := testuvm.CreateLCOW(pCtx, t, defaultLCOWOptions(t)) + pCtx := util.Context(context.Background(), t) + vm, cleanup := testuvm.CreateLCOW(pCtx, t, defaultLCOWOptions(pCtx, t)) t.Cleanup(func() { cleanup(pCtx) }) ctx, cancel := context.WithTimeout(pCtx, 3*time.Second) @@ -68,24 +67,27 @@ func TestLCOW_UVMCreateWait(t *testing.T) { // TestLCOW_UVMNoSCSINoVPMemInitrd starts an LCOW utility VM without a SCSI controller and // no VPMem device. Uses initrd. func TestLCOW_UVMNoSCSINoVPMemInitrd(t *testing.T) { - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) - opts := defaultLCOWOptions(t) + ctx := util.Context(context.Background(), t) + opts := defaultLCOWOptions(ctx, t) opts.SCSIControllerCount = 0 opts.VPMemDeviceCount = 0 opts.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd opts.RootFSFile = uvm.InitrdFile opts.KernelDirect = false + opts.KernelFile = uvm.KernelFile testLCOWUVMNoSCSISingleVPMem(t, opts, fmt.Sprintf("Command line: initrd=/%s", opts.RootFSFile)) } // TestLCOW_UVMNoSCSISingleVPMemVHD starts an LCOW utility VM without a SCSI controller and -// only a single VPMem device. Uses VPMEM VHD +// only a single VPMem device. Uses VPMEM VHD. func TestLCOW_UVMNoSCSISingleVPMemVHD(t *testing.T) { - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) - opts := defaultLCOWOptions(t) + ctx := util.Context(context.Background(), t) + opts := defaultLCOWOptions(ctx, t) opts.SCSIControllerCount = 0 opts.VPMemDeviceCount = 1 opts.PreferredRootFSType = uvm.PreferredRootFSTypeVHD @@ -97,16 +99,16 @@ func TestLCOW_UVMNoSCSISingleVPMemVHD(t *testing.T) { func testLCOWUVMNoSCSISingleVPMem(t *testing.T, opts *uvm.OptionsLCOW, expected ...string) { t.Helper() require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW) - ctx := context.Background() + requireFeatures(t, featureLCOW, featureUVM) + ctx := util.Context(context.Background(), t) lcowUVM := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) - defer lcowUVM.Close() io := testcmd.NewBufferedIO() // c := cmd.Command(lcowUVM, "dmesg") c := testcmd.Create(ctx, t, lcowUVM, &specs.Process{Args: []string{"dmesg"}}, io) - testcmd.Run(ctx, t, c) + testcmd.Start(ctx, t, c) + testcmd.WaitExitCode(ctx, t, c, 0) out, err := io.Output() @@ -127,7 +129,7 @@ func testLCOWUVMNoSCSISingleVPMem(t *testing.T, opts *uvm.OptionsLCOW, expected // attached root filesystem a number of times. func TestLCOW_TimeUVMStartVHD(t *testing.T) { require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) testLCOWTimeUVMStart(t, false, uvm.PreferredRootFSTypeVHD) } @@ -137,7 +139,7 @@ func TestLCOW_TimeUVMStartVHD(t *testing.T) { // Kernel directly and skipping EFI. func TestLCOW_UVMStart_KernelDirect_VHD(t *testing.T) { require.Build(t, 18286) - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) testLCOWTimeUVMStart(t, true, uvm.PreferredRootFSTypeVHD) } @@ -146,7 +148,7 @@ func TestLCOW_UVMStart_KernelDirect_VHD(t *testing.T) { // attached root file system a number of times. func TestLCOW_TimeUVMStartInitRD(t *testing.T) { require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) testLCOWTimeUVMStart(t, false, uvm.PreferredRootFSTypeInitRd) } @@ -156,18 +158,23 @@ func TestLCOW_TimeUVMStartInitRD(t *testing.T) { // Linux Kernel directly and skipping EFI. func TestLCOW_UVMStart_KernelDirect_InitRd(t *testing.T) { require.Build(t, 18286) - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) testLCOWTimeUVMStart(t, true, uvm.PreferredRootFSTypeInitRd) } func testLCOWTimeUVMStart(t *testing.T, kernelDirect bool, rfsType uvm.PreferredRootFSType) { t.Helper() - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) + ctx := util.Context(context.Background(), t) for i := 0; i < 3; i++ { - opts := defaultLCOWOptions(t) + opts := defaultLCOWOptions(ctx, t) opts.KernelDirect = kernelDirect + if !kernelDirect { + // can only use the uncompressed kernel with direct boot + opts.KernelFile = uvm.KernelFile + } opts.VPMemDeviceCount = 32 opts.PreferredRootFSType = rfsType switch opts.PreferredRootFSType { @@ -178,7 +185,7 @@ func testLCOWTimeUVMStart(t *testing.T, kernelDirect bool, rfsType uvm.Preferred } lcowUVM := testuvm.CreateAndStartLCOWFromOpts(context.Background(), t, opts) - lcowUVM.Close() + testuvm.Close(ctx, t, lcowUVM) } } @@ -186,7 +193,7 @@ func TestLCOWSimplePodScenario(t *testing.T) { t.Skip("Doesn't work quite yet") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW, featureContainer) + requireFeatures(t, featureLCOW, featureUVM, featureContainer) layers := linuxImageLayers(context.Background(), t) @@ -245,11 +252,11 @@ func TestLCOWSimplePodScenario(t *testing.T) { } // Create the two containers - c1hcsSystem, c1Resources, err := CreateContainerTestWrapper(context.Background(), c1Opts) + c1hcsSystem, c1Resources, err := hcsoci.CreateContainer(context.Background(), c1Opts) if err != nil { t.Fatal(err) } - c2hcsSystem, c2Resources, err := CreateContainerTestWrapper(context.Background(), c2Opts) + c2hcsSystem, c2Resources, err := hcsoci.CreateContainer(context.Background(), c2Opts) if err != nil { t.Fatal(err) } diff --git a/test/functional/main_test.go b/test/functional/main_test.go index da10421ab5..b2f5f9d5cc 100644 --- a/test/functional/main_test.go +++ b/test/functional/main_test.go @@ -10,10 +10,10 @@ package functional import ( "context" "flag" + "fmt" "io" "os" "os/exec" - "strconv" "strings" "testing" "time" @@ -22,20 +22,20 @@ import ( "github.com/Microsoft/go-winio/pkg/etwlogrus" "github.com/containerd/containerd/namespaces" "github.com/sirupsen/logrus" + "github.com/urfave/cli/v2" "go.opencensus.io/trace" - "github.com/Microsoft/hcsshim/internal/cow" - "github.com/Microsoft/hcsshim/internal/hcsoci" "github.com/Microsoft/hcsshim/internal/log" "github.com/Microsoft/hcsshim/internal/oc" - "github.com/Microsoft/hcsshim/internal/resources" + "github.com/Microsoft/hcsshim/internal/sync" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/winapi" + "github.com/Microsoft/hcsshim/osversion" - "github.com/Microsoft/hcsshim/test/internal/layers" + testlayers "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/internal/util" testflag "github.com/Microsoft/hcsshim/test/pkg/flag" - "github.com/Microsoft/hcsshim/test/pkg/images" + testimages "github.com/Microsoft/hcsshim/test/pkg/images" "github.com/Microsoft/hcsshim/test/pkg/require" testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) @@ -43,118 +43,187 @@ import ( // owner field for uVMs. const hcsOwner = "hcsshim-functional-tests" -// how long to allow a benchmark iteration to run for +// how long to allow a benchmark iteration to run for. const benchmarkIterationTimeout = 30 * time.Second -var ( - alpineImagePaths = &layers.LazyImageLayers{ - Image: images.ImageLinuxAlpineLatest, - Platform: images.PlatformLinux, - } - //TODO: pick appropriate image based on OS build - nanoserverImagePaths = &layers.LazyImageLayers{ - Image: images.ImageWindowsNanoserverLTSC2022, - Platform: images.PlatformWindows, - } - // wcow tests originally used busyboxw; cannot find image on docker or mcr - servercoreImagePaths = &layers.LazyImageLayers{ - Image: images.ImageWindowsServercoreLTSC2022, - Platform: images.PlatformWindows, +// Linux image(s) + +var alpineImagePaths = &testlayers.LazyImageLayers{ + Image: testimages.ImageLinuxAlpineLatest, + Platform: testimages.PlatformLinux, + AppendVerity: false, // will be set to true if [featureLCOWIntegrity] is passed +} + +// Windows images + +// group wcow images together for shared init. +// want to avoid erroring if WCOW tests are not selected, and also prevet accidentally accessing values without checking +// error value first. +type wcowImages struct { + nanoserver *testlayers.LazyImageLayers + + // wcow tests originally used busyboxw; cannot find image on docker or mcr. + servercore *testlayers.LazyImageLayers +} + +var wcowImagePathsOnce = sync.OnceValue(func() (*wcowImages, error) { + build := osversion.Build() + tag, err := testimages.ImageFromBuild(build) + if err != nil || tag == "" { + return nil, fmt.Errorf("Windows images init: could not look up image tag for build %d", build) } -) + + return &wcowImages{ + nanoserver: &testlayers.LazyImageLayers{ + Image: testimages.NanoserverImage(tag), + Platform: testimages.PlatformWindows, + }, + servercore: &testlayers.LazyImageLayers{ + Image: testimages.ServercoreImage(tag), + Platform: testimages.PlatformWindows, + }, + }, nil +}) const ( - featureLCOW = "LCOW" - featureLCOWIntegrity = "LCOWIntegrity" - featureWCOW = "WCOW" - featureContainer = "container" - featureHostProcess = "HostProcess" - featureUVMMem = "UVMMem" - featurePlan9 = "Plan9" - featureSCSI = "SCSI" - featureScratch = "Scratch" - featureVSMB = "vSMB" - featureVPMEM = "vPMEM" + // container and uVM types. + + featureLCOW = "LCOW" // Linux containers or uVM tests; requires [featureUVM] + featureLCOWIntegrity = "LCOWIntegrity" // Linux confidential/policy tests + featureWCOW = "WCOW" // Windows containers or uVM tests + featureUVM = "uVM" // tests that create a utility VM + featureContainer = "container" // tests that create a container (either process or hyper-v isolated) + featureHostProcess = "HostProcess" // tests that create a Windows HostProcess container; requires [featureWCOW] + + // resources and misc functionality. + + featureScratch = "Scratch" // validate scratch layer mounting + featurePlan9 = "Plan9" // Plan9 file shares + featureSCSI = "SCSI" // SCSI disk (virtuall and physical) mounts + featureVSMB = "vSMB" // virtual SMB file shares + featureVPMEM = "vPMEM" // virtual PMEM mounts ) var allFeatures = []string{ featureLCOW, featureLCOWIntegrity, featureWCOW, - featureHostProcess, + featureUVM, featureContainer, - featureUVMMem, + featureHostProcess, + featureScratch, featurePlan9, featureSCSI, - featureScratch, featureVSMB, featureVPMEM, } var ( - flagPauseAfterCreateContainerFailure time.Duration - - flagLogLevel = testflag.NewLogrusLevel("log-level", defaultLogLevel(), "logrus logging `level`") + flagLogLevel = testflag.NewLogrusLevel("log-level", logrus.WarnLevel.String(), "logrus logging `level`") flagFeatures = testflag.NewFeatureFlag(allFeatures) flagContainerdNamespace = flag.String("ctr-namespace", hcsOwner, "containerd `namespace` to use when creating OCI specs") flagLCOWLayerPaths = testflag.NewStringSet("lcow-layer-paths", "comma separated list of image layer `paths` to use as LCOW container rootfs. "+ "If empty, \""+alpineImagePaths.Image+"\" will be pulled and unpacked.", true) - //nolint:unused // will be used when WCOW tests are updated flagWCOWLayerPaths = testflag.NewStringSet("wcow-layer-paths", "comma separated list of image layer `paths` to use as WCOW uVM and container rootfs. "+ - "If empty, \""+nanoserverImagePaths.Image+"\" will be pulled and unpacked.", true) + "If empty, \""+testimages.NanoserverImage("")+"\" will be pulled and unpacked.", true) flagLayerTempDir = flag.String("layer-temp-dir", "", "`directory` to unpack image layers to, if not provided. Leave empty to use os.TempDir.") flagLinuxBootFilesPath = flag.String("linux-bootfiles", "", "override default `path` for LCOW uVM boot files (rootfs.vhd, initrd.img, kernel, and vmlinux)") ) -func init() { - if !winapi.IsElevated() { - panic("tests must be run in an elevated context") - } +func TestMain(m *testing.M) { + flag.Parse() + + if err := runTests(m); err != nil { + fmt.Fprintln(os.Stderr, err) - // This allows for debugging a utility VM. - if s := os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES"); s != "" { - if t, err := strconv.Atoi(s); err == nil { - flagPauseAfterCreateContainerFailure = time.Duration(t) * time.Minute + // if `m.Run()` returns an exit code, use that + // otherwise, use exit code `1` + c := 1 + if ec, ok := err.(cli.ExitCoder); ok { //nolint:errorlint + c = ec.ExitCode() } + os.Exit(c) } - flag.DurationVar(&flagPauseAfterCreateContainerFailure, - "container-creation-failure-pause", - flagPauseAfterCreateContainerFailure, - "the number of minutes to wait after a container creation failure to try again "+ - "[%HCSSHIM_FUNCTIONAL_TESTS_PAUSE_ON_CREATECONTAINER_FAIL_IN_MINUTES%]") } -func TestMain(m *testing.M) { - flag.Parse() +func runTests(m *testing.M) error { + // ! don't call os.Exit/log.Fatal here, sine that will skip deferred statements + + ctx := context.Background() + + if !winapi.IsElevated() { + return fmt.Errorf("tests must be run in an elevated context") + } trace.ApplyConfig(trace.Config{DefaultSampler: oc.DefaultSampler}) trace.RegisterExporter(&oc.LogrusExporter{}) // default is stderr, but test2json does not consume stderr, so logs would be out of sync // and powershell considers output on stderr as an error when execing + // + // ! keep defer statement in [util.RunningBenchmarks()] in sync with output/formatter settings here logrus.SetOutput(os.Stdout) logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) logrus.SetLevel(flagLogLevel.Level) logrus.Debugf("using features: %s", flagFeatures.Strings()) - images := []*layers.LazyImageLayers{alpineImagePaths, nanoserverImagePaths, servercoreImagePaths} - for _, l := range images { + if flagFeatures.IsSet(featureLCOWIntegrity) { + logrus.Info("appending verity information to LCOW images") + alpineImagePaths.AppendVerity = true + } + + imgs := []*testlayers.LazyImageLayers{} + if flagFeatures.IsSet(featureLCOWIntegrity) || flagFeatures.IsSet(featureLCOW) { + imgs = append(imgs, alpineImagePaths) + } + + if flagFeatures.IsSet(featureWCOW) { + wcow, err := wcowImagePathsOnce() + if err != nil { + return err + } + + logrus.WithField("image", wcow.nanoserver.Image).Info("using Nano Server image") + logrus.WithField("image", wcow.nanoserver.Image).Info("using Server Core image") + + imgs = append(imgs, wcow.nanoserver, wcow.servercore) + } + + for _, l := range imgs { l.TempPath = *flagLayerTempDir } - // print additional configuration options when running benchmarks, so we can better track performance. + defer func(ctx context.Context) { + cleanupComputeSystems(ctx, hcsOwner) + + for _, l := range imgs { + if l == nil { + continue + } + // just log errors: no other cleanup possible + if err := l.Close(ctx); err != nil { + log.G(ctx).WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "image": l.Image, + "platform": l.Platform, + }).Warning("image cleanup failed") + } + } + }(ctx) + + // print additional configuration options when running benchmarks, so we can track performance. + // + // also, print to ETW instead of stdout to mirror actual deployments, and to prevent logs from + // interfering with benchmarking output if util.RunningBenchmarks() { util.PrintAdditionalBenchmarkConfig() - // also, print to ETW instead of stdout to mirror actual deployments, and prevent logs from - // interfering with benchmarking output - provider, err := etw.NewProviderWithOptions("Microsoft.Virtualization.RunHCS") if err != nil { logrus.Error(err) @@ -169,75 +238,53 @@ func TestMain(m *testing.M) { // regardless of ETW provider status, still discard logs logrus.SetFormatter(log.NopFormatter{}) logrus.SetOutput(io.Discard) - } - e := m.Run() - - if util.RunningBenchmarks() { - // un-discard logs during cleanup - logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) - logrus.SetOutput(os.Stdout) + defer func() { + // un-discard logs during cleanup + logrus.SetFormatter(&logrus.TextFormatter{FullTimestamp: true}) + logrus.SetOutput(os.Stdout) + }() } - // close any uVMs that escaped - cmdStr := ` foreach ($vm in Get-ComputeProcess -Owner '` + hcsOwner + - `') { Write-Output "uVM $($vm.Id) was left running" ; Stop-ComputeProcess -Force -Id $vm.Id } ` - cmd := exec.Command("powershell.exe", "-NoLogo", " -NonInteractive", "-Command", cmdStr) - o, err := cmd.CombinedOutput() - s := string(o) - if err != nil { - logrus.Warningf("failed to cleanup remaining uVMs with command %q: %s: %v", cmdStr, s, err) - } else if len(o) > 0 { - logrus.Warningf("cleaned up left over uVMs: %s", strings.Split(s, "\r\n")) - } - - // delete downloaded layers; cant use defer, since os.exit does not run them - for _, l := range images { - // just log errors: no other cleanup possible - if err := l.Close(context.Background()); err != nil { - logrus.WithFields(logrus.Fields{ - logrus.ErrorKey: err, - "image": l.Image, - "platform": l.Platform, - }).Warning("layer cleanup failed") - } + if e := m.Run(); e != 0 { + return cli.Exit("", e) } - - os.Exit(e) + return nil } -func CreateContainerTestWrapper(ctx context.Context, options *hcsoci.CreateOptions) (cow.Container, *resources.Resources, error) { - if flagPauseAfterCreateContainerFailure != 0 { - options.DoNotReleaseResourcesOnFailure = true - } - s, r, err := hcsoci.CreateContainer(ctx, options) - if err != nil { - logrus.Warnf("Test is pausing for %s for debugging CreateContainer failure", flagPauseAfterCreateContainerFailure) - time.Sleep(flagPauseAfterCreateContainerFailure) - _ = resources.ReleaseResources(ctx, r, options.HostingSystem, true) - } - - return s, r, err -} +// misc helper functions/"global" values func requireFeatures(tb testing.TB, features ...string) { tb.Helper() require.Features(tb, flagFeatures, features...) } -func defaultLCOWOptions(tb testing.TB) *uvm.OptionsLCOW { +func defaultLCOWOptions(ctx context.Context, tb testing.TB) *uvm.OptionsLCOW { tb.Helper() - opts := testuvm.DefaultLCOWOptions(context.TODO(), tb, util.CleanName(tb.Name()), hcsOwner) + + opts := testuvm.DefaultLCOWOptions(ctx, tb, testName(tb), hcsOwner) if p := *flagLinuxBootFilesPath; p != "" { - opts.UpdateBootFilesPath(context.TODO(), p) + opts.UpdateBootFilesPath(ctx, p) } return opts } -//nolint:unused // will be used when WCOW tests are updated -func defaultWCOWOptions(tb testing.TB) *uvm.OptionsWCOW { +func defaultWCOWOptions(ctx context.Context, tb testing.TB) *uvm.OptionsWCOW { + tb.Helper() + + opts := testuvm.DefaultWCOWOptions(ctx, tb, testName(tb), hcsOwner) + uvmLayers := windowsImageLayers(ctx, tb) + scratchDir := testlayers.WCOWScratchDir(ctx, tb, "") + opts.LayerFolders = append(opts.LayerFolders, uvmLayers...) + opts.LayerFolders = append(opts.LayerFolders, scratchDir) + + return opts +} + +func testName(tb testing.TB, xs ...any) string { tb.Helper() - return uvm.NewDefaultOptionsWCOW(util.CleanName(tb.Name()), hcsOwner) + + return util.CleanName(tb.Name()) + util.RandNameSuffix(xs...) } // linuxImageLayers returns image layer paths appropriate for use as a container rootfs. @@ -248,28 +295,25 @@ func linuxImageLayers(ctx context.Context, tb testing.TB) []string { if ss := flagLCOWLayerPaths.Strings(); len(ss) > 0 { return ss } - if flagFeatures.IsSet(featureLCOWIntegrity) { - alpineWithVerity := &layers.LazyImageLayers{ - Image: images.ImageLinuxAlpineLatest, - Platform: images.PlatformLinux, - AppendVerity: true, - } - return alpineWithVerity.Layers(ctx, tb) - } return alpineImagePaths.Layers(ctx, tb) } // windowsImageLayers returns image layer paths appropriate for use as a uVM or container rootfs. // If layer paths were provided on the command line, they are returned. // Otherwise, it pulls an appropriate image. -// -//nolint:unused // will be used when WCOW tests are updated func windowsImageLayers(ctx context.Context, tb testing.TB) []string { tb.Helper() if ss := flagWCOWLayerPaths.Strings(); len(ss) > 0 { return ss } - return nanoserverImagePaths.Layers(ctx, tb) + + // should have checked error value before running tests, but just in case... + wcow, err := wcowImagePathsOnce() + if err != nil { + tb.Fatalf("could not get Windows Nano Server image: %v", err) + } + + return wcow.nanoserver.Layers(ctx, tb) } // windowsServercoreImageLayers returns image layer paths for Windows servercore. @@ -277,18 +321,47 @@ func windowsImageLayers(ctx context.Context, tb testing.TB) []string { // See [windowsImageLayers] for more. func windowsServercoreImageLayers(ctx context.Context, tb testing.TB) []string { tb.Helper() - return servercoreImagePaths.Layers(ctx, tb) + + wcow, err := wcowImagePathsOnce() + if err != nil { + tb.Fatalf("could not get Windows Server Core image: %v", err) + } + + return wcow.servercore.Layers(ctx, tb) } // namespacedContext returns a [context.Context] with the provided namespace added via // [github.com/containerd/containerd/namespaces.WithNamespace]. -func namespacedContext() context.Context { - return namespaces.WithNamespace(context.Background(), *flagContainerdNamespace) +func namespacedContext(ctx context.Context) context.Context { + // since this (usually) called at the start of a test, add the testing timeout to it + // for the entire test run + return namespaces.WithNamespace(ctx, *flagContainerdNamespace) } -func defaultLogLevel() string { - if os.Getenv("HCSSHIM_FUNCTIONAL_TESTS_DEBUG") != "" { - return logrus.DebugLevel.String() +// cleanupComputeSystems close any uVMs or containers that escaped during tests. +func cleanupComputeSystems(ctx context.Context, owner string) { + cmd := exec.Command("powershell.exe", "-NoProfile", "-NoLogo", "-NonInteractive", "-Command", + `foreach ( $s in Get-ComputeProcess -Owner '`+owner+`' ) { `+ + `Write-Output $s.Id ; $null = Stop-ComputeProcess -Force -Id $s.Id`+ + ` }`, + ) + + e := log.G(ctx).WithFields(logrus.Fields{ + "cmd": cmd.String(), + "owner": owner, + }) + e.Debug("removing leftover compute systems") + + o, err := cmd.CombinedOutput() + s := strings.TrimSpace(string(o)) + if err != nil { + e.WithFields(logrus.Fields{ + logrus.ErrorKey: err, + "output": s, + }).Warning("failed to cleanup leftover compute systems") + } else if len(o) > 0 { + e.WithField( + "systems", strings.Split(s, "\r\n"), // cmd should output one ID per line + ).Warning("cleaned up leftover compute systems") } - return logrus.WarnLevel.String() } diff --git a/test/functional/uvm_mem_backingtype_test.go b/test/functional/uvm_mem_backingtype_test.go index fece0b30ff..96b86f1fea 100644 --- a/test/functional/uvm_mem_backingtype_test.go +++ b/test/functional/uvm_mem_backingtype_test.go @@ -1,6 +1,5 @@ -//go:build windows && (functional || uvmmem) -// +build windows -// +build functional uvmmem +//go:build windows && functional +// +build windows,functional package functional @@ -10,8 +9,8 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" - "github.com/Microsoft/hcsshim/test/pkg/require" + "github.com/Microsoft/hcsshim/test/pkg/require" testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) @@ -23,6 +22,8 @@ func runMemStartLCOWTest(t *testing.T, opts *uvm.OptionsLCOW) { func runMemStartWCOWTest(t *testing.T, opts *uvm.OptionsWCOW) { t.Helper() + + //nolint:staticcheck // SA1019: deprecated; will be replaced when test is updated u, _, _ := testuvm.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") u.Close() } @@ -42,13 +43,13 @@ func runMemTests(t *testing.T, os string) { for _, bt := range testCases { if os == "windows" { - wopts := uvm.NewDefaultOptionsWCOW(t.Name(), "") + wopts := defaultWCOWOptions(context.Background(), t) wopts.MemorySizeInMB = 512 wopts.AllowOvercommit = bt.allowOvercommit wopts.EnableDeferredCommit = bt.enableDeferredCommit runMemStartWCOWTest(t, wopts) } else { - lopts := defaultLCOWOptions(t) + lopts := defaultLCOWOptions(context.Background(), t) lopts.MemorySizeInMB = 512 lopts.AllowOvercommit = bt.allowOvercommit lopts.EnableDeferredCommit = bt.enableDeferredCommit @@ -61,7 +62,7 @@ func TestMemBackingTypeWCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureUVM) runMemTests(t, "windows") } @@ -69,7 +70,7 @@ func TestMemBackingTypeLCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) runMemTests(t, "linux") } @@ -86,7 +87,7 @@ func runBenchMemStartTest(b *testing.B, opts *uvm.OptionsLCOW) { } } -func runBenchMemStartLcowTest(b *testing.B, allowOvercommit bool, enableDeferredCommit bool) { +func runBenchMemStartLCOWTest(b *testing.B, allowOvercommit bool, enableDeferredCommit bool) { b.Helper() for i := 0; i < b.N; i++ { opts := uvm.NewDefaultOptionsLCOW(b.Name(), "") @@ -101,25 +102,25 @@ func BenchmarkMemBackingTypeVirtualLCOW(b *testing.B) { b.Skip("not yet updated") require.Build(b, osversion.RS5) - requireFeatures(b, featureLCOW) + requireFeatures(b, featureLCOW, featureUVM) - runBenchMemStartLcowTest(b, true, false) + runBenchMemStartLCOWTest(b, true, false) } func BenchmarkMemBackingTypeVirtualDeferredLCOW(b *testing.B) { b.Skip("not yet updated") require.Build(b, osversion.RS5) - requireFeatures(b, featureLCOW) + requireFeatures(b, featureLCOW, featureUVM) - runBenchMemStartLcowTest(b, true, true) + runBenchMemStartLCOWTest(b, true, true) } func BenchmarkMemBackingTypePhyscialLCOW(b *testing.B) { b.Skip("not yet updated") require.Build(b, osversion.RS5) - requireFeatures(b, featureLCOW) + requireFeatures(b, featureLCOW, featureUVM) - runBenchMemStartLcowTest(b, false, false) + runBenchMemStartLCOWTest(b, false, false) } diff --git a/test/functional/uvm_memory_test.go b/test/functional/uvm_memory_test.go index c2ec1c71e0..f934452e83 100644 --- a/test/functional/uvm_memory_test.go +++ b/test/functional/uvm_memory_test.go @@ -19,12 +19,12 @@ func TestUVMMemoryUpdateLCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) defer cancel() - opts := defaultLCOWOptions(t) + opts := defaultLCOWOptions(ctx, t) opts.MemorySizeInMB = 1024 * 2 u := tuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) defer u.Close() @@ -47,7 +47,7 @@ func TestUVMMemoryUpdateWCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureUVM) ctx, cancel := context.WithTimeout(context.Background(), 40*time.Second) defer cancel() @@ -55,6 +55,7 @@ func TestUVMMemoryUpdateWCOW(t *testing.T) { opts := uvm.NewDefaultOptionsWCOW(t.Name(), "") opts.MemorySizeInMB = 1024 * 2 + //nolint:staticcheck // SA1019: deprecated; will be replaced when test is updated u, _, _ := tuvm.CreateWCOWUVMFromOptsWithImage(ctx, t, opts, "mcr.microsoft.com/windows/nanoserver:1909") defer u.Close() diff --git a/test/functional/uvm_plannine_test.go b/test/functional/uvm_plannine_test.go index 7f5c13c2bf..3b0b3d582d 100644 --- a/test/functional/uvm_plannine_test.go +++ b/test/functional/uvm_plannine_test.go @@ -1,7 +1,7 @@ //go:build windows && functional // +build windows,functional -// This file isn't called uvm_plan9_test.go as go test skips when a number is in it... go figure (pun intended) +// This file isn't called uvm_plan9_test.go as go assumes that it should only run on plan9 OS's... go figure (pun intended) package functional @@ -16,6 +16,7 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) @@ -26,11 +27,10 @@ func TestPlan9(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW, featurePlan9) - ctx := context.Background() + requireFeatures(t, featureLCOW, featureUVM, featurePlan9) + ctx := util.Context(context.Background(), t) - vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, defaultLCOWOptions(t)) - defer vm.Close() + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, defaultLCOWOptions(ctx, t)) dir := t.TempDir() var iterations uint32 = 64 @@ -52,26 +52,23 @@ func TestPlan9(t *testing.T) { } func TestPlan9_Writable(t *testing.T) { - // t.Skip("not yet updated") - require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW, featurePlan9) - ctx := context.Background() + requireFeatures(t, featureLCOW, featureUVM, featurePlan9) + ctx := util.Context(context.Background(), t) - opts := defaultLCOWOptions(t) + opts := defaultLCOWOptions(ctx, t) opts.NoWritableFileShares = true vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) - defer vm.Close() dir := t.TempDir() // mount as writable should fail - share, err := vm.AddPlan9(context.Background(), dir, fmt.Sprintf("/tmp/%s", filepath.Base(dir)), false, false, nil) + share, err := vm.AddPlan9(ctx, dir, fmt.Sprintf("/tmp/%s", filepath.Base(dir)), false, false, nil) defer func() { if share == nil { return } - if err := vm.RemovePlan9(context.Background(), share); err != nil { + if err := vm.RemovePlan9(ctx, share); err != nil { t.Fatalf("RemovePlan9 failed: %s", err) } }() @@ -80,7 +77,7 @@ func TestPlan9_Writable(t *testing.T) { } // mount as read-only should succeed - share, err = vm.AddPlan9(context.Background(), dir, fmt.Sprintf("/tmp/%s", filepath.Base(dir)), true, false, nil) + share, err = vm.AddPlan9(ctx, dir, fmt.Sprintf("/tmp/%s", filepath.Base(dir)), true, false, nil) if err != nil { t.Fatalf("AddPlan9 failed: %v", err) } diff --git a/test/functional/uvm_properties_test.go b/test/functional/uvm_properties_test.go index b9c74887c2..eaa039e57f 100644 --- a/test/functional/uvm_properties_test.go +++ b/test/functional/uvm_properties_test.go @@ -1,6 +1,5 @@ -//go:build windows && (functional || uvmproperties) -// +build windows -// +build functional uvmproperties +//go:build windows && functional +// +build windows,functional package functional @@ -9,17 +8,20 @@ import ( "testing" "github.com/Microsoft/hcsshim/osversion" + + "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" - tuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) func TestPropertiesGuestConnection_LCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) - uvm := tuvm.CreateAndStartLCOWFromOpts(context.Background(), t, defaultLCOWOptions(t)) + ctx := util.Context(context.Background(), t) + uvm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, defaultLCOWOptions(ctx, t)) defer uvm.Close() p, gc := uvm.Capabilities() @@ -34,9 +36,11 @@ func TestPropertiesGuestConnection_WCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureUVM) - uvm, _, _ := tuvm.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") + ctx := util.Context(context.Background(), t) + //nolint:staticcheck // SA1019: deprecated; will be replaced when test is updated + uvm, _, _ := testuvm.CreateWCOWUVM(ctx, t, t.Name(), "microsoft/nanoserver") defer uvm.Close() p, gc := uvm.Capabilities() diff --git a/test/functional/uvm_scratch_test.go b/test/functional/uvm_scratch_test.go index 0604380b10..ada16b2fea 100644 --- a/test/functional/uvm_scratch_test.go +++ b/test/functional/uvm_scratch_test.go @@ -1,6 +1,5 @@ -//go:build windows && (functional || uvmscratch) -// +build windows -// +build functional uvmscratch +//go:build windows && functional +// +build windows,functional package functional @@ -12,6 +11,7 @@ import ( "github.com/Microsoft/hcsshim/internal/lcow" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/test/pkg/require" tuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) @@ -20,7 +20,7 @@ func TestScratchCreateLCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW, featureScratch) + requireFeatures(t, featureLCOW, featureUVM, featureScratch) tempDir := t.TempDir() firstUVM := tuvm.CreateAndStartLCOW(context.Background(), t, "TestCreateLCOWScratch") @@ -64,6 +64,7 @@ func TestScratchCreateLCOW(t *testing.T) { //// VHDX and format it ext4. //func TestCreateLCOWScratch(t *testing.T) { // t.Skip("for now") +// // cacheDir := createTempDir(t) // cacheFile := filepath.Join(cacheDir, "cache.vhdx") // uvm, err := CreateContainer(&CreateOptions{Spec: getDefaultLinuxSpec(t)}) diff --git a/test/functional/uvm_scsi_test.go b/test/functional/uvm_scsi_test.go index 7b612ab1ef..7a5f3974ff 100644 --- a/test/functional/uvm_scsi_test.go +++ b/test/functional/uvm_scsi_test.go @@ -1,6 +1,5 @@ -//go:build windows && (functional || uvmscsi) -// +build windows -// +build functional uvmscsi +//go:build windows && functional +// +build windows,functional package functional @@ -14,27 +13,30 @@ import ( "sync" "testing" - "github.com/Microsoft/hcsshim/internal/wclayer" + "github.com/sirupsen/logrus" "github.com/Microsoft/hcsshim/internal/lcow" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/internal/uvm/scsi" + "github.com/Microsoft/hcsshim/internal/wclayer" "github.com/Microsoft/hcsshim/osversion" + testutilities "github.com/Microsoft/hcsshim/test/internal" + "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" - tuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" - "github.com/sirupsen/logrus" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) -// TestSCSIAddRemovev2LCOW validates adding and removing SCSI disks +// TestSCSIAddRemove2LCOW validates adding and removing SCSI disks // from a utility VM in both attach-only and with a container path. func TestSCSIAddRemoveLCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW, featureSCSI) + requireFeatures(t, featureLCOW, featureUVM, featureSCSI) - u := tuvm.CreateAndStartLCOWFromOpts(context.Background(), t, defaultLCOWOptions(t)) + ctx := util.Context(context.Background(), t) + u := testuvm.CreateAndStartLCOWFromOpts(ctx, t, defaultLCOWOptions(ctx, t)) defer u.Close() testSCSIAddRemoveMultiple(t, u, `/run/gcs/c/0/scsi`, "linux", []string{}) @@ -46,10 +48,12 @@ func TestSCSIAddRemoveWCOW(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureWCOW, featureSCSI) + requireFeatures(t, featureWCOW, featureUVM, featureSCSI) + ctx := util.Context(context.Background(), t) // TODO make the image configurable to the build we're testing on - u, layers, _ := tuvm.CreateWCOWUVM(context.Background(), t, t.Name(), "mcr.microsoft.com/windows/nanoserver:1903") + //nolint:staticcheck // SA1019: deprecated; will be replaced when test is updated + u, layers, _ := testuvm.CreateWCOWUVM(ctx, t, t.Name(), "mcr.microsoft.com/windows/nanoserver:1903") defer u.Close() testSCSIAddRemoveSingle(t, u, `c:\`, "windows", layers) @@ -221,14 +225,15 @@ func TestParallelScsiOps(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW, featureSCSI) + requireFeatures(t, featureLCOW, featureUVM, featureSCSI) - u := tuvm.CreateAndStartLCOWFromOpts(context.Background(), t, defaultLCOWOptions(t)) + ctx := util.Context(context.Background(), t) + u := testuvm.CreateAndStartLCOWFromOpts(ctx, t, defaultLCOWOptions(ctx, t)) defer u.Close() // Create a sandbox to use tempDir := t.TempDir() - if err := lcow.CreateScratch(context.Background(), u, filepath.Join(tempDir, "sandbox.vhdx"), lcow.DefaultScratchSizeGB, ""); err != nil { + if err := lcow.CreateScratch(ctx, u, filepath.Join(tempDir, "sandbox.vhdx"), lcow.DefaultScratchSizeGB, ""); err != nil { t.Fatalf("failed to create EXT4 scratch for LCOW test cases: %s", err) } copySandbox := func(dir string, workerId, iteration int) (string, error) { diff --git a/test/functional/uvm_update_test.go b/test/functional/uvm_update_test.go index 1f804350e2..3118d3fc7f 100644 --- a/test/functional/uvm_update_test.go +++ b/test/functional/uvm_update_test.go @@ -14,14 +14,17 @@ import ( "github.com/Microsoft/hcsshim/osversion" "github.com/Microsoft/hcsshim/pkg/ctrdtaskapi" + "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" - "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) -func Test_LCOW_Update_Resources(t *testing.T) { - requireFeatures(t, featureLCOW) +func TestLCOW_Update_Resources(t *testing.T) { + requireFeatures(t, featureLCOW, featureUVM) require.Build(t, osversion.RS5) + ctx := util.Context(context.Background(), t) + for _, config := range []struct { name string resource interface{} @@ -54,9 +57,8 @@ func Test_LCOW_Update_Resources(t *testing.T) { }, } { t.Run(config.name, func(t *testing.T) { - ctx := context.Background() - vm, cleanup := uvm.CreateLCOW(ctx, t, defaultLCOWOptions(t)) - uvm.Start(ctx, t, vm) + vm, cleanup := testuvm.CreateLCOW(ctx, t, defaultLCOWOptions(ctx, t)) + testuvm.Start(ctx, t, vm) defer cleanup(ctx) if err := vm.Update(ctx, config.resource, nil); err != nil { if config.valid { diff --git a/test/functional/uvm_virtualdevice_test.go b/test/functional/uvm_virtualdevice_test.go index 725f9e0553..5a8aba8a43 100644 --- a/test/functional/uvm_virtualdevice_test.go +++ b/test/functional/uvm_virtualdevice_test.go @@ -12,8 +12,9 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/test/pkg/require" - tuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) // findTestDevices returns the first pcip device on the host @@ -32,7 +33,7 @@ func TestVirtualDevice(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.V20H1) - requireFeatures(t, featureLCOW) + requireFeatures(t, featureLCOW, featureUVM) ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() @@ -56,7 +57,7 @@ func TestVirtualDevice(t *testing.T) { opts.PreferredRootFSType = uvm.PreferredRootFSTypeInitRd // create test uvm and ensure we can assign and remove the device - vm := tuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) + vm := testuvm.CreateAndStartLCOWFromOpts(ctx, t, opts) defer vm.Close() vpci, err := vm.AssignDevice(ctx, testDeviceInstanceID, 0, "") if err != nil { diff --git a/test/functional/uvm_vpmem_test.go b/test/functional/uvm_vpmem_test.go index 78f9ee7645..2fee325f71 100644 --- a/test/functional/uvm_vpmem_test.go +++ b/test/functional/uvm_vpmem_test.go @@ -1,6 +1,5 @@ -//go:build windows && (functional || uvmvpmem) -// +build windows -// +build functional uvmvpmem +//go:build windows && functional +// +build windows,functional package functional @@ -12,20 +11,22 @@ import ( "github.com/Microsoft/hcsshim/internal/copyfile" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + + "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" - tuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) -// TestVPMEM tests adding/removing VPMem Read-Only layers from a v2 Linux utility VM +// TestVPMEM tests adding/removing VPMem Read-Only layers from a v2 Linux utility VM. func TestVPMEM(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureLCOW, featureVPMEM) + requireFeatures(t, featureLCOW, featureUVM, featureVPMEM) - ctx := context.Background() + ctx := util.Context(context.Background(), t) layers := linuxImageLayers(ctx, t) - u := tuvm.CreateAndStartLCOW(ctx, t, t.Name()) + u := testuvm.CreateAndStartLCOW(ctx, t, t.Name()) defer u.Close() var iterations uint32 = uvm.MaxVPMEMCount diff --git a/test/functional/uvm_vsmb_test.go b/test/functional/uvm_vsmb_test.go index 9c037603b5..1e4b2b152f 100644 --- a/test/functional/uvm_vsmb_test.go +++ b/test/functional/uvm_vsmb_test.go @@ -1,6 +1,5 @@ -//go:build windows && (functional || uvmvsmb) -// +build windows -// +build functional uvmvsmb +//go:build windows && functional +// +build windows,functional package functional @@ -13,18 +12,21 @@ import ( "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/osversion" + "github.com/Microsoft/hcsshim/test/internal/util" "github.com/Microsoft/hcsshim/test/pkg/require" - tuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" + testuvm "github.com/Microsoft/hcsshim/test/pkg/uvm" ) -// TestVSMB tests adding/removing VSMB layers from a v2 Windows utility VM +// TestVSMB tests adding/removing VSMB layers from a v2 Windows utility VM. func TestVSMB(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureWCOW, featureVSMB) + requireFeatures(t, featureWCOW, featureUVM, featureVSMB) - uvm, _, _ := tuvm.CreateWCOWUVM(context.Background(), t, t.Name(), "microsoft/nanoserver") + ctx := util.Context(context.Background(), t) + //nolint:staticcheck // SA1019: deprecated; will be replaced when test is updated + uvm, _, _ := testuvm.CreateWCOWUVM(ctx, t, t.Name(), "microsoft/nanoserver") defer uvm.Close() dir := t.TempDir() @@ -32,14 +34,14 @@ func TestVSMB(t *testing.T) { options := uvm.DefaultVSMBOptions(true) options.TakeBackupPrivilege = true for i := 0; i < int(iterations); i++ { - if _, err := uvm.AddVSMB(context.Background(), dir, options); err != nil { + if _, err := uvm.AddVSMB(ctx, dir, options); err != nil { t.Fatalf("AddVSMB failed: %s", err) } } // Remove them all for i := 0; i < int(iterations); i++ { - if err := uvm.RemoveVSMB(context.Background(), dir, true); err != nil { + if err := uvm.RemoveVSMB(ctx, dir, true); err != nil { t.Fatalf("RemoveVSMB failed: %s", err) } } @@ -51,23 +53,25 @@ func TestVSMB_Writable(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureWCOW, featureVSMB) + requireFeatures(t, featureWCOW, featureUVM, featureVSMB) + ctx := util.Context(context.Background(), t) opts := uvm.NewDefaultOptionsWCOW(t.Name(), "") opts.NoWritableFileShares = true - vm, _, _ := tuvm.CreateWCOWUVMFromOptsWithImage(context.Background(), t, opts, "microsoft/nanoserver") + //nolint:staticcheck // SA1019: deprecated; will be replaced when test is updated + vm, _, _ := testuvm.CreateWCOWUVMFromOptsWithImage(ctx, t, opts, "microsoft/nanoserver") defer vm.Close() dir := t.TempDir() options := vm.DefaultVSMBOptions(true) options.TakeBackupPrivilege = true options.ReadOnly = false - _, err := vm.AddVSMB(context.Background(), dir, options) + _, err := vm.AddVSMB(ctx, dir, options) defer func() { if err == nil { return } - if err = vm.RemoveVSMB(context.Background(), dir, true); err != nil { + if err = vm.RemoveVSMB(ctx, dir, true); err != nil { t.Fatalf("RemoveVSMB failed: %s", err) } }() @@ -77,7 +81,7 @@ func TestVSMB_Writable(t *testing.T) { } options.ReadOnly = true - _, err = vm.AddVSMB(context.Background(), dir, options) + _, err = vm.AddVSMB(ctx, dir, options) if err != nil { t.Fatalf("AddVSMB failed: %s", err) } diff --git a/test/functional/wcow_test.go b/test/functional/wcow_test.go index 3061fdffa8..513ab742ef 100644 --- a/test/functional/wcow_test.go +++ b/test/functional/wcow_test.go @@ -1,6 +1,5 @@ -//go:build windows && (functional || wcow) -// +build windows -// +build functional wcow +//go:build windows && functional +// +build windows,functional package functional @@ -368,7 +367,7 @@ func generateShimLayersStruct(t *testing.T, imageLayers []string) []hcsshim.Laye func TestWCOWArgonShim(t *testing.T) { t.Skip("not yet updated") - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureContainer) imageLayers := windowsServercoreImageLayers(context.Background(), t) @@ -429,7 +428,7 @@ func TestWCOWArgonShim(t *testing.T) { func TestWCOWXenonShim(t *testing.T) { t.Skip("not yet updated") - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureUVM) imageLayers := windowsServercoreImageLayers(context.Background(), t) @@ -500,7 +499,7 @@ func generateWCOWOciTestSpec(t *testing.T, imageLayers []string, scratchPath, ho func TestWCOWArgonOciV1(t *testing.T) { t.Skip("not yet updated") - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureContainer) imageLayers := windowsServercoreImageLayers(context.Background(), t) argonOci1Mounted := false @@ -548,7 +547,7 @@ func TestWCOWArgonOciV1(t *testing.T) { func TestWCOWXenonOciV1(t *testing.T) { t.Skip("not yet updated") - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureUVM) imageLayers := windowsServercoreImageLayers(context.Background(), t) xenonOci1Mounted := false @@ -605,7 +604,7 @@ func TestWCOWArgonOciV2(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureContainer) imageLayers := windowsServercoreImageLayers(context.Background(), t) argonOci2Mounted := false @@ -655,7 +654,7 @@ func TestWCOWXenonOciV2(t *testing.T) { t.Skip("not yet updated") require.Build(t, osversion.RS5) - requireFeatures(t, featureWCOW) + requireFeatures(t, featureWCOW, featureUVM) imageLayers := windowsServercoreImageLayers(context.Background(), t) xenonOci2Mounted := false diff --git a/test/go.mod b/test/go.mod index c2a7632513..7e83eefc3e 100644 --- a/test/go.mod +++ b/test/go.mod @@ -18,6 +18,7 @@ require ( github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 + github.com/urfave/cli/v2 v2.25.7 go.opencensus.io v0.24.0 golang.org/x/sync v0.6.0 golang.org/x/sys v0.17.0 @@ -42,6 +43,7 @@ require ( github.com/containerd/stargz-snapshotter/estargz v0.14.3 // indirect github.com/containerd/typeurl/v2 v2.1.1 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/docker/cli v24.0.0+incompatible // indirect github.com/docker/distribution v2.8.2+incompatible // indirect @@ -82,6 +84,7 @@ require ( github.com/opencontainers/selinux v1.11.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect + github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/vbatts/tar-split v0.11.3 // indirect github.com/vektah/gqlparser/v2 v2.4.5 // indirect @@ -91,6 +94,7 @@ require ( github.com/x448/float16 v0.8.4 // indirect github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect github.com/yashtewari/glob-intersection v0.1.0 // indirect go.opentelemetry.io/otel v1.21.0 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect diff --git a/test/go.sum b/test/go.sum index 5d49d9743f..60896fc349 100644 --- a/test/go.sum +++ b/test/go.sum @@ -1517,6 +1517,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -2255,6 +2256,7 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= @@ -2358,6 +2360,8 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= @@ -2391,6 +2395,8 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yashtewari/glob-intersection v0.1.0 h1:6gJvMYQlTDOL3dMsPF6J0+26vwX9MB8/1q3uAdhmTrg= github.com/yashtewari/glob-intersection v0.1.0/go.mod h1:LK7pIC3piUjovexikBbJ26Yml7g8xa5bsjfx2v1fwok= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/test/internal/cmd/cmd.go b/test/internal/cmd/cmd.go index af00ef4cde..8e61478919 100644 --- a/test/internal/cmd/cmd.go +++ b/test/internal/cmd/cmd.go @@ -24,7 +24,11 @@ const ForcedKilledExitCode = 137 func desc(c *cmd.Cmd) string { desc := "init command" if c.Spec != nil { - desc = strings.Join(c.Spec.Args, " ") + if c.Spec.CommandLine != "" { + desc = c.Spec.CommandLine + } else { + desc = strings.Join(c.Spec.Args, " ") + } } return desc diff --git a/test/internal/container/container.go b/test/internal/container/container.go index 24cb2fe11d..5246c257c5 100644 --- a/test/internal/container/container.go +++ b/test/internal/container/container.go @@ -12,61 +12,80 @@ import ( "github.com/Microsoft/hcsshim/internal/cmd" "github.com/Microsoft/hcsshim/internal/cow" "github.com/Microsoft/hcsshim/internal/hcsoci" + "github.com/Microsoft/hcsshim/internal/jobcontainers" "github.com/Microsoft/hcsshim/internal/layers" + "github.com/Microsoft/hcsshim/internal/oci" "github.com/Microsoft/hcsshim/internal/resources" "github.com/Microsoft/hcsshim/internal/uvm" testcmd "github.com/Microsoft/hcsshim/test/internal/cmd" ) +// TODO: update cleanup func to accept context, same as uVM cleanup + +// a test version of the container creation logic in: cmd\containerd-shim-runhcs-v1\task_hcs.go:createContainer func Create( ctx context.Context, tb testing.TB, vm *uvm.UtilityVM, spec *specs.Spec, name, owner string, -) (cow.Container, *resources.Resources, func()) { +) (c cow.Container, r *resources.Resources, _ func()) { tb.Helper() if spec.Windows == nil || spec.Windows.Network == nil || spec.Windows.LayerFolders == nil { tb.Fatalf("improperly configured windows spec for container %q: %#+v", name, spec.Windows) } - co := &hcsoci.CreateOptions{ - ID: name, - HostingSystem: vm, - Owner: owner, - Spec: spec, - // dont create a network namespace on the host side - NetworkNamespace: "", //spec.Windows.Network.NetworkNamespace, - } - - if co.Spec.Linux != nil { - var layerFolders []string - if co.Spec.Windows != nil { - layerFolders = co.Spec.Windows.LayerFolders - } - if len(layerFolders) <= 1 { - tb.Fatalf("LCOW requires at least 2 layers (including scratch): %v", layerFolders) + var err error + if oci.IsJobContainer(spec) { + c, r, err = jobcontainers.Create(ctx, name, spec) + } else { + co := &hcsoci.CreateOptions{ + ID: name, + HostingSystem: vm, + Owner: owner, + Spec: spec, + // Don't create a network namespace on the host side: + // If one is needed, it'll be created manually during testing. + // Additionally, these are "standalone" containers, and not CRI pod/workload containers, + // so leave end-to-end testing with namespaces for CRI tests + NetworkNamespace: "", } - scratch := layerFolders[len(layerFolders)-1] - parents := layerFolders[:len(layerFolders)-1] - // todo: support partitioned layers - co.LCOWLayers = &layers.LCOWLayers{ - Layers: make([]*layers.LCOWLayer, 0, len(parents)), - ScratchVHDPath: filepath.Join(scratch, "sandbox.vhdx"), + if co.Spec.Linux != nil { + if vm == nil { + tb.Fatalf("LCOW requires a uVM") + } + + var layerFolders []string + if co.Spec.Windows != nil { + layerFolders = co.Spec.Windows.LayerFolders + } + if len(layerFolders) <= 1 { + tb.Fatalf("LCOW requires at least 2 layers (including scratch): %v", layerFolders) + } + scratch := layerFolders[len(layerFolders)-1] + parents := layerFolders[:len(layerFolders)-1] + + // todo: support partitioned layers + co.LCOWLayers = &layers.LCOWLayers{ + Layers: make([]*layers.LCOWLayer, 0, len(parents)), + ScratchVHDPath: filepath.Join(scratch, "sandbox.vhdx"), + } + + for _, p := range parents { + co.LCOWLayers.Layers = append(co.LCOWLayers.Layers, &layers.LCOWLayer{VHDPath: filepath.Join(p, "layer.vhd")}) + } } - for _, p := range parents { - co.LCOWLayers.Layers = append(co.LCOWLayers.Layers, &layers.LCOWLayer{VHDPath: filepath.Join(p, "layer.vhd")}) - } + c, r, err = hcsoci.CreateContainer(ctx, co) } - c, r, err := hcsoci.CreateContainer(ctx, co) if err != nil { - tb.Fatalf("could not create container %q: %v", co.ID, err) + tb.Fatalf("could not create container %q: %v", name, err) } + f := func() { if err := resources.ReleaseResources(ctx, r, vm, true); err != nil { tb.Errorf("failed to release container resources: %v", err) @@ -79,14 +98,27 @@ func Create( return c, r, f } +// todo: unify Start and StartWithSpec and add logic to check for WCOW + +// for starting an LCOW container, where no process spec is passed +// +// see: +// - github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/exec_hcs.go: (*hcsExec).startInternal +// - github.com/Microsoft/hcsshim/cmd/internal/cmd/cmd.go: (*Cmd).Start func Start(ctx context.Context, tb testing.TB, c cow.Container, io *testcmd.BufferedIO) *cmd.Cmd { tb.Helper() + + // OCI spec is nil to tell bridge to start container's init process + return StartWithSpec(ctx, tb, c, nil, io) +} + +func StartWithSpec(ctx context.Context, tb testing.TB, c cow.Container, p *specs.Process, io *testcmd.BufferedIO) *cmd.Cmd { + tb.Helper() if err := c.Start(ctx); err != nil { tb.Fatalf("could not start %q: %v", c.ID(), err) } - // OCI spec is nil to tell bridge to start container's init process - init := testcmd.Create(ctx, tb, c, nil, io) + init := testcmd.Create(ctx, tb, c, p, io) testcmd.Start(ctx, tb, init) return init diff --git a/test/internal/layers/lazy.go b/test/internal/layers/lazy.go index a922e4bf53..f0fa97096f 100644 --- a/test/internal/layers/lazy.go +++ b/test/internal/layers/lazy.go @@ -7,19 +7,24 @@ import ( "fmt" "io" "os" + "os/exec" "path/filepath" "runtime" "strconv" + "strings" "sync" "testing" - "github.com/Microsoft/hcsshim/ext4/dmverity" - "github.com/Microsoft/hcsshim/internal/security" + "github.com/Microsoft/go-winio" "github.com/google/go-containerregistry/pkg/crane" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/sirupsen/logrus" + "github.com/Microsoft/hcsshim/ext4/dmverity" "github.com/Microsoft/hcsshim/ext4/tar2ext4" "github.com/Microsoft/hcsshim/internal/log" + "github.com/Microsoft/hcsshim/internal/security" + isync "github.com/Microsoft/hcsshim/internal/sync" "github.com/Microsoft/hcsshim/pkg/ociwclayer" "github.com/Microsoft/hcsshim/test/internal/util" @@ -28,6 +33,8 @@ import ( // helper utilities for dealing with images +// TODO: create a `type innerImageLayers struct{ dir, layers }` and use `testsync.Once[innerImageLayers]` instead + type LazyImageLayers struct { Image string Platform string @@ -38,7 +45,7 @@ type LazyImageLayers struct { // dedicated directory, under [TempPath], to store layers in dir string once sync.Once - layers []string + layers []string // extracted layer directories, under [dir] } type extractHandler func(ctx context.Context, rc io.ReadCloser, dir string, parents []string) error @@ -47,10 +54,19 @@ type extractHandler func(ctx context.Context, rc io.ReadCloser, dir string, pare // // Does not take a [testing.TB] so it can be used in TestMain or init. func (x *LazyImageLayers) Close(ctx context.Context) error { - if x.dir == "" { + if x == nil || x.dir == "" { return nil } + log.G(ctx).WithFields(logrus.Fields{ + "dir": x.dir, + "image": x.Image, + }).Debug("removing image") + + if _, err := os.Stat(x.dir); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("path %q is not valid: %w", x.dir, err) + } + // DestroyLayer will remove the entire directory and all its contents, regardless of if // its a Windows container layer or not. if err := util.DestroyLayer(ctx, x.dir); err != nil { @@ -63,6 +79,11 @@ func (x *LazyImageLayers) Close(ctx context.Context) error { func (x *LazyImageLayers) Layers(ctx context.Context, tb testing.TB) []string { // basically combo of containerd fetch and unpack (snapshotter + differ) tb.Helper() + + if x == nil { + return nil + } + var err error x.once.Do(func() { err = x.extractLayers(ctx) @@ -77,10 +98,15 @@ func (x *LazyImageLayers) Layers(ctx context.Context, tb testing.TB) []string { // don't use tb.Error/Log inside Once.Do stack, since we cannot call tb.Helper before executing f() // within Once.Do and that will therefore show the wrong stack/location func (x *LazyImageLayers) extractLayers(ctx context.Context) (err error) { - log.G(ctx).Infof("pulling and unpacking %s image %q", x.Platform, x.Image) + if x.Image == "" { + return fmt.Errorf("cannot return layers for an empty image") + } if x.TempPath == "" { - dir := os.TempDir() + dir, err := tempDirOnce(ctx) + if err != nil { + return err + } x.dir, err = os.MkdirTemp(dir, util.CleanName(x.Image)) if err != nil { return fmt.Errorf("failed to create temp directory: %w", err) @@ -92,6 +118,12 @@ func (x *LazyImageLayers) extractLayers(ctx context.Context) (err error) { } } + log.G(ctx).WithFields(logrus.Fields{ + "platform": x.Platform, + "image": x.Image, + "path": x.dir, + }).Info("pulling and unpacking image layers") + extract, err := extractImageHandler(x.Platform, x.AppendVerity) if err != nil { return err @@ -133,7 +165,7 @@ func extractImageHandler(platform string, appendVerity bool) (extractHandler, er if appendVerity { extract = withAppendVerity(extract) } - extract = withVhdFooter(extract) + extract = withVHDFooter(extract) return extract, nil } else if platform == images.PlatformWindows { return windowsImage, nil @@ -185,7 +217,7 @@ func withAppendVerity(fn extractHandler) extractHandler { } } -func withVhdFooter(fn extractHandler) extractHandler { +func withVHDFooter(fn extractHandler) extractHandler { return func(ctx context.Context, rc io.ReadCloser, dir string, parents []string) error { if err := fn(ctx, rc, dir, parents); err != nil { return err @@ -207,10 +239,56 @@ func withVhdFooter(fn extractHandler) extractHandler { } } +var procPrivilegesOnce = isync.OnceValueCtx(func(ctx context.Context) (struct{}, error) { + privs := []string{winio.SeBackupPrivilege, winio.SeRestorePrivilege} + log.G(ctx).WithField("privileges", privs).Infof("enableing process privileges") + + return struct{}{}, winio.EnableProcessPrivileges(privs) +}) + func windowsImage(ctx context.Context, rc io.ReadCloser, dir string, parents []string) error { + if _, err := procPrivilegesOnce(ctx); err != nil { + return fmt.Errorf("enable process Backup and Restore privileges: %w", err) + } + if _, err := ociwclayer.ImportLayerFromTar(ctx, rc, dir, parents); err != nil { return fmt.Errorf("import wc layer %s: %w", dir, err) } return nil } + +// tempDirOnce returns a dedicated folder in [os.TempDir] that is used for all image layers +// (that don't specify their [TempDir] field), as well as scratch and cache VHDs. +// +// The directory is excluded from MS Defender, and persisted between test runs to reduce +// the overhead associated with unpacking images and creating uVMs. +// +// Since the folder is under TempDir, it will be removed on OS restart, so it is fine to persist +// between test runs +var tempDirOnce = isync.OnceValueCtx(func(ctx context.Context) (string, error) { + // ! DO NOT DELETE THE FOLDER CREATED BY THIS FUNC: + // We want a "stable" (relative to OS restart) directory for image layers and + // scratch files that we can avoid needing to recreate and re-adding defender exclusions to. + + dir := filepath.Join(os.TempDir(), "hcsshim-test") + if err := os.MkdirAll(dir, 0700); err != nil { + return "", fmt.Errorf("create hcsshim testing temp directory: %w", err) + } + + cmd := exec.Command("powershell.exe", "-NoLogo", "-NonInteractive", "-Command", + "Add-MpPreference -ExclusionPath '"+dir+"'") + o, err := cmd.CombinedOutput() + if err != nil { + // not necessary to creating the image layers path, so log then ignore error + log.G(ctx).WithFields(logrus.Fields{ + "cmd": cmd.String(), + "output": strings.TrimSpace(string(o)), + "path": dir, + }).WithError(err).Warning("failed to add MS defender exclusion for image layers directory") + } else { + log.G(ctx).WithField("path", dir).Info("added MS Defender exclusion for image layers directory") + } + + return dir, nil +}) diff --git a/test/internal/layers/scratch.go b/test/internal/layers/scratch.go index 82b7a0ae81..8d8f24402e 100644 --- a/test/internal/layers/scratch.go +++ b/test/internal/layers/scratch.go @@ -4,6 +4,7 @@ package layers import ( "context" + "os" "path/filepath" "testing" @@ -19,29 +20,25 @@ const ( UVMScratchSpaceName = "uvmscratch.vhdx" ) -func CacheFile(_ context.Context, tb testing.TB, dir string) string { +func CacheFile(ctx context.Context, tb testing.TB, dir string) string { tb.Helper() if dir == "" { - dir = tb.TempDir() - tb.Cleanup(func() { - _ = util.RemoveAll(dir) - }) + dir = newTestTempDir(ctx, tb, "") } cache := filepath.Join(dir, CacheFileName) return cache } -// ScratchSpace creates an LCOW scratch space VHD at `dir\name`, and returns the dir and name. -// If name, (dir, or cache) are empty, ScratchSpace uses [ScratchSpace] (creates a temporary +// ScratchSpace creates an LCOW scratch space VHD at `dir/name`, and returns the directory and +// scratch space file path. +// If name (or dir or cache) is empty, ScratchSpace uses [ScratchSpaceName] (creates a temporary // directory), respectively. func ScratchSpace(ctx context.Context, tb testing.TB, vm *uvm.UtilityVM, name, dir, cache string) (string, string) { tb.Helper() if dir == "" { - dir = tb.TempDir() - tb.Cleanup(func() { - _ = util.RemoveAll(dir) - }) + dir = newTestTempDir(ctx, tb, vm.ID()) } + if cache == "" { cache = CacheFile(ctx, tb, dir) } @@ -56,3 +53,40 @@ func ScratchSpace(ctx context.Context, tb testing.TB, vm *uvm.UtilityVM, name, d return dir, scratch } + +func WCOWScratchDir(ctx context.Context, tb testing.TB, dir string) string { + tb.Helper() + if dir == "" { + dir = newTestTempDir(ctx, tb, "") + } + + tb.Cleanup(func() { + if err := util.DestroyLayer(ctx, dir); err != nil { + tb.Errorf("failed to destroy %q: %v", dir, err) + } + }) + + return dir +} + +func newTestTempDir(ctx context.Context, tb testing.TB, name string) string { + tb.Helper() + dir, err := tempDirOnce(ctx) + if err != nil { + tb.Fatal(err) + } + + if name == "" { + name = util.CleanName(tb.Name()) + } + dir, err = os.MkdirTemp(dir, name) + if err != nil { + tb.Fatalf("create test temp directory: %v", err) + } + + tb.Cleanup(func() { + _ = util.RemoveAll(dir) + }) + + return dir +} diff --git a/test/internal/oci/oci.go b/test/internal/oci/oci.go index ed1068f8ed..197bf94cf0 100644 --- a/test/internal/oci/oci.go +++ b/test/internal/oci/oci.go @@ -5,12 +5,14 @@ import ( "errors" "testing" - "github.com/Microsoft/hcsshim/test/pkg/images" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/namespaces" ctrdoci "github.com/containerd/containerd/oci" criconstants "github.com/containerd/containerd/pkg/cri/constants" + criopts "github.com/containerd/containerd/pkg/cri/opts" "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/Microsoft/hcsshim/test/pkg/images" ) // @@ -30,16 +32,14 @@ const ( func DefaultLinuxSpecOpts(nns string, extra ...ctrdoci.SpecOpts) []ctrdoci.SpecOpts { opts := []ctrdoci.SpecOpts{ - WithoutRunMount, + ctrdoci.WithoutRunMount, ctrdoci.WithRootFSReadonly(), - WithDisabledCgroups, // we set our own cgroups + criopts.WithDisabledCgroups, // we set our own cgroups ctrdoci.WithDefaultUnixDevices, ctrdoci.WithDefaultPathEnv, - WithWindowsNetworkNamespace(nns), + ctrdoci.WithWindowsNetworkNamespace(nns), } - opts = append(opts, extra...) - - return opts + return append(opts, extra...) } // DefaultLinuxSpec returns a default OCI spec for a Linux container. @@ -68,7 +68,7 @@ func CreateWindowsSpec(ctx context.Context, tb testing.TB, id string, opts ...ct // CreateSpecWithPlatform returns the OCI spec for the specified platform. // The context must contain a containerd namespace added by -// [github.com/containerd/containerd/namespaces.WithNamespace] +// [github.com/containerd/containerd/namespaces.WithNamespace]. func CreateSpecWithPlatform(ctx context.Context, tb testing.TB, plat, id string, opts ...ctrdoci.SpecOpts) *specs.Spec { tb.Helper() container := &containers.Container{ID: id} @@ -82,7 +82,7 @@ func CreateSpecWithPlatform(ctx context.Context, tb testing.TB, plat, id string, } func WithWindowsLayerFolders(layers []string) ctrdoci.SpecOpts { - return func(ctx context.Context, client ctrdoci.Client, c *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ ctrdoci.Client, _ *containers.Container, s *specs.Spec) error { if len(layers) < 2 { return errors.New("at least two layers are required, including the sandbox path") } @@ -95,37 +95,3 @@ func WithWindowsLayerFolders(layers []string) ctrdoci.SpecOpts { return nil } } - -//defined in containerd\pkg\cri\opts\spec_windows.go - -func WithWindowsNetworkNamespace(path string) ctrdoci.SpecOpts { - return func(ctx context.Context, client ctrdoci.Client, c *containers.Container, s *specs.Spec) error { - if s.Windows == nil { - s.Windows = &specs.Windows{} - } - if s.Windows.Network == nil { - s.Windows.Network = &specs.WindowsNetwork{} - } - s.Windows.Network.NetworkNamespace = path - - return nil - } -} - -//defined in containerd\pkg\cri\opts\spec_linux.go - -// WithDisabledCgroups clears the Cgroups Path from the spec -func WithDisabledCgroups(_ context.Context, _ ctrdoci.Client, c *containers.Container, s *specs.Spec) error { - if s.Linux == nil { - s.Linux = &specs.Linux{} - } - s.Linux.CgroupsPath = "" - return nil -} - -// defined in containerd\oci\spec_opts_linux.go - -// WithoutRunMount removes the `/run` inside the spec -func WithoutRunMount(ctx context.Context, client ctrdoci.Client, c *containers.Container, s *specs.Spec) error { - return ctrdoci.WithoutMounts("/run")(ctx, client, c, s) -} diff --git a/test/internal/util/util.go b/test/internal/util/util.go index 22ffe0a428..edab2fc495 100644 --- a/test/internal/util/util.go +++ b/test/internal/util/util.go @@ -1,6 +1,7 @@ package util import ( + "context" "crypto/rand" "encoding/hex" "flag" @@ -153,3 +154,48 @@ func repeat(f func() error, n int, d time.Duration) (err error) { return err } + +// Context creates a [context.Context] that uses the testing.Deadline minus a small grace period (if applicable) +// and the cancellation to the testing cleanup. +// +// Based heavily on (copied directly from): Go lang's src/internal/testenv/Command.Context +// https://cs.opensource.google/go/go/+/master:src/internal/testenv/exec.go;l=133;drc=5613882df7555484680ecabc0462b7c23c6f5205 +func Context(ctx context.Context, tb testing.TB) context.Context { + tb.Helper() + + var ( + cancelCtx context.CancelFunc + gracePeriod time.Duration // unlimited unless the test has a deadline (to allow for interactive debugging) + ) + + if t, ok := tb.(interface { + testing.TB + Deadline() (time.Time, bool) + }); ok { + if td, ok := t.Deadline(); ok { + // Start with a minimum grace period, to allow cleanup before testing is stopped + gracePeriod = 100 * time.Millisecond + + // If time allows, increase the termination grace period to 5% of the + // test's remaining time. + testTimeout := time.Until(td) + if gp := testTimeout / 20; gp > gracePeriod { + gracePeriod = gp + } + + timeout := testTimeout - 2*gracePeriod + if cd, ok := ctx.Deadline(); !ok || time.Until(cd) > timeout { + // Either ctx doesn't have a deadline, or its deadline would expire + // after (or too close before) the test has already timed out. + // Add a shorter timeout so that the test will produce useful output. + ctx, cancelCtx = context.WithTimeout(ctx, timeout) + } + } + } + + if cancelCtx != nil { + tb.Cleanup(cancelCtx) + } + + return ctx +} diff --git a/test/pkg/uvm/lcow.go b/test/pkg/uvm/lcow.go index 6b898915b7..cdb65917f2 100644 --- a/test/pkg/uvm/lcow.go +++ b/test/pkg/uvm/lcow.go @@ -9,12 +9,11 @@ import ( "path/filepath" "testing" + "github.com/Microsoft/hcsshim/internal/sync" "github.com/Microsoft/hcsshim/internal/uvm" ) -var lcowOSBootFiles string - -func init() { +var lcowOSBootFilesOnce = sync.OnceValue(func() (string, error) { // since the tests can be run from directories outside of where containerd and the // LinuxBootFiles are, search through potential locations for the boot files // first start with where containerd is, since there may be a leftover C:\ContainerPlat @@ -27,11 +26,11 @@ func init() { for _, p := range paths { p = filepath.Join(p, "LinuxBootFiles") if _, err := os.Stat(p); err == nil { - lcowOSBootFiles = p - break + return p, nil } } -} + return "", nil +}) // DefaultLCOWOptions returns default options for a bootable LCOW uVM, but first checks // if `containerd.exe` is in the path, or C:\ContainerPlat\LinuxBootFiles exists, and @@ -43,9 +42,10 @@ func init() { // See [uvm.NewDefaultOptionsLCOW] for more information. func DefaultLCOWOptions(ctx context.Context, tb testing.TB, id, owner string) *uvm.OptionsLCOW { tb.Helper() + opts := uvm.NewDefaultOptionsLCOW(id, owner) - if lcowOSBootFiles != "" { - opts.UpdateBootFilesPath(ctx, lcowOSBootFiles) + if v, _ := lcowOSBootFilesOnce(); v != "" { + opts.UpdateBootFilesPath(ctx, v) } return opts } @@ -82,12 +82,7 @@ func CreateLCOW(ctx context.Context, tb testing.TB, opts *uvm.OptionsLCOW) (*uvm tb.Fatalf("could not create LCOW UVM: %v", err) } - f := func(ctx context.Context) { - if err := vm.CloseCtx(ctx); err != nil { - tb.Logf("could not close vm %q: %v", vm.ID(), err) - } - } - return vm, f + return vm, newCleanupFn(ctx, tb, vm) } func SetSecurityPolicy(ctx context.Context, tb testing.TB, vm *uvm.UtilityVM, policy string) { diff --git a/test/pkg/uvm/uvm.go b/test/pkg/uvm/uvm.go index 63b7a295f6..2cd0a36d51 100644 --- a/test/pkg/uvm/uvm.go +++ b/test/pkg/uvm/uvm.go @@ -11,6 +11,20 @@ import ( type CleanupFn = func(context.Context) +func newCleanupFn(_ context.Context, tb testing.TB, vm *uvm.UtilityVM) CleanupFn { + tb.Helper() + + return func(ctx context.Context) { + if vm == nil { + return + } + + if err := vm.CloseCtx(ctx); err != nil { + tb.Errorf("could not close vm %q: %v", vm.ID(), err) + } + } +} + func Start(ctx context.Context, tb testing.TB, vm *uvm.UtilityVM) { tb.Helper() err := vm.Start(ctx) @@ -36,7 +50,6 @@ func Kill(ctx context.Context, tb testing.TB, vm *uvm.UtilityVM) { func Close(ctx context.Context, tb testing.TB, vm *uvm.UtilityVM) { tb.Helper() - // Terminate will error on context cancellation, but close does not accept contexts if err := vm.CloseCtx(ctx); err != nil { tb.Fatalf("could not close uvm %q: %s", vm.ID(), err) } diff --git a/test/pkg/uvm/wcow.go b/test/pkg/uvm/wcow.go index a158d573d0..02f9cabe09 100644 --- a/test/pkg/uvm/wcow.go +++ b/test/pkg/uvm/wcow.go @@ -6,23 +6,34 @@ import ( "context" "testing" + hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" "github.com/Microsoft/hcsshim/internal/uvm" "github.com/Microsoft/hcsshim/test/internal/layers" "github.com/Microsoft/hcsshim/test/pkg/images" ) -// TODO: add cleanup return to CreateWCOWUVM*(), and have it remove layer and scratch dirs +// DefaultWCOWOptions returns default options for a bootable WCOW uVM. +// +// See [uvm.NewDefaultOptionsWCOW] for more information. +func DefaultWCOWOptions(_ context.Context, tb testing.TB, id, owner string) *uvm.OptionsWCOW { + // mostly here to match [DefaultLCOWOptions] + tb.Helper() + opts := uvm.NewDefaultOptionsWCOW(id, owner) + return opts +} // CreateWCOWUVM creates a WCOW utility VM with all default options. Returns the // UtilityVM object; folder used as its scratch. +// +// Deprecated: use [CreateWCOW] and [layers.WCOWScratchDir]. func CreateWCOWUVM(ctx context.Context, tb testing.TB, id, image string) (*uvm.UtilityVM, []string, string) { tb.Helper() return CreateWCOWUVMFromOptsWithImage(ctx, tb, uvm.NewDefaultOptionsWCOW(id, ""), image) } -// CreateWCOWUVMFromOpts creates a WCOW utility VM with the passed opts. -func CreateWCOWUVMFromOpts(ctx context.Context, tb testing.TB, opts *uvm.OptionsWCOW) *uvm.UtilityVM { +// CreateWCOW creates a WCOW utility VM with the passed opts. +func CreateWCOW(ctx context.Context, tb testing.TB, opts *uvm.OptionsWCOW) (*uvm.UtilityVM, CleanupFn) { tb.Helper() if opts == nil || len(opts.LayerFolders) < 2 { @@ -31,20 +42,18 @@ func CreateWCOWUVMFromOpts(ctx context.Context, tb testing.TB, opts *uvm.Options vm, err := uvm.CreateWCOW(ctx, opts) if err != nil { - tb.Fatal(err) - } - if err := vm.Start(ctx); err != nil { - _ = vm.CloseCtx(ctx) - tb.Fatal(err) + tb.Fatalf("could not create WCOW UVM: %v", err) } - return vm + return vm, newCleanupFn(ctx, tb, vm) } // CreateWCOWUVMFromOptsWithImage creates a WCOW utility VM with the passed opts // builds the LayerFolders based on `image`. Returns the UtilityVM object; // folder used as its scratch. // +// Deprecated: use [CreateWCOW] and [layers.WCOWScratchDir]. +// //nolint:staticcheck // SA5011: staticcheck thinks `opts` may be nil, even though we fail if it is func CreateWCOWUVMFromOptsWithImage( ctx context.Context, @@ -69,5 +78,25 @@ func CreateWCOWUVMFromOptsWithImage( opts.LayerFolders = append(opts.LayerFolders, uvmLayers...) opts.LayerFolders = append(opts.LayerFolders, scratchDir) - return CreateWCOWUVMFromOpts(ctx, tb, opts), uvmLayers, scratchDir + vm, cleanup := CreateWCOW(ctx, tb, opts) + tb.Cleanup(func() { cleanup(ctx) }) + + return vm, uvmLayers, scratchDir +} + +func AddVSMB(ctx context.Context, tb testing.TB, vm *uvm.UtilityVM, path string, options *hcsschema.VirtualSmbShareOptions) *uvm.VSMBShare { + tb.Helper() + + s, err := vm.AddVSMB(ctx, path, options) + if err != nil { + tb.Fatalf("failed to add vSMB share: %v", err) + } + + tb.Cleanup(func() { + if err := s.Release(ctx); err != nil { + tb.Fatalf("failed to remove vSMB share: %v", err) + } + }) + + return s }