Parametrize e2e command

pkg/cmd/tests/options.go

@@ -3,9 +3,12 @@ package tests
 import (
 	"fmt"
 	"os"
+	"path"
 	"strings"
 
+	"github.com/onsi/ginkgo/v2"
 	scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
+	"github.com/scylladb/scylla-operator/pkg/genericclioptions"
 	"github.com/scylladb/scylla-operator/pkg/helpers/slices"
 	"github.com/scylladb/scylla-operator/test/e2e/framework"
 	"github.com/spf13/cobra"
@@ -35,6 +38,8 @@ var supportedBroadcastAddressTypes = []scyllav1.BroadcastAddressType{
 }
 
 type TestFrameworkOptions struct {
+	genericclioptions.ClientConfig
+
 	ArtifactsDir                 string
 	DeleteTestingNSPolicyUntyped string
 	DeleteTestingNSPolicy        framework.DeleteTestingNSPolicyType
@@ -47,8 +52,9 @@ type TestFrameworkOptions struct {
 	gcsServiceAccountKey         []byte
 }
 
-func NewTestFrameworkOptions() TestFrameworkOptions {
-	return TestFrameworkOptions{
+func NewTestFrameworkOptions(streams genericclioptions.IOStreams, userAgent string) *TestFrameworkOptions {
+	return &TestFrameworkOptions{
+		ClientConfig:                 genericclioptions.NewClientConfig(userAgent),
 		ArtifactsDir:                 "",
 		DeleteTestingNSPolicyUntyped: string(framework.DeleteTestingNSPolicyAlways),
 		IngressController:            &IngressControllerOptions{},
@@ -65,6 +71,8 @@ func NewTestFrameworkOptions() TestFrameworkOptions {
 }
 
 func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) {
+	o.ClientConfig.AddFlags(cmd)
+
 	cmd.PersistentFlags().StringVarP(&o.ArtifactsDir, "artifacts-dir", "", o.ArtifactsDir, "A directory for storing test artifacts. No data is collected until set.")
 	cmd.PersistentFlags().StringVarP(&o.DeleteTestingNSPolicyUntyped, "delete-namespace-policy", "", o.DeleteTestingNSPolicyUntyped, fmt.Sprintf("Namespace deletion policy. Allowed values are [%s].", strings.Join(
 		[]string{
@@ -93,9 +101,14 @@ func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) {
 	cmd.PersistentFlags().StringVarP(&o.GCSServiceAccountKeyPath, "gcs-service-account-key-path", "", o.GCSServiceAccountKeyPath, "Path to a file containing a GCS service account key.")
 }
 
-func (o *TestFrameworkOptions) Validate() error {
+func (o *TestFrameworkOptions) Validate(args []string) error {
 	var errors []error
 
+	err := o.ClientConfig.Validate()
+	if err != nil {
+		errors = append(errors, err)
+	}
+
 	switch p := framework.DeleteTestingNSPolicyType(o.DeleteTestingNSPolicyUntyped); p {
 	case framework.DeleteTestingNSPolicyAlways,
 		framework.DeleteTestingNSPolicyOnSuccess,
@@ -127,7 +140,12 @@ func (o *TestFrameworkOptions) Validate() error {
 	return apierrors.NewAggregate(errors)
 }
 
-func (o *TestFrameworkOptions) Complete() error {
+func (o *TestFrameworkOptions) Complete(args []string) error {
+	err := o.ClientConfig.Complete()
+	if err != nil {
+		return err
+	}
+
 	o.DeleteTestingNSPolicy = framework.DeleteTestingNSPolicyType(o.DeleteTestingNSPolicyUntyped)
 
 	// Trim spaces so we can reason later if the dir is set or not
@@ -153,5 +171,29 @@ func (o *TestFrameworkOptions) Complete() error {
 		o.gcsServiceAccountKey = gcsServiceAccountKey
 	}
 
+	framework.TestContext = &framework.TestContextType{
+		RestConfig:            o.RestConfig,
+		ArtifactsDir:          o.ArtifactsDir,
+		DeleteTestingNSPolicy: o.DeleteTestingNSPolicy,
+		ScyllaClusterOptions:  o.scyllaClusterOptions,
+		ObjectStorageType:     o.objectStorageType,
+		ObjectStorageBucket:   o.ObjectStorageBucket,
+		GCSServiceAccountKey:  o.gcsServiceAccountKey,
+	}
+
+	if o.IngressController != nil {
+		framework.TestContext.IngressController = &framework.IngressController{
+			Address:           o.IngressController.Address,
+			IngressClassName:  o.IngressController.IngressClassName,
+			CustomAnnotations: o.IngressController.CustomAnnotations,
+		}
+	}
+
+	if len(o.ArtifactsDir) != 0 {
+		_, reporterConfig := ginkgo.GinkgoConfiguration()
+		reporterConfig.JUnitReport = path.Join(o.ArtifactsDir, "e2e.junit.xml")
+		reporterConfig.JSONReport = path.Join(o.ArtifactsDir, "e2e.json")
+	}
+
 	return nil
 }

pkg/cmd/tests/tests.go

@@ -5,17 +5,48 @@ import (
 
 	"github.com/scylladb/scylla-operator/pkg/cmdutil"
 	"github.com/scylladb/scylla-operator/pkg/genericclioptions"
+	ginkgotest "github.com/scylladb/scylla-operator/pkg/test/ginkgo"
+	"github.com/scylladb/scylla-operator/test/e2e/framework"
 	"github.com/spf13/cobra"
 	"go.uber.org/automaxprocs/maxprocs"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/klog/v2"
 	"k8s.io/kubectl/pkg/util/templates"
+
+	// Include suites
+	_ "github.com/scylladb/scylla-operator/test/e2e"
 )
 
 const (
 	EnvVarPrefix = "SCYLLA_OPERATOR_TESTS_"
 )
 
+var Suites = ginkgotest.TestSuites{
+	{
+		Name: "all",
+		Description: templates.LongDesc(`
+		Runs all tests.`,
+		),
+		DefaultParallelism: 42,
+	},
+	{
+		Name: "scylla-operator/conformance/parallel",
+		Description: templates.LongDesc(`
+		Tests that ensure an Scylla Operator is working properly.
+		`),
+		LabelFilter:        fmt.Sprintf("!%s", framework.SerialLabelName),
+		DefaultParallelism: 42,
+	},
+	{
+		Name: "scylla-operator/conformance/serial",
+		Description: templates.LongDesc(`
+		Tests that ensure an Scylla Operator is working properly.
+		`),
+		LabelFilter:        fmt.Sprintf("%s", framework.SerialLabelName),
+		DefaultParallelism: 1,
+	},
+}
+
 func NewTestsCommand(streams genericclioptions.IOStreams) *cobra.Command {
 	cmd := &cobra.Command{
 		Use: "scylla-operator-tests",
@@ -45,7 +76,8 @@ func NewTestsCommand(streams genericclioptions.IOStreams) *cobra.Command {
 		SilenceErrors: true,
 	}
 
-	cmd.AddCommand(NewRunCommand(streams))
+	userAgent := "scylla-operator-e2e"
+	cmd.AddCommand(NewRunCommand(streams, Suites, userAgent))
 
 	// TODO: wrap help func for the root command and every subcommand to add a line about automatic env vars and the prefix.
 

pkg/cmd/tests/tests_run.go

@@ -1,3 +1,5 @@
+// Copyright (c) 2024 ScyllaDB.
+
 package tests
 
 import (
@@ -7,7 +9,6 @@ import (
 	"fmt"
 	"os"
 	"os/exec"
-	"path"
 	"strings"
 	"sync"
 	"time"
@@ -25,15 +26,11 @@ import (
 	ginkgotest "github.com/scylladb/scylla-operator/pkg/test/ginkgo"
 	"github.com/scylladb/scylla-operator/pkg/thirdparty/github.com/onsi/ginkgo/v2/exposedinternal/parallel_support"
 	"github.com/scylladb/scylla-operator/pkg/version"
-	"github.com/scylladb/scylla-operator/test/e2e/framework"
 	"github.com/spf13/cobra"
 	apierrors "k8s.io/apimachinery/pkg/util/errors"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/klog/v2"
 	"k8s.io/kubectl/pkg/util/templates"
-
-	// Include suites
-	_ "github.com/scylladb/scylla-operator/test/e2e"
 )
 
 const (
@@ -43,35 +40,8 @@ const (
 	ginkgoOutputInterceptorModeNone = "none"
 )
 
-var suites = ginkgotest.TestSuites{
-	{
-		Name: "all",
-		Description: templates.LongDesc(`
-		Runs all tests.
-		`),
-		DefaultParallelism: 42,
-	},
-	{
-		Name: "scylla-operator/conformance/parallel",
-		Description: templates.LongDesc(`
-		Tests that ensure an Scylla Operator is working properly.
-		`),
-		LabelFilter:        fmt.Sprintf("!%s", framework.SerialLabelName),
-		DefaultParallelism: 42,
-	},
-	{
-		Name: "scylla-operator/conformance/serial",
-		Description: templates.LongDesc(`
-		Tests that ensure an Scylla Operator is working properly.
-		`),
-		LabelFilter:        fmt.Sprintf("%s", framework.SerialLabelName),
-		DefaultParallelism: 1,
-	},
-}
-
 type RunOptions struct {
 	genericclioptions.IOStreams
-	genericclioptions.ClientConfig
 	TestFrameworkOptions
 
 	Timeout               time.Duration
@@ -90,14 +60,13 @@ type RunOptions struct {
 	ParallelServerAddress string
 	ParallelLogLevel      int32
 
+	TestSuites    ginkgotest.TestSuites
 	SelectedSuite *ginkgotest.TestSuite
 }
 
-func NewRunOptions(streams genericclioptions.IOStreams) *RunOptions {
-	return &RunOptions{
-		ClientConfig:         genericclioptions.NewClientConfig("scylla-operator-e2e"),
-		TestFrameworkOptions: NewTestFrameworkOptions(),
-
+func NewRunOptions(streams genericclioptions.IOStreams, testSuites ginkgotest.TestSuites, userAgent string) RunOptions {
+	return RunOptions{
+		TestFrameworkOptions:  *NewTestFrameworkOptions(streams, userAgent),
 		Timeout:               24 * time.Hour,
 		Quiet:                 false,
 		ShowProgress:          true,
@@ -113,18 +82,24 @@ func NewRunOptions(streams genericclioptions.IOStreams) *RunOptions {
 		ParallelShard:         0,
 		ParallelServerAddress: "",
 		ParallelLogLevel:      0,
+
+		TestSuites: testSuites,
 	}
 }
 
-func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
-	o := NewRunOptions(streams)
+func NewRunCommand(streams genericclioptions.IOStreams, testSuites ginkgotest.TestSuites, userAgent string) *cobra.Command {
+	o := NewRunOptions(streams, testSuites, userAgent)
+
+	testSuiteNames := slices.ConvertSlice(testSuites, func(ts *ginkgotest.TestSuite) string {
+		return ts.Name
+	})
 
 	cmd := &cobra.Command{
 		Use: "run SUITE_NAME",
 		Long: templates.LongDesc(`
 		Runs a test suite
 		`),
-		ValidArgs: suites.Names(),
+		ValidArgs: testSuiteNames,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			err := o.Validate(args)
 			if err != nil {
@@ -148,9 +123,13 @@ func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
 		SilenceUsage:  true,
 	}
 
-	o.ClientConfig.AddFlags(cmd)
 	o.TestFrameworkOptions.AddFlags(cmd)
+	o.AddFlags(cmd)
 
+	return cmd
+}
+
+func (o *RunOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().DurationVarP(&o.Timeout, "timeout", "", o.Timeout, "If the overall suite(s) duration exceed this value, tests will be terminated.")
 	cmd.Flags().BoolVarP(&o.Quiet, "quiet", "", o.Quiet, "Reduces the tests output.")
 	cmd.Flags().BoolVarP(&o.ShowProgress, "progress", "", o.ShowProgress, "Shows progress during test run. Only applies to serial execution.")
@@ -168,15 +147,15 @@ func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
 	cmd.Flags().MarkHidden(parallelShardFlagKey)
 	cmd.Flags().StringVarP(&o.ParallelServerAddress, parallelServerAddressFlagKey, "", o.ParallelServerAddress, "")
 	cmd.Flags().MarkHidden(parallelServerAddressFlagKey)
-
-	return cmd
 }
 
 func (o *RunOptions) Validate(args []string) error {
 	var errs []error
 
-	errs = append(errs, o.ClientConfig.Validate())
-	errs = append(errs, o.TestFrameworkOptions.Validate())
+	err := o.TestFrameworkOptions.Validate(args)
+	if err != nil {
+		errs = append(errs, err)
+	}
 
 	if o.FlakeAttempts < 0 {
 		errs = append(errs, fmt.Errorf("flake attempts can't be negative"))
@@ -202,12 +181,12 @@ func (o *RunOptions) Validate(args []string) error {
 	case 0:
 		errs = append(errs, fmt.Errorf(
 			"you have to specify at least one suite from [%s]",
-			strings.Join(suites.Names(), ", ")),
+			strings.Join(o.TestSuites.Names(), ", ")),
 		)
 
 	case 1:
 		suiteName := args[0]
-		o.SelectedSuite = suites.Find(suiteName)
+		o.SelectedSuite = o.TestSuites.Find(suiteName)
 		if o.SelectedSuite == nil {
 			errs = append(errs, fmt.Errorf("suite %q doesn't exist", suiteName))
 		}
@@ -220,17 +199,14 @@ func (o *RunOptions) Validate(args []string) error {
 }
 
 func (o *RunOptions) Complete(args []string) error {
-	err := o.ClientConfig.Complete()
-	if err != nil {
-		return err
-	}
+	var errs []error
 
-	err = o.TestFrameworkOptions.Complete()
+	err := o.TestFrameworkOptions.Complete(args)
 	if err != nil {
-		return err
+		errs = append(errs, err)
 	}
 
-	return nil
+	return apierrors.NewAggregate(errs)
 }
 
 func (o *RunOptions) Run(streams genericclioptions.IOStreams, cmd *cobra.Command) error {
@@ -257,23 +233,6 @@ var _ ginkgo.GinkgoTestingT = &fakeT{}
 func (o *RunOptions) run(ctx context.Context, streams genericclioptions.IOStreams) error {
 	const suite = "Scylla operator E2E tests"
 
-	framework.TestContext = &framework.TestContextType{
-		RestConfig:            o.RestConfig,
-		ArtifactsDir:          o.ArtifactsDir,
-		DeleteTestingNSPolicy: o.DeleteTestingNSPolicy,
-		ScyllaClusterOptions:  o.scyllaClusterOptions,
-		ObjectStorageType:     o.objectStorageType,
-		ObjectStorageBucket:   o.ObjectStorageBucket,
-		GCSServiceAccountKey:  o.gcsServiceAccountKey,
-	}
-	if o.IngressController != nil {
-		framework.TestContext.IngressController = &framework.IngressController{
-			Address:           o.IngressController.Address,
-			IngressClassName:  o.IngressController.IngressClassName,
-			CustomAnnotations: o.IngressController.CustomAnnotations,
-		}
-	}
-
 	suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration()
 
 	suiteConfig.Timeout = o.Timeout
@@ -319,11 +278,6 @@ func (o *RunOptions) run(ctx context.Context, streams genericclioptions.IOStream
 
 	gomega.RegisterFailHandler(ginkgo.Fail)
 
-	if len(o.ArtifactsDir) > 0 {
-		reporterConfig.JUnitReport = path.Join(o.ArtifactsDir, "e2e.junit.xml")
-		reporterConfig.JSONReport = path.Join(o.ArtifactsDir, "e2e.json")
-	}
-
 	suiteConfig.ParallelTotal = o.Parallelism
 	if suiteConfig.ParallelTotal == 0 {
 		if o.DryRun {