Skip to content

Commit

Permalink
Parametrize e2e command
Browse files Browse the repository at this point in the history 
This allows for reusing e2e server and runner in other commands.
  • Loading branch information
@zimnx
zimnx committed May 15, 2024
1 parent 06038de commit ebd5630
Show file tree
Hide file tree
Showing 3 changed files with 109 additions and 81 deletions.
50 changes: 46 additions & 4 deletions pkg/cmd/tests/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,12 @@ package tests
import (
"fmt"
"os"
"path"
"strings"

"github.com/onsi/ginkgo/v2"
scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
"github.com/scylladb/scylla-operator/pkg/genericclioptions"
"github.com/scylladb/scylla-operator/pkg/helpers/slices"
"github.com/scylladb/scylla-operator/test/e2e/framework"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -35,6 +38,8 @@ var supportedBroadcastAddressTypes = []scyllav1.BroadcastAddressType{
}

type TestFrameworkOptions struct {
genericclioptions.ClientConfig

ArtifactsDir string
DeleteTestingNSPolicyUntyped string
DeleteTestingNSPolicy framework.DeleteTestingNSPolicyType
Expand All @@ -47,8 +52,9 @@ type TestFrameworkOptions struct {
gcsServiceAccountKey []byte
}

func NewTestFrameworkOptions() TestFrameworkOptions {
return TestFrameworkOptions{
func NewTestFrameworkOptions(streams genericclioptions.IOStreams, userAgent string) *TestFrameworkOptions {
return &TestFrameworkOptions{
ClientConfig: genericclioptions.NewClientConfig(userAgent),
ArtifactsDir: "",
DeleteTestingNSPolicyUntyped: string(framework.DeleteTestingNSPolicyAlways),
IngressController: &IngressControllerOptions{},
Expand All @@ -65,6 +71,8 @@ func NewTestFrameworkOptions() TestFrameworkOptions {
}

func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) {
o.ClientConfig.AddFlags(cmd)

cmd.PersistentFlags().StringVarP(&o.ArtifactsDir, "artifacts-dir", "", o.ArtifactsDir, "A directory for storing test artifacts. No data is collected until set.")
cmd.PersistentFlags().StringVarP(&o.DeleteTestingNSPolicyUntyped, "delete-namespace-policy", "", o.DeleteTestingNSPolicyUntyped, fmt.Sprintf("Namespace deletion policy. Allowed values are [%s].", strings.Join(
[]string{
Expand Down Expand Up @@ -93,9 +101,14 @@ func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) {
cmd.PersistentFlags().StringVarP(&o.GCSServiceAccountKeyPath, "gcs-service-account-key-path", "", o.GCSServiceAccountKeyPath, "Path to a file containing a GCS service account key.")
}

func (o *TestFrameworkOptions) Validate() error {
func (o *TestFrameworkOptions) Validate(args []string) error {
var errors []error

err := o.ClientConfig.Validate()
if err != nil {
errors = append(errors, err)
}

switch p := framework.DeleteTestingNSPolicyType(o.DeleteTestingNSPolicyUntyped); p {
case framework.DeleteTestingNSPolicyAlways,
framework.DeleteTestingNSPolicyOnSuccess,
Expand Down Expand Up @@ -127,7 +140,12 @@ func (o *TestFrameworkOptions) Validate() error {
return apierrors.NewAggregate(errors)
}

func (o *TestFrameworkOptions) Complete() error {
func (o *TestFrameworkOptions) Complete(args []string) error {
err := o.ClientConfig.Complete()
if err != nil {
return err
}

o.DeleteTestingNSPolicy = framework.DeleteTestingNSPolicyType(o.DeleteTestingNSPolicyUntyped)

// Trim spaces so we can reason later if the dir is set or not
Expand All @@ -153,5 +171,29 @@ func (o *TestFrameworkOptions) Complete() error {
o.gcsServiceAccountKey = gcsServiceAccountKey
}

framework.TestContext = &framework.TestContextType{
RestConfig: o.RestConfig,
ArtifactsDir: o.ArtifactsDir,
DeleteTestingNSPolicy: o.DeleteTestingNSPolicy,
ScyllaClusterOptions: o.scyllaClusterOptions,
ObjectStorageType: o.objectStorageType,
ObjectStorageBucket: o.ObjectStorageBucket,
GCSServiceAccountKey: o.gcsServiceAccountKey,
}

if o.IngressController != nil {
framework.TestContext.IngressController = &framework.IngressController{
Address: o.IngressController.Address,
IngressClassName: o.IngressController.IngressClassName,
CustomAnnotations: o.IngressController.CustomAnnotations,
}
}

if len(o.ArtifactsDir) != 0 {
_, reporterConfig := ginkgo.GinkgoConfiguration()
reporterConfig.JUnitReport = path.Join(o.ArtifactsDir, "e2e.junit.xml")
reporterConfig.JSONReport = path.Join(o.ArtifactsDir, "e2e.json")
}

return nil
}
34 changes: 33 additions & 1 deletion pkg/cmd/tests/tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,17 +5,48 @@ import (

"github.com/scylladb/scylla-operator/pkg/cmdutil"
"github.com/scylladb/scylla-operator/pkg/genericclioptions"
ginkgotest "github.com/scylladb/scylla-operator/pkg/test/ginkgo"
"github.com/scylladb/scylla-operator/test/e2e/framework"
"github.com/spf13/cobra"
"go.uber.org/automaxprocs/maxprocs"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/klog/v2"
"k8s.io/kubectl/pkg/util/templates"

// Include suites
_ "github.com/scylladb/scylla-operator/test/e2e"
)

const (
EnvVarPrefix = "SCYLLA_OPERATOR_TESTS_"
)

var Suites = ginkgotest.TestSuites{
{
Name: "all",
Description: templates.LongDesc(`
Runs all tests.`,
),
DefaultParallelism: 42,
},
{
Name: "scylla-operator/conformance/parallel",
Description: templates.LongDesc(`
Tests that ensure an Scylla Operator is working properly.
`),
LabelFilter: fmt.Sprintf("!%s", framework.SerialLabelName),
DefaultParallelism: 42,
},
{
Name: "scylla-operator/conformance/serial",
Description: templates.LongDesc(`
Tests that ensure an Scylla Operator is working properly.
`),
LabelFilter: fmt.Sprintf("%s", framework.SerialLabelName),
DefaultParallelism: 1,
},
}

func NewTestsCommand(streams genericclioptions.IOStreams) *cobra.Command {
cmd := &cobra.Command{
Use: "scylla-operator-tests",
Expand Down Expand Up @@ -45,7 +76,8 @@ func NewTestsCommand(streams genericclioptions.IOStreams) *cobra.Command {
SilenceErrors: true,
}

cmd.AddCommand(NewRunCommand(streams))
userAgent := "scylla-operator-e2e"
cmd.AddCommand(NewRunCommand(streams, Suites, userAgent))

// TODO: wrap help func for the root command and every subcommand to add a line about automatic env vars and the prefix.

Expand Down
106 changes: 30 additions & 76 deletions pkg/cmd/tests/tests_run.go
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// Copyright (c) 2024 ScyllaDB.

package tests

import (
Expand All @@ -7,7 +9,6 @@ import (
"fmt"
"os"
"os/exec"
"path"
"strings"
"sync"
"time"
Expand All @@ -25,15 +26,11 @@ import (
ginkgotest "github.com/scylladb/scylla-operator/pkg/test/ginkgo"
"github.com/scylladb/scylla-operator/pkg/thirdparty/github.com/onsi/ginkgo/v2/exposedinternal/parallel_support"
"github.com/scylladb/scylla-operator/pkg/version"
"github.com/scylladb/scylla-operator/test/e2e/framework"
"github.com/spf13/cobra"
apierrors "k8s.io/apimachinery/pkg/util/errors"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
"k8s.io/kubectl/pkg/util/templates"

// Include suites
_ "github.com/scylladb/scylla-operator/test/e2e"
)

const (
Expand All @@ -43,35 +40,8 @@ const (
ginkgoOutputInterceptorModeNone = "none"
)

var suites = ginkgotest.TestSuites{
{
Name: "all",
Description: templates.LongDesc(`
Runs all tests.
`),
DefaultParallelism: 42,
},
{
Name: "scylla-operator/conformance/parallel",
Description: templates.LongDesc(`
Tests that ensure an Scylla Operator is working properly.
`),
LabelFilter: fmt.Sprintf("!%s", framework.SerialLabelName),
DefaultParallelism: 42,
},
{
Name: "scylla-operator/conformance/serial",
Description: templates.LongDesc(`
Tests that ensure an Scylla Operator is working properly.
`),
LabelFilter: fmt.Sprintf("%s", framework.SerialLabelName),
DefaultParallelism: 1,
},
}

type RunOptions struct {
genericclioptions.IOStreams
genericclioptions.ClientConfig
TestFrameworkOptions

Timeout time.Duration
Expand All @@ -90,14 +60,13 @@ type RunOptions struct {
ParallelServerAddress string
ParallelLogLevel int32

TestSuites ginkgotest.TestSuites
SelectedSuite *ginkgotest.TestSuite
}

func NewRunOptions(streams genericclioptions.IOStreams) *RunOptions {
return &RunOptions{
ClientConfig: genericclioptions.NewClientConfig("scylla-operator-e2e"),
TestFrameworkOptions: NewTestFrameworkOptions(),

func NewRunOptions(streams genericclioptions.IOStreams, testSuites ginkgotest.TestSuites, userAgent string) RunOptions {
return RunOptions{
TestFrameworkOptions: *NewTestFrameworkOptions(streams, userAgent),
Timeout: 24 * time.Hour,
Quiet: false,
ShowProgress: true,
Expand All @@ -113,18 +82,24 @@ func NewRunOptions(streams genericclioptions.IOStreams) *RunOptions {
ParallelShard: 0,
ParallelServerAddress: "",
ParallelLogLevel: 0,

TestSuites: testSuites,
}
}

func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
o := NewRunOptions(streams)
func NewRunCommand(streams genericclioptions.IOStreams, testSuites ginkgotest.TestSuites, userAgent string) *cobra.Command {
o := NewRunOptions(streams, testSuites, userAgent)

testSuiteNames := slices.ConvertSlice(testSuites, func(ts *ginkgotest.TestSuite) string {
return ts.Name
})

cmd := &cobra.Command{
Use: "run SUITE_NAME",
Long: templates.LongDesc(`
Runs a test suite
`),
ValidArgs: suites.Names(),
ValidArgs: testSuiteNames,
RunE: func(cmd *cobra.Command, args []string) error {
err := o.Validate(args)
if err != nil {
Expand All @@ -148,9 +123,13 @@ func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
SilenceUsage: true,
}

o.ClientConfig.AddFlags(cmd)
o.TestFrameworkOptions.AddFlags(cmd)
o.AddFlags(cmd)

return cmd
}

func (o *RunOptions) AddFlags(cmd *cobra.Command) {
cmd.Flags().DurationVarP(&o.Timeout, "timeout", "", o.Timeout, "If the overall suite(s) duration exceed this value, tests will be terminated.")
cmd.Flags().BoolVarP(&o.Quiet, "quiet", "", o.Quiet, "Reduces the tests output.")
cmd.Flags().BoolVarP(&o.ShowProgress, "progress", "", o.ShowProgress, "Shows progress during test run. Only applies to serial execution.")
Expand All @@ -168,15 +147,15 @@ func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
cmd.Flags().MarkHidden(parallelShardFlagKey)
cmd.Flags().StringVarP(&o.ParallelServerAddress, parallelServerAddressFlagKey, "", o.ParallelServerAddress, "")
cmd.Flags().MarkHidden(parallelServerAddressFlagKey)

return cmd
}

func (o *RunOptions) Validate(args []string) error {
var errs []error

errs = append(errs, o.ClientConfig.Validate())
errs = append(errs, o.TestFrameworkOptions.Validate())
err := o.TestFrameworkOptions.Validate(args)
if err != nil {
errs = append(errs, err)
}

if o.FlakeAttempts < 0 {
errs = append(errs, fmt.Errorf("flake attempts can't be negative"))
Expand All @@ -202,12 +181,12 @@ func (o *RunOptions) Validate(args []string) error {
case 0:
errs = append(errs, fmt.Errorf(
"you have to specify at least one suite from [%s]",
strings.Join(suites.Names(), ", ")),
strings.Join(o.TestSuites.Names(), ", ")),
)

case 1:
suiteName := args[0]
o.SelectedSuite = suites.Find(suiteName)
o.SelectedSuite = o.TestSuites.Find(suiteName)
if o.SelectedSuite == nil {
errs = append(errs, fmt.Errorf("suite %q doesn't exist", suiteName))
}
Expand All @@ -220,17 +199,14 @@ func (o *RunOptions) Validate(args []string) error {
}

func (o *RunOptions) Complete(args []string) error {
err := o.ClientConfig.Complete()
if err != nil {
return err
}
var errs []error

err = o.TestFrameworkOptions.Complete()
err := o.TestFrameworkOptions.Complete(args)
if err != nil {
return err
errs = append(errs, err)
}

return nil
return apierrors.NewAggregate(errs)
}

func (o *RunOptions) Run(streams genericclioptions.IOStreams, cmd *cobra.Command) error {
Expand All @@ -257,23 +233,6 @@ var _ ginkgo.GinkgoTestingT = &fakeT{}
func (o *RunOptions) run(ctx context.Context, streams genericclioptions.IOStreams) error {
const suite = "Scylla operator E2E tests"

framework.TestContext = &framework.TestContextType{
RestConfig: o.RestConfig,
ArtifactsDir: o.ArtifactsDir,
DeleteTestingNSPolicy: o.DeleteTestingNSPolicy,
ScyllaClusterOptions: o.scyllaClusterOptions,
ObjectStorageType: o.objectStorageType,
ObjectStorageBucket: o.ObjectStorageBucket,
GCSServiceAccountKey: o.gcsServiceAccountKey,
}
if o.IngressController != nil {
framework.TestContext.IngressController = &framework.IngressController{
Address: o.IngressController.Address,
IngressClassName: o.IngressController.IngressClassName,
CustomAnnotations: o.IngressController.CustomAnnotations,
}
}

suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration()

suiteConfig.Timeout = o.Timeout
Expand Down Expand Up @@ -319,11 +278,6 @@ func (o *RunOptions) run(ctx context.Context, streams genericclioptions.IOStream

gomega.RegisterFailHandler(ginkgo.Fail)

if len(o.ArtifactsDir) > 0 {
reporterConfig.JUnitReport = path.Join(o.ArtifactsDir, "e2e.junit.xml")
reporterConfig.JSONReport = path.Join(o.ArtifactsDir, "e2e.json")
}

suiteConfig.ParallelTotal = o.Parallelism
if suiteConfig.ParallelTotal == 0 {
if o.DryRun {
Expand Down

0 comments on commit ebd5630

Please sign in to comment.