Skip to content

Commit

Permalink
Parametrize e2e command
Browse files Browse the repository at this point in the history 
This allows for reusing e2e server and runner in other commands.
  • Loading branch information
@zimnx
zimnx committed Mar 27, 2024
1 parent b749b9b commit 714a412
Show file tree
Hide file tree
Showing 3 changed files with 115 additions and 75 deletions.
42 changes: 40 additions & 2 deletions pkg/cmd/tests/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,12 @@ package tests

import (
"fmt"
"path"
"strings"

"github.com/onsi/ginkgo/v2"
scyllav1 "github.com/scylladb/scylla-operator/pkg/api/scylla/v1"
"github.com/scylladb/scylla-operator/pkg/genericclioptions"
"github.com/scylladb/scylla-operator/pkg/helpers/slices"
"github.com/scylladb/scylla-operator/test/e2e/framework"
"github.com/spf13/cobra"
Expand Down Expand Up @@ -34,6 +37,7 @@ var supportedBroadcastAddressTypes = []scyllav1.BroadcastAddressType{
}

type TestFrameworkOptions struct {
genericclioptions.ClientConfig
ArtifactsDir string
DeleteTestingNSPolicyUntyped string
DeleteTestingNSPolicy framework.DeleteTestingNSPolicyType
Expand All @@ -42,8 +46,9 @@ type TestFrameworkOptions struct {
scyllaClusterOptions *framework.ScyllaClusterOptions
}

func NewTestFrameworkOptions() TestFrameworkOptions {
return TestFrameworkOptions{
func NewTestFrameworkOptions(userAgent string) *TestFrameworkOptions {
return &TestFrameworkOptions{
ClientConfig: genericclioptions.NewClientConfig(userAgent),
ArtifactsDir: "",
DeleteTestingNSPolicyUntyped: string(framework.DeleteTestingNSPolicyAlways),
IngressController: &IngressControllerOptions{},
Expand All @@ -56,6 +61,8 @@ func NewTestFrameworkOptions() TestFrameworkOptions {
}

func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) {
o.ClientConfig.AddFlags(cmd)

cmd.PersistentFlags().StringVarP(&o.ArtifactsDir, "artifacts-dir", "", o.ArtifactsDir, "A directory for storing test artifacts. No data is collected until set.")
cmd.PersistentFlags().StringVarP(&o.DeleteTestingNSPolicyUntyped, "delete-namespace-policy", "", o.DeleteTestingNSPolicyUntyped, fmt.Sprintf("Namespace deletion policy. Allowed values are [%s].", strings.Join(
[]string{
Expand Down Expand Up @@ -85,6 +92,11 @@ func (o *TestFrameworkOptions) AddFlags(cmd *cobra.Command) {
func (o *TestFrameworkOptions) Validate() error {
var errors []error

err := o.ClientConfig.Validate()
if err != nil {
errors = append(errors, err)
}

switch p := framework.DeleteTestingNSPolicyType(o.DeleteTestingNSPolicyUntyped); p {
case framework.DeleteTestingNSPolicyAlways,
framework.DeleteTestingNSPolicyOnSuccess,
Expand All @@ -109,6 +121,11 @@ func (o *TestFrameworkOptions) Validate() error {
}

func (o *TestFrameworkOptions) Complete() error {
err := o.ClientConfig.Complete()
if err != nil {
return err
}

o.DeleteTestingNSPolicy = framework.DeleteTestingNSPolicyType(o.DeleteTestingNSPolicyUntyped)

// Trim spaces so we can reason later if the dir is set or not
Expand All @@ -122,5 +139,26 @@ func (o *TestFrameworkOptions) Complete() error {
},
}

framework.TestContext = &framework.TestContextType{
RestConfig: o.RestConfig,
ArtifactsDir: o.ArtifactsDir,
DeleteTestingNSPolicy: o.DeleteTestingNSPolicy,
ScyllaClusterOptions: o.scyllaClusterOptions,
}

if o.IngressController != nil {
framework.TestContext.IngressController = &framework.IngressController{
Address: o.IngressController.Address,
IngressClassName: o.IngressController.IngressClassName,
CustomAnnotations: o.IngressController.CustomAnnotations,
}
}

if len(o.ArtifactsDir) != 0 {
_, reporterConfig := ginkgo.GinkgoConfiguration()
reporterConfig.JUnitReport = path.Join(o.ArtifactsDir, "e2e.junit.xml")
reporterConfig.JSONReport = path.Join(o.ArtifactsDir, "e2e.json")
}

return nil
}
29 changes: 28 additions & 1 deletion pkg/cmd/tests/tests.go
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,43 @@
package tests

import (
"fmt"

"github.com/scylladb/scylla-operator/pkg/cmdutil"
"github.com/scylladb/scylla-operator/pkg/genericclioptions"
ginkgotest "github.com/scylladb/scylla-operator/pkg/test/ginkgo"
"github.com/scylladb/scylla-operator/test/e2e/framework"
"github.com/spf13/cobra"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/kubectl/pkg/util/templates"

// Include suites
_ "github.com/scylladb/scylla-operator/test/e2e"
)

const (
EnvVarPrefix = "SCYLLA_OPERATOR_TESTS_"
)

var Suites = ginkgotest.TestSuites{
{
Name: "scylla-operator/conformance/parallel",
Description: templates.LongDesc(`
Tests that ensure an Scylla Operator is working properly.
`),
LabelFilter: fmt.Sprintf("!%s", framework.SerialLabelName),
DefaultParallelism: 42,
},
{
Name: "scylla-operator/conformance/serial",
Description: templates.LongDesc(`
Tests that ensure an Scylla Operator is working properly.
`),
LabelFilter: fmt.Sprintf("%s", framework.SerialLabelName),
DefaultParallelism: 1,
},
}

func NewTestsCommand(streams genericclioptions.IOStreams) *cobra.Command {
cmd := &cobra.Command{
Use: "scylla-operator-tests",
Expand All @@ -29,7 +55,8 @@ func NewTestsCommand(streams genericclioptions.IOStreams) *cobra.Command {
SilenceErrors: true,
}

cmd.AddCommand(NewRunCommand(streams))
userAgent := "scylla-operator-e2e"
cmd.AddCommand(NewRunCommand(streams, Suites, NewTestFrameworkOptions(userAgent)))

// TODO: wrap help func for the root command and every subcommand to add a line about automatic env vars and the prefix.

Expand Down
119 changes: 47 additions & 72 deletions pkg/cmd/tests/tests_run.go
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// Copyright (c) 2024 ScyllaDB.

package tests

import (
Expand All @@ -7,7 +9,6 @@ import (
"fmt"
"os"
"os/exec"
"path"
"strings"
"sync"
"time"
Expand All @@ -25,15 +26,11 @@ import (
ginkgotest "github.com/scylladb/scylla-operator/pkg/test/ginkgo"
"github.com/scylladb/scylla-operator/pkg/thirdparty/github.com/onsi/ginkgo/v2/exposedinternal/parallel_support"
"github.com/scylladb/scylla-operator/pkg/version"
"github.com/scylladb/scylla-operator/test/e2e/framework"
"github.com/spf13/cobra"
apierrors "k8s.io/apimachinery/pkg/util/errors"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
"k8s.io/kubectl/pkg/util/templates"

// Include suites
_ "github.com/scylladb/scylla-operator/test/e2e"
)

const (
Expand All @@ -43,36 +40,9 @@ const (
ginkgoOutputInterceptorModeNone = "none"
)

var suites = ginkgotest.TestSuites{
{
Name: "all",
Description: templates.LongDesc(`
Runs all tests.
`),
DefaultParallelism: 42,
},
{
Name: "scylla-operator/conformance/parallel",
Description: templates.LongDesc(`
Tests that ensure an Scylla Operator is working properly.
`),
LabelFilter: fmt.Sprintf("!%s", framework.SerialLabelName),
DefaultParallelism: 42,
},
{
Name: "scylla-operator/conformance/serial",
Description: templates.LongDesc(`
Tests that ensure an Scylla Operator is working properly.
`),
LabelFilter: fmt.Sprintf("%s", framework.SerialLabelName),
DefaultParallelism: 1,
},
}

type RunOptions struct {
genericclioptions.IOStreams
genericclioptions.ClientConfig
TestFrameworkOptions
CustomOptions []CustomFrameworkOptions

Timeout time.Duration
Quiet bool
Expand All @@ -90,13 +60,19 @@ type RunOptions struct {
ParallelServerAddress string
ParallelLogLevel int32

testSuites ginkgotest.TestSuites
SelectedSuite *ginkgotest.TestSuite
}

func NewRunOptions(streams genericclioptions.IOStreams) *RunOptions {
type CustomFrameworkOptions interface {
Validate() error
Complete() error
AddFlags(command *cobra.Command)
}

func NewRunOptions(streams genericclioptions.IOStreams, testSuites ginkgotest.TestSuites, options ...CustomFrameworkOptions) *RunOptions {
return &RunOptions{
ClientConfig: genericclioptions.NewClientConfig("scylla-operator-e2e"),
TestFrameworkOptions: NewTestFrameworkOptions(),
CustomOptions: options,

Timeout: 24 * time.Hour,
Quiet: false,
Expand All @@ -113,18 +89,37 @@ func NewRunOptions(streams genericclioptions.IOStreams) *RunOptions {
ParallelShard: 0,
ParallelServerAddress: "",
ParallelLogLevel: 0,

testSuites: testSuites,
}
}

func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
o := NewRunOptions(streams)
func NewRunCommand(streams genericclioptions.IOStreams, customTestSuites ginkgotest.TestSuites, customOptions ...CustomFrameworkOptions) *cobra.Command {
testSuites := make(ginkgotest.TestSuites, 0, len(customTestSuites)+1)
for _, cts := range customTestSuites {
testSuites = append(testSuites, cts)
}

testSuites = append(testSuites, &ginkgotest.TestSuite{
Name: "all",
Description: templates.LongDesc(`
Runs all tests.`,
),
DefaultParallelism: 42,
})

o := NewRunOptions(streams, testSuites, customOptions...)

validTestSuiteNames := slices.ConvertSlice(testSuites, func(ts *ginkgotest.TestSuite) string {
return ts.Name
})

cmd := &cobra.Command{
Use: "run SUITE_NAME",
Long: templates.LongDesc(`
Runs a test suite
`),
ValidArgs: suites.Names(),
ValidArgs: validTestSuiteNames,
RunE: func(cmd *cobra.Command, args []string) error {
err := o.Validate(args)
if err != nil {
Expand All @@ -148,8 +143,9 @@ func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
SilenceUsage: true,
}

o.ClientConfig.AddFlags(cmd)
o.TestFrameworkOptions.AddFlags(cmd)
for _, co := range o.CustomOptions {
co.AddFlags(cmd)
}

cmd.Flags().DurationVarP(&o.Timeout, "timeout", "", o.Timeout, "If the overall suite(s) duration exceed this value, tests will be terminated.")
cmd.Flags().BoolVarP(&o.Quiet, "quiet", "", o.Quiet, "Reduces the tests output.")
Expand All @@ -175,8 +171,9 @@ func NewRunCommand(streams genericclioptions.IOStreams) *cobra.Command {
func (o *RunOptions) Validate(args []string) error {
var errs []error

errs = append(errs, o.ClientConfig.Validate())
errs = append(errs, o.TestFrameworkOptions.Validate())
for _, co := range o.CustomOptions {
errs = append(errs, co.Validate())
}

if o.FlakeAttempts < 0 {
errs = append(errs, fmt.Errorf("flake attempts can't be negative"))
Expand All @@ -202,12 +199,12 @@ func (o *RunOptions) Validate(args []string) error {
case 0:
errs = append(errs, fmt.Errorf(
"you have to specify at least one suite from [%s]",
strings.Join(suites.Names(), ", ")),
strings.Join(o.testSuites.Names(), ", ")),
)

case 1:
suiteName := args[0]
o.SelectedSuite = suites.Find(suiteName)
o.SelectedSuite = o.testSuites.Find(suiteName)
if o.SelectedSuite == nil {
errs = append(errs, fmt.Errorf("suite %q doesn't exist", suiteName))
}
Expand All @@ -220,14 +217,11 @@ func (o *RunOptions) Validate(args []string) error {
}

func (o *RunOptions) Complete(args []string) error {
err := o.ClientConfig.Complete()
if err != nil {
return err
}

err = o.TestFrameworkOptions.Complete()
if err != nil {
return err
for _, co := range o.CustomOptions {
err := co.Complete()
if err != nil {
return err
}
}

return nil
Expand Down Expand Up @@ -257,20 +251,6 @@ var _ ginkgo.GinkgoTestingT = &fakeT{}
func (o *RunOptions) run(ctx context.Context, streams genericclioptions.IOStreams) error {
const suite = "Scylla operator E2E tests"

framework.TestContext = &framework.TestContextType{
RestConfig: o.RestConfig,
ArtifactsDir: o.ArtifactsDir,
DeleteTestingNSPolicy: o.DeleteTestingNSPolicy,
ScyllaClusterOptions: o.scyllaClusterOptions,
}
if o.IngressController != nil {
framework.TestContext.IngressController = &framework.IngressController{
Address: o.IngressController.Address,
IngressClassName: o.IngressController.IngressClassName,
CustomAnnotations: o.IngressController.CustomAnnotations,
}
}

suiteConfig, reporterConfig := ginkgo.GinkgoConfiguration()

suiteConfig.Timeout = o.Timeout
Expand Down Expand Up @@ -316,11 +296,6 @@ func (o *RunOptions) run(ctx context.Context, streams genericclioptions.IOStream

gomega.RegisterFailHandler(ginkgo.Fail)

if len(o.ArtifactsDir) > 0 {
reporterConfig.JUnitReport = path.Join(o.ArtifactsDir, "e2e.junit.xml")
reporterConfig.JSONReport = path.Join(o.ArtifactsDir, "e2e.json")
}

suiteConfig.ParallelTotal = o.Parallelism
if suiteConfig.ParallelTotal == 0 {
if o.DryRun {
Expand Down

0 comments on commit 714a412

Please sign in to comment.