Add build-tool cli with a crd validation command

Gopkg.toml

@@ -31,6 +31,7 @@ required = [
   "k8s.io/code-generator/cmd/client-gen",
   "k8s.io/code-generator/cmd/lister-gen",
   "k8s.io/code-generator/cmd/informer-gen",
+  "k8s.io/kube-openapi/cmd/openapi-gen",
   "k8s.io/gengo/args",
 ]
 
@@ -85,6 +86,10 @@ required = [
   # revision = "6702109cc68eb6fe6350b83e14407c8d7309fd1a"
   version = "kubernetes-1.13.5"
 
+[[override]]
+  name = "k8s.io/kube-openapi"
+  revision = "743ec37842bffe49dd4221d9026f30fb1d5adbc4"
+
 [[override]]
   name = "github.com/graymeta/stow"
   revision = "903027f87de7054953efcdb8ba70d5dc02df38c7"
@@ -100,3 +105,12 @@ required = [
 [[override]]
   branch = "master"
   name = "golang.org/x/net"
+
+[[constraint]]
+  name = "github.com/kubeflow/crd-validation"
+  source = "github.com/bnsblue/crd-validation.git"
+  branch = "master"
+
+[[override]]
+  name = "github.com/spf13/viper"
+  version = "1.4.0"

Makefile

@@ -10,21 +10,25 @@ update_boilerplate:
 linux_compile:
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts/flytepropeller ./cmd/controller/main.go
 	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts/kubectl-flyte ./cmd/kubectl-flyte/main.go
+	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o /artifacts/build-tool ./cmd/build-tool/main.go
 
 .PHONY: compile
 compile:
 	mkdir -p ./bin
 	go build -o bin/flytepropeller ./cmd/controller/main.go 
 	go build -o bin/kubectl-flyte ./cmd/kubectl-flyte/main.go && cp bin/kubectl-flyte ${GOPATH}/bin
+	go build -o bin/build-tool ./cmd/build-tool/main.go && cp bin/build-tool ${GOPATH}/bin
 
 cross_compile:
 	@glide install
 	@mkdir -p ./bin/cross
 	GOOS=linux GOARCH=amd64 go build -o bin/cross/flytepropeller ./cmd/controller/main.go
 	GOOS=linux GOARCH=amd64 go build -o bin/cross/kubectl-flyte ./cmd/kubectl-flyte/main.go
+	GOOS=linux GOARCH=amd64 go build -o bin/cross/build-tool ./cmd/build-tool/main.go
 
 op_code_generate:
 	@RESOURCE_NAME=flyteworkflow OPERATOR_PKG=github.com/lyft/flytepropeller ./hack/update-codegen.sh
+	@openapi-gen -i github.com/lyft/flytepropeller/pkg/apis/flyteworkflow/v1alpha1 -p github.com/lyft/flytepropeller/pkg/apis/flyteworkflow/v1alpha1
 
 benchmark:
 	mkdir -p ./bin/benchmark 
@@ -41,4 +45,5 @@ clean:
 # Generate golden files. Add test packages that generate golden files here.
 golden:
 	go test ./cmd/kubectl-flyte/cmd -update
+	go test ./cmd/build-tool/cmd -update
 	go test ./pkg/compiler/test -update

cmd/build-tool/cmd/crd/flyteworkflow.go

@@ -0,0 +1,43 @@
+package crd
+
+import (
+	"github.com/lyft/flytepropeller/pkg/apis/flyteworkflow/v1alpha1"
+	apiextensions "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	"log"
+
+	"github.com/kubeflow/crd-validation/pkg/crd/exporter"
+	"github.com/kubeflow/crd-validation/pkg/utils"
+)
+
+const (
+	// CRDName is the name for FlyteWorkflow.
+	CRDNameFlyteWorkflow = "github.com/lyft/flytepropeller/pkg/apis/flyteworkflow/v1alpha1.FlyteWorkflow"
+
+	generatedFileFlyteWorkflow = "flyteworkflow-crd-v1alpha1.yaml"
+)
+
+// FlyteWorkflowGenerator is the type for FlyteWorkflow CRD generator.
+type FlyteWorkflowGenerator struct {
+	*exporter.Exporter
+}
+
+// Creates a new CRD generator which outputs to a file.
+func NewFlyteWorkflowGenerator(outputDir string) *FlyteWorkflowGenerator {
+	return &FlyteWorkflowGenerator{
+		Exporter: exporter.NewFileExporter(outputDir, generatedFileFlyteWorkflow),
+	}
+}
+
+// Creates a new CRD generator which outputs to stdout.
+func NewFlyteWorkflowGeneratorStdout() *FlyteWorkflowGenerator {
+	return &FlyteWorkflowGenerator{
+		Exporter: exporter.NewStdoutExporter(),
+	}
+}
+
+// Generate generates the crd.
+func (t FlyteWorkflowGenerator) Generate(original *apiextensions.CustomResourceDefinition) *apiextensions.CustomResourceDefinition {
+	log.Println("Generating validation")
+	original.Spec.Validation = utils.GetCustomResourceValidation(CRDNameFlyteWorkflow, v1alpha1.GetOpenAPIDefinitions)
+	return original
+}

cmd/build-tool/cmd/crd_validation.go

@@ -0,0 +1,95 @@
+package cmd
+
+import (
+	"github.com/spf13/viper"
+	"log"
+
+	compilerErrors "github.com/lyft/flytepropeller/pkg/compiler/errors"
+
+	"github.com/pkg/errors"
+	"github.com/spf13/cobra"
+
+	"github.com/kubeflow/crd-validation/pkg/config"
+	"github.com/lyft/flytepropeller/cmd/build-tool/cmd/crd"
+)
+
+const (
+	configKey      = "config-file"
+	baseCrdKey      = "base-crd"
+)
+
+
+const crdValidationCmdName = "crd-validation"
+
+type CrdValidationOpts struct {
+	*RootOptions
+	configFile  string
+	baseCrdFile string
+	dryRun      bool
+}
+
+func NewCrdValidationCommand(opts *RootOptions) *cobra.Command {
+
+	crdValidationOpts := &CrdValidationOpts{
+		RootOptions: opts,
+	}
+
+	crdValidationCmd := &cobra.Command{
+		Use:     crdValidationCmdName,
+		Aliases: []string{"validate"},
+		Short:   "Augment a CRD YAML file with validation section based on a base CRD file",
+		Long:    ``,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := requiredFlags(cmd, baseCrdKey); err != nil {
+				return err
+			}
+
+			compilerErrors.SetIncludeSource()
+
+			return crdValidationOpts.generateValidation()
+		},
+	}
+
+	crdValidationCmd.Flags().StringVarP(&crdValidationOpts.configFile, configKey, "c", "", "Path of the config file for the execution of CRD validation")
+	crdValidationCmd.Flags().StringVarP(&crdValidationOpts.baseCrdFile, baseCrdKey, "b", "", "Path to base CRD file.")
+	crdValidationCmd.Flags().BoolVarP(&crdValidationOpts.dryRun, "dry-run", "d", false, "Compiles and transforms, but does not create a workflow. OutputsRef ts to STDOUT.")
+
+	return crdValidationCmd
+}
+
+func (c *CrdValidationOpts) initConfig() error {
+	if c.configFile != "" { // enable ability to specify config file via flag
+		viper.SetConfigFile(c.configFile)
+		log.Println("Using config file:", viper.ConfigFileUsed())
+	}
+
+	viper.SetConfigType("yaml")  // Set config type to yaml
+
+	// If a config file is found, read it in.
+	if err := viper.ReadInConfig(); err != nil {
+		return errors.Wrapf(err, "Failed to read config file.")
+	} else {
+		log.Println("Using config file:", viper.ConfigFileUsed())
+	}
+	return nil
+}
+
+
+func (c *CrdValidationOpts) generateValidation() error {
+
+	err := c.initConfig()
+	var generator *crd.FlyteWorkflowGenerator
+	if err != nil {
+		log.Println("Output will be written to Stdout")
+		generator = crd.NewFlyteWorkflowGeneratorStdout()
+	} else {
+		crdValidationConfig := config.GetCrdValidationConfig()
+		generator = crd.NewFlyteWorkflowGenerator(crdValidationConfig.OutputDir)
+	}
+
+	original := config.NewCustomResourceDefinition(c.baseCrdFile)
+	final := generator.Generate(original)
+	generator.Export(final)
+
+	return nil
+}

cmd/build-tool/cmd/root.go

@@ -0,0 +1,55 @@
+package cmd
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"os"
+	"runtime"
+
+	"github.com/lyft/flytestdlib/logger"
+	"github.com/lyft/flytestdlib/version"
+	"github.com/spf13/pflag"
+
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	pflag.CommandLine.AddGoFlagSet(flag.CommandLine)
+	err := flag.CommandLine.Parse([]string{})
+	if err != nil {
+		logger.Error(context.TODO(), "Error in initializing: %v", err)
+		os.Exit(-1)
+	}
+}
+
+type RootOptions struct {
+	configFile string
+}
+
+func (r *RootOptions) executeRootCmd() error {
+	ctx := context.TODO()
+	logger.Infof(ctx, "Go Version: %s", runtime.Version())
+	logger.Infof(ctx, "Go OS/Arch: %s/%s", runtime.GOOS, runtime.GOARCH)
+	version.LogBuildInformation("build-tool")
+	return fmt.Errorf("use one of the sub-commands")
+}
+
+// NewCommand returns a new instance of an argo command
+func NewBuildToolCommand() *cobra.Command {
+	rootOpts := &RootOptions{}
+	command := &cobra.Command{
+		Use:   "build-tool",
+		Short: "build-tool are utility commands that help validating crds, etc.",
+		Long: `Flyte is a serverless workflow processing platform built for native execution on K8s.
+      It is extensible and flexible to allow adding new operators and comes with many operators built in`,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return rootOpts.executeRootCmd()
+		},
+	}
+
+	command.AddCommand(NewCrdValidationCommand(rootOpts))
+	return command
+}
+
+

cmd/build-tool/cmd/utils.go

@@ -0,0 +1,18 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+func requiredFlags(cmd *cobra.Command, flags ...string) error {
+	for _, flag := range flags {
+		f := cmd.Flag(flag)
+		if f == nil {
+			return fmt.Errorf("unable to find Key [%v]", flag)
+		}
+	}
+
+	return nil
+}

cmd/build-tool/main.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/lyft/flytepropeller/cmd/build-tool/cmd"
+)
+
+func main() {
+
+	rootCmd := cmd.NewBuildToolCommand()
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

cmd/kubectl-flyte/cmd/root.go

@@ -98,6 +98,7 @@ func NewFlyteCommand() *cobra.Command {
 	command.AddCommand(NewGetCommand(rootOpts))
 	command.AddCommand(NewVisualizeCommand(rootOpts))
 	command.AddCommand(NewCreateCommand(rootOpts))
+	// command.AddCommand(cmd.NewCrdValidationCommand(rootOpts))
 	command.AddCommand(NewCompileCommand(rootOpts))
 
 	loadingRules := clientcmd.NewDefaultClientConfigLoadingRules()

pkg/apis/flyteworkflow/v1alpha1/branch.go

@@ -71,6 +71,7 @@ func (in *IfBlock) GetThenNode() *NodeID {
 
 type BranchNodeSpec struct {
 	If       IfBlock    `json:"if"`
+	// +listType=atomic
 	ElseIf   []*IfBlock `json:"elseIf,omitempty"`
 	Else     *NodeID    `json:"else,omitempty"`
 	ElseFail *Error     `json:"elseFail,omitempty"`

pkg/apis/flyteworkflow/v1alpha1/doc.go

@@ -1,4 +1,5 @@
 // +k8s:deepcopy-gen=package
+// +k8s:openapi-gen=true
 
 // Package v1alpha1 is the v1alpha1 version of the API.
 // +groupName=flyteworkflow.flyte.net

pkg/apis/flyteworkflow/v1alpha1/nodes.go

@@ -97,9 +97,11 @@ type NodeSpec struct {
 	BranchNode    *BranchNodeSpec               `json:"branch,omitempty"`
 	TaskRef       *TaskID                       `json:"task,omitempty"`
 	WorkflowNode  *WorkflowNodeSpec             `json:"workflow,omitempty"`
+	// +listType=atomic
 	InputBindings []*Binding                    `json:"inputBindings,omitempty"`
 	Config        *typesv1.ConfigMap            `json:"config,omitempty"`
 	RetryStrategy *RetryStrategy                `json:"retry,omitempty"`
+	// +listType=atomic
 	OutputAliases []Alias                       `json:"outputAlias,omitempty"`
 
 	// SecurityContext holds pod-level security attributes and common container settings.
@@ -113,6 +115,7 @@ type NodeSpec struct {
 	// +optional
 	// +patchMergeKey=name
 	// +patchStrategy=merge
+	// +listType=atomic
 	ImagePullSecrets []typesv1.LocalObjectReference `json:"imagePullSecrets,omitempty" patchStrategy:"merge" patchMergeKey:"name" protobuf:"bytes,15,rep,name=imagePullSecrets"`
 	// Specifies the hostname of the Pod
 	// If not specified, the pod's hostname will be set to a system-defined value.
@@ -131,6 +134,7 @@ type NodeSpec struct {
 	SchedulerName string `json:"schedulerName,omitempty" protobuf:"bytes,19,opt,name=schedulerName"`
 	// If specified, the pod's tolerations.
 	// +optional
+	// +listType=atomic
 	Tolerations []typesv1.Toleration `json:"tolerations,omitempty" protobuf:"bytes,22,opt,name=tolerations"`
 	// StartTime before the system will actively try to mark it failed and kill associated containers.
 	// Value must be a positive integer.