From 9d7fd346495b119e895747d747c1c0a5bacb988e Mon Sep 17 00:00:00 2001 From: Zbynek Roubalik <726523+zroubalik@users.noreply.github.com> Date: Wed, 2 Jun 2021 09:20:28 +0200 Subject: [PATCH] feat: reference Secrets in `envs` and `volumes` sections in config (#369) * feat: reference Secrets in `envs` and `volumes` sections in config Signed-off-by: Zbynek Roubalik --- cmd/delete_test.go | 2 +- cmd/deploy.go | 34 +++- cmd/root.go | 62 +++--- cmd/root_test.go | 97 +++++++-- cmd/run.go | 37 +++- config.go | 136 ++++++++++++- config_test.go | 419 +++++++++++++++++++++++++++++++++++++++ docker/runner.go | 43 +++- docs/guides/func_yaml.md | 40 ++-- function.go | 6 +- knative/client.go | 11 + knative/deployer.go | 357 ++++++++++++++++++++++++++++++--- knative/deployer_test.go | 2 +- 13 files changed, 1124 insertions(+), 122 deletions(-) create mode 100644 config_test.go diff --git a/cmd/delete_test.go b/cmd/delete_test.go index b965ea5d9c..9bcafa544d 100644 --- a/cmd/delete_test.go +++ b/cmd/delete_test.go @@ -51,7 +51,7 @@ trigger: http builder: quay.io/boson/faas-go-builder builderMap: default: quay.io/boson/faas-go-builder -env: {} +envs: [] annotations: {} ` tmpDir, err := ioutil.TempDir("", "bar") diff --git a/cmd/deploy.go b/cmd/deploy.go index 31bfaec4c3..14d0c97337 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -13,6 +13,7 @@ import ( "github.com/pkg/errors" "github.com/spf13/cobra" "golang.org/x/term" + "knative.dev/client/pkg/util" bosonFunc "github.com/boson-project/func" "github.com/boson-project/func/buildpacks" @@ -25,8 +26,8 @@ import ( func init() { root.AddCommand(deployCmd) deployCmd.Flags().BoolP("confirm", "c", false, "Prompt to confirm all configuration options (Env: $FUNC_CONFIRM)") - deployCmd.Flags().StringArrayP("env", "e", []string{}, "Environment variable to set in the form NAME=VALUE. " + - "You may provide this flag multiple times for setting multiple environment variables. " + + deployCmd.Flags().StringArrayP("env", "e", []string{}, "Environment variable to set in the form NAME=VALUE. "+ + "You may provide this flag multiple times for setting multiple environment variables. "+ "To unset, specify the environment variable name followed by a \"-\" (e.g., NAME-).") deployCmd.Flags().StringP("image", "i", "", "Full image name in the form [registry]/[namespace]/[name]:[tag] (optional). This option takes precedence over --registry (Env: $FUNC_IMAGE") deployCmd.Flags().StringP("namespace", "n", "", "Namespace of the function to undeploy. By default, the namespace in func.yaml is used or the actual active namespace if not set in the configuration. (Env: $FUNC_NAMESPACE)") @@ -66,14 +67,21 @@ kn func deploy --image quay.io/myuser/myfunc -n myns func runDeploy(cmd *cobra.Command, _ []string) (err error) { - config := newDeployConfig(cmd).Prompt() + config, err := newDeployConfig(cmd) + if err != nil { + return err + } + config = config.Prompt() function, err := functionWithOverrides(config.Path, functionOverrides{Namespace: config.Namespace, Image: config.Image}) if err != nil { return } - function.Env = mergeEnvMaps(function.Env, config.Env) + function.Envs, err = mergeEnvs(function.Envs, config.EnvToUpdate, config.EnvToRemove) + if err != nil { + return + } // Check if the Function has been initialized if !function.Initialized() { @@ -260,12 +268,21 @@ type deployConfig struct { // Build the associated Function before deploying. Build bool - Env map[string]string + // Envs passed via cmd to be added/updated + EnvToUpdate *util.OrderedMap + + // Envs passed via cmd to removed + EnvToRemove []string } // newDeployConfig creates a buildConfig populated from command flags and // environment variables; in that precedence. -func newDeployConfig(cmd *cobra.Command) deployConfig { +func newDeployConfig(cmd *cobra.Command) (deployConfig, error) { + envToUpdate, envToRemove, err := envFromCmd(cmd) + if err != nil { + return deployConfig{}, err + } + return deployConfig{ buildConfig: newBuildConfig(), Namespace: viper.GetString("namespace"), @@ -273,8 +290,9 @@ func newDeployConfig(cmd *cobra.Command) deployConfig { Verbose: viper.GetBool("verbose"), // defined on root Confirm: viper.GetBool("confirm"), Build: viper.GetBool("build"), - Env: envFromCmd(cmd), - } + EnvToUpdate: envToUpdate, + EnvToRemove: envToRemove, + }, nil } // Prompt the user with value of config members, allowing for interaractive changes. diff --git a/cmd/root.go b/cmd/root.go index e01814376f..a4a39a8989 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -10,6 +10,8 @@ import ( "github.com/mitchellh/go-homedir" "github.com/ory/viper" "github.com/spf13/cobra" + "k8s.io/apimachinery/pkg/util/sets" + "knative.dev/client/pkg/util" bosonFunc "github.com/boson-project/func" ) @@ -251,42 +253,52 @@ func deriveImage(explicitImage, defaultRegistry, path string) string { return derivedValue // Use the func system's derivation logic. } -func envFromCmd(cmd *cobra.Command) map[string]string { - envM := make(map[string]string) +func envFromCmd(cmd *cobra.Command) (*util.OrderedMap, []string, error) { if cmd.Flags().Changed("env") { - envA, err := cmd.Flags().GetStringArray("env") - if err == nil { - for _, s := range envA { - kvp := strings.Split(s, "=") - if len(kvp) == 2 && kvp[0] != "" { - envM[kvp[0]] = kvp[1] - } else if len(kvp) == 1 && kvp[0] != "" { - envM[kvp[0]] = "" - } - } + env, err := cmd.Flags().GetStringArray("env") + if err != nil { + return nil, []string{}, fmt.Errorf("Invalid --env: %w", err) } + return util.OrderedMapAndRemovalListFromArray(env, "=") } - return envM + return util.NewOrderedMap(), []string{}, nil } -func mergeEnvMaps(dest, src map[string]string) map[string]string { - result := make(map[string]string, len(dest)+len(src)) +func mergeEnvs(envs bosonFunc.Envs, envToUpdate *util.OrderedMap, envToRemove []string) (bosonFunc.Envs, error) { + updated := sets.NewString() - for name, value := range dest { - if strings.HasSuffix(name, "-") { - if _, ok := src[strings.TrimSuffix(name, "-")]; !ok { - result[name] = value + for i := range envs { + if envs[i].Name != nil { + value, present := envToUpdate.GetString(*envs[i].Name) + if present { + envs[i].Value = &value + updated.Insert(*envs[i].Name) } - } else { - if _, ok := src[name+"-"]; !ok { - result[name] = value + } + } + + it := envToUpdate.Iterator() + for name, value, ok := it.NextString(); ok; name, value, ok = it.NextString() { + if !updated.Has(name) { + n := name + v := value + envs = append(envs, bosonFunc.Env{Name: &n, Value: &v}) + } + } + + for _, name := range envToRemove { + for i, envVar := range envs { + if *envVar.Name == name { + envs = append(envs[:i], envs[i+1:]...) + break } } } - for name, value := range src { - result[name] = value + errMsg := bosonFunc.ValidateEnvs(envs) + if len(errMsg) > 0 { + return bosonFunc.Envs{}, fmt.Errorf(strings.Join(errMsg, "\n")) } - return result + return envs, nil } diff --git a/cmd/root_test.go b/cmd/root_test.go index ef39d579a4..6dde24f6de 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -1,57 +1,118 @@ package cmd import ( + "fmt" "reflect" "testing" + + "knative.dev/client/pkg/util" + + bosonFunc "github.com/boson-project/func" ) func Test_mergeEnvMaps(t *testing.T) { + + a := "A" + b := "B" + v1 := "x" + v2 := "y" + type args struct { - dest map[string]string - src map[string]string + envs bosonFunc.Envs + toUpdate *util.OrderedMap + toRemove []string } tests := []struct { name string args args - want map[string]string + want bosonFunc.Envs }{ + { + "add new var to empty list", + args{ + bosonFunc.Envs{}, + util.NewOrderedMapWithKVStrings([][]string{{a, v1}}), + []string{}, + }, + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v1}}, + }, { "add new var", args{ - map[string]string{"A": "1"}, - map[string]string{"B": "2"}, + bosonFunc.Envs{bosonFunc.Env{Name: &b, Value: &v2}}, + util.NewOrderedMapWithKVStrings([][]string{{a, v1}}), + []string{}, }, - map[string]string{"A": "1", "B": "2"}, + bosonFunc.Envs{bosonFunc.Env{Name: &b, Value: &v2}, bosonFunc.Env{Name: &a, Value: &v1}}, }, { "update var", args{ - map[string]string{"A": "1"}, - map[string]string{"A": "2"}, + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v1}}, + util.NewOrderedMapWithKVStrings([][]string{{a, v2}}), + []string{}, + }, + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v2}}, + }, + { + "update multiple vars", + args{ + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v1}, bosonFunc.Env{Name: &b, Value: &v2}}, + util.NewOrderedMapWithKVStrings([][]string{{a, v2}, {b, v1}}), + []string{}, }, - map[string]string{"A": "2"}, + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v2}, bosonFunc.Env{Name: &b, Value: &v1}}, }, { "remove var", args{ - map[string]string{"A": "1"}, - map[string]string{"A-": ""}, + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v1}}, + util.NewOrderedMap(), + []string{a}, + }, + bosonFunc.Envs{}, + }, + { + "remove multiple vars", + args{ + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v1}, bosonFunc.Env{Name: &b, Value: &v2}}, + util.NewOrderedMap(), + []string{a, b}, }, - map[string]string{"A-": ""}, + bosonFunc.Envs{}, }, { - "re-add var", + "update and remove vars", args{ - map[string]string{"A-": ""}, - map[string]string{"A": "1"}, + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v1}, bosonFunc.Env{Name: &b, Value: &v2}}, + util.NewOrderedMapWithKVStrings([][]string{{a, v2}}), + []string{b}, }, - map[string]string{"A": "1"}, + bosonFunc.Envs{bosonFunc.Env{Name: &a, Value: &v2}}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := mergeEnvMaps(tt.args.dest, tt.args.src); !reflect.DeepEqual(got, tt.want) { - t.Errorf("mergeEnvMaps() = %v, want %v", got, tt.want) + got, err := mergeEnvs(tt.args.envs, tt.args.toUpdate, tt.args.toRemove) + if err != nil { + t.Errorf("mergeEnvs() for initial vars %v and toUpdate %v and toRemove %v got error %v", + tt.args.envs, tt.args.toUpdate, tt.args.toRemove, err) + } + if !reflect.DeepEqual(got, tt.want) { + + gotString := "{ " + for _, e := range got { + gotString += fmt.Sprintf("{ %s: %s } ", *e.Name, *e.Value) + } + gotString += "}" + + wantString := "{ " + for _, e := range tt.want { + wantString += fmt.Sprintf("{ %s: %s } ", *e.Name, *e.Value) + } + wantString += "}" + + t.Errorf("mergeEnvs() = got: %s, want %s", gotString, wantString) } }) } diff --git a/cmd/run.go b/cmd/run.go index b5b4a1ed2f..e8081095ce 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -5,6 +5,7 @@ import ( "github.com/ory/viper" "github.com/spf13/cobra" + "knative.dev/client/pkg/util" bosonFunc "github.com/boson-project/func" "github.com/boson-project/func/docker" @@ -13,8 +14,8 @@ import ( func init() { // Add the run command as a subcommand of root. root.AddCommand(runCmd) - runCmd.Flags().StringArrayP("env", "e", []string{}, "Environment variable to set in the form NAME=VALUE. " + - "You may provide this flag multiple times for setting multiple environment variables. " + + runCmd.Flags().StringArrayP("env", "e", []string{}, "Environment variable to set in the form NAME=VALUE. "+ + "You may provide this flag multiple times for setting multiple environment variables. "+ "To unset, specify the environment variable name followed by a \"-\" (e.g., NAME-).") runCmd.Flags().StringP("path", "p", cwd(), "Path to the project directory (Env: $FUNC_PATH)") } @@ -40,14 +41,20 @@ kn func run } func runRun(cmd *cobra.Command, args []string) (err error) { - config := newRunConfig(cmd) + config, err := newRunConfig(cmd) + if err != nil { + return + } function, err := bosonFunc.NewFunction(config.Path) if err != nil { return } - function.Env = mergeEnvMaps(function.Env, config.Env) + function.Envs, err = mergeEnvs(function.Envs, config.EnvToUpdate, config.EnvToRemove) + if err != nil { + return + } err = function.WriteConfig() if err != nil { @@ -78,13 +85,23 @@ type runConfig struct { // Verbose logging. Verbose bool - Env map[string]string + // Envs passed via cmd to be added/updated + EnvToUpdate *util.OrderedMap + + // Envs passed via cmd to removed + EnvToRemove []string } -func newRunConfig(cmd *cobra.Command) runConfig { - return runConfig{ - Path: viper.GetString("path"), - Verbose: viper.GetBool("verbose"), // defined on root - Env: envFromCmd(cmd), +func newRunConfig(cmd *cobra.Command) (runConfig, error) { + envToUpdate, envToRemove, err := envFromCmd(cmd) + if err != nil { + return runConfig{}, err } + + return runConfig{ + Path: viper.GetString("path"), + Verbose: viper.GetBool("verbose"), // defined on root + EnvToUpdate: envToUpdate, + EnvToRemove: envToRemove, + }, nil } diff --git a/config.go b/config.go index d43bc1b126..9a6f8b57c5 100644 --- a/config.go +++ b/config.go @@ -2,6 +2,7 @@ package function import ( "errors" + "fmt" "io/ioutil" "os" "path/filepath" @@ -14,6 +15,19 @@ import ( // ConfigFile is the name of the config's serialized form. const ConfigFile = "func.yaml" +type Volumes []Volume +type Volume struct { + Secret *string `yaml:"secret,omitempty"` + ConfigMap *string `yaml:"configMap,omitempty"` + Path *string `yaml:"path"` +} + +type Envs []Env +type Env struct { + Name *string `yaml:"name,omitempty"` + Value *string `yaml:"value"` +} + // Config represents the serialized state of a Function's metadata. // See the Function struct for attribute documentation. type config struct { @@ -25,7 +39,8 @@ type config struct { Trigger string `yaml:"trigger"` Builder string `yaml:"builder"` BuilderMap map[string]string `yaml:"builderMap"` - Env map[string]string `yaml:"env"` + Volumes Volumes `yaml:"volumes"` + Envs Envs `yaml:"envs"` Annotations map[string]string `yaml:"annotations"` // Add new values to the toConfig/fromConfig functions. } @@ -49,19 +64,54 @@ func newConfig(root string) (c config, err error) { return } + errMsg := "" + errMsgHeader := "'func.yaml' config file is not valid:\n" + errMsgReg := regexp.MustCompile("not found in type .*") + + // Let's try to unmarshal the config file, any fields that are found + // in the data that do not have corresponding struct members, or mapping + // keys that are duplicates, will result in an error. err = yaml.UnmarshalStrict(bb, &c) if err != nil { - errMsg := err.Error() - reg := regexp.MustCompile("not found in type .*") + errMsg = err.Error() if strings.HasPrefix(errMsg, "yaml: unmarshal errors:") { - errMsg = reg.ReplaceAllString(errMsg, "is not valid") - err = errors.New(strings.Replace(errMsg, "yaml: unmarshal errors:", "'func.yaml' config file is not valid:", 1)) + errMsg = errMsgReg.ReplaceAllString(errMsg, "is not valid") + errMsg = strings.Replace(errMsg, "yaml: unmarshal errors:\n", errMsgHeader, 1) } else if strings.HasPrefix(errMsg, "yaml:") { - errMsg = reg.ReplaceAllString(errMsg, "is not valid") - err = errors.New(strings.Replace(errMsg, "yaml: ", "'func.yaml' config file is not valid:\n ", 1)) + errMsg = errMsgReg.ReplaceAllString(errMsg, "is not valid") + errMsg = strings.Replace(errMsg, "yaml: ", errMsgHeader+" ", 1) + } + } + + // Let's check that all entries in `volumes` and `envs` contain all required fields + volumesErrors := validateVolumes(c.Volumes) + envsErrors := ValidateEnvs(c.Envs) + if len(volumesErrors) > 0 || len(envsErrors) > 0 { + // if there aren't any previously reported errors, we need to set the error message header first + if errMsg == "" { + errMsg = errMsgHeader + } else { + // if there are some previously reporeted errors, we need to indent them + errMsg = errMsg + "\n" } + + // lets make the error message a little bit nice -> indent each error message + for i := range volumesErrors { + volumesErrors[i] = " " + volumesErrors[i] + } + for i := range envsErrors { + envsErrors[i] = " " + envsErrors[i] + } + + errMsg = errMsg + strings.Join(volumesErrors, "\n") + errMsg = errMsg + strings.Join(envsErrors, "\n") } + + if errMsg != "" { + err = errors.New(errMsg) + } + return } @@ -77,7 +127,8 @@ func fromConfig(c config) (f Function) { Trigger: c.Trigger, Builder: c.Builder, BuilderMap: c.BuilderMap, - Env: c.Env, + Volumes: c.Volumes, + Envs: c.Envs, Annotations: c.Annotations, } } @@ -93,7 +144,8 @@ func toConfig(f Function) config { Trigger: f.Trigger, Builder: f.Builder, BuilderMap: f.BuilderMap, - Env: f.Env, + Volumes: f.Volumes, + Envs: f.Envs, Annotations: f.Annotations, } } @@ -108,3 +160,69 @@ func writeConfig(f Function) (err error) { } return ioutil.WriteFile(path, bb, 0644) } + +// validateVolumes checks that input Volumes are correct and contain all necessary fields. +// Returns array of error messages, empty if none +// +// Allowed settings: +// - secret: example-secret # mount Secret as Volume +// path: /etc/secret-volume +func validateVolumes(volumes Volumes) (errors []string) { + + for i, vol := range volumes { + if vol.Path == nil && vol.Secret == nil { + errors = append(errors, fmt.Sprintf("volume entry #%d is not properly set", i)) + } else if vol.Path == nil { + errors = append(errors, fmt.Sprintf("volume entry #%d is missing path field, only secret '%s' is set", i, *vol.Secret)) + } else if vol.Secret == nil { + errors = append(errors, fmt.Sprintf("volume entry #%d is missing secret field, only path '%s' is set", i, *vol.Path)) + } + } + + return +} + +// ValidateEnvs checks that input Envs are correct and contain all necessary fields. +// Returns array of error messages, empty if none +// +// Allowed settings: +// - name: EXAMPLE1 # ENV directly from a value +// value: value1 +// - name: EXAMPLE2 # ENV from the local ENV var +// value: {{ env.MY_ENV }} +// - name: EXAMPLE3 +// value: {{ secret.secretName.key }} # ENV from a key in secret +// - value: {{ secret.secretName }} # all key-pair values from secret are set as ENV +func ValidateEnvs(envs Envs) (errors []string) { + + // there could be '-' char in the secret name, but not in the key + regWholeSecret := regexp.MustCompile(`^{{\s*secret\.(?:\w|['-]\w)+\s*}}$`) + regKeyFromSecret := regexp.MustCompile(`^{{\s*secret\.(?:\w|['-]\w)+\.\w+\s*}}$`) + regLocalEnv := regexp.MustCompile(`^{{\s*env\.(\w+)\s*}}$`) + + for i, env := range envs { + if env.Name == nil && env.Value == nil { + errors = append(errors, fmt.Sprintf("env entry #%d is not properly set", i)) + } else if env.Value == nil { + errors = append(errors, fmt.Sprintf("env entry #%d is missing value field, only name '%s' is set", i, *env.Name)) + } else if env.Name == nil { + // all key-pair values from secret are set as ENV; {{ secret.secretName }} + if !regWholeSecret.MatchString(*env.Value) { + errors = append(errors, fmt.Sprintf("env entry #%d has invalid value field set, it has '%s', but allowed is only '{{ secret.secretName }}'", i, *env.Value)) + } + } else { + if strings.HasPrefix(*env.Value, "{{") { + // ENV from the local ENV var; {{ env.MY_ENV }} + // or + // ENV from a key in secret; {{ secret.secretName.key }} + if !regLocalEnv.MatchString(*env.Value) && !regKeyFromSecret.MatchString(*env.Value) { + errors = append(errors, + fmt.Sprintf("env entry #%d with name '%s' has invalid value field set, it has '%s', but allowed is only '{{ env.MY_ENV }}' or '{{ secret.secretName.key }}'", i, *env.Name, *env.Value)) + } + } + + } + } + + return +} diff --git a/config_test.go b/config_test.go new file mode 100644 index 0000000000..ed48e262d0 --- /dev/null +++ b/config_test.go @@ -0,0 +1,419 @@ +// +build !integration + +package function + +import ( + "testing" +) + +func Test_validateVolumes(t *testing.T) { + + secret := "secret" + path := "path" + secret2 := "secret2" + path2 := "path2" + + tests := []struct { + name string + volumes Volumes + errs int + }{ + { + "correct entry - single volume", + Volumes{ + Volume{ + Secret: &secret, + Path: &path, + }, + }, + 0, + }, + { + "correct entry - multiple volumes", + Volumes{ + Volume{ + Secret: &secret, + Path: &path, + }, + Volume{ + Secret: &secret2, + Path: &path2, + }, + }, + 0, + }, + { + "missing secret - single volume", + Volumes{ + Volume{ + Path: &path, + }, + }, + 1, + }, + { + "missing secret - single volume", + Volumes{ + Volume{ + Secret: &secret, + }, + }, + 1, + }, + { + "missing secret and path - single volume", + Volumes{ + Volume{}, + }, + 1, + }, + { + "missing secret in one volume - multiple volumes", + Volumes{ + Volume{ + Secret: &secret, + Path: &path, + }, + Volume{ + Path: &path2, + }, + }, + 1, + }, + { + "missing secret and path in two different volumes - multiple volumes", + Volumes{ + Volume{ + Secret: &secret, + Path: &path, + }, + Volume{ + Secret: &secret, + }, + Volume{ + Path: &path2, + }, + }, + 2, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := validateVolumes(tt.volumes); len(got) != tt.errs { + t.Errorf("validateVolumes() = %v\n got %d errors but want %d", got, len(got), tt.errs) + } + }) + } + +} + +func Test_validateEnvs(t *testing.T) { + + name := "name" + name2 := "name2" + value := "value" + value2 := "value2" + + valueLocalEnv := "{{ env.MY_ENV }}" + valueLocalEnv2 := "{{ env.MY_ENV2 }}" + valueLocalEnv3 := "{{env.MY_ENV3}}" + valueLocalEnvIncorrect := "{{ envs.MY_ENV }}" + valueLocalEnvIncorrect2 := "{{ MY_ENV }}" + valueLocalEnvIncorrect3 := "{{env.MY_ENV}}foo" + + valueSecretKey := "{{ secret.mysecret.key }}" + valueSecretKey2 := "{{secret.my-secret.key }}" + valueSecretKey3 := "{{secret.my-secret.key2}}" + valueSecretKeyIncorrect := "{{ secret.my-secret.key.key }}" + valueSecretKeyIncorrect2 := "{{ my-secret.key }}" + valueSecretKeyIncorrect3 := "{{ secret.my-secret.key }}foo" + + valueSecret := "{{ secret.my-secret }}" + valueSecret2 := "{{ secret.mysecret }}" + valueSecret3 := "{{ secret.mysecret}}" + valueSecretIncorrect := "{{ my-secret }}" + valueSecretIncorrect2 := "my-secret" + valueSecretIncorrect3 := "{{ secret.my-secret }}foo" + + tests := []struct { + name string + envs Envs + errs int + }{ + { + "correct entry - single env with value", + Envs{ + Env{ + Name: &name, + Value: &value, + }, + }, + 0, + }, + { + "incorrect entry - missing value", + Envs{ + Env{ + Name: &name, + }, + }, + 1, + }, + { + "correct entry - multiple envs with value", + Envs{ + Env{ + Name: &name, + Value: &value, + }, + Env{ + Name: &name2, + Value: &value2, + }, + }, + 0, + }, + { + "incorrect entry - mmissing value - multiple envs", + Envs{ + Env{ + Name: &name, + }, + Env{ + Name: &name2, + }, + }, + 2, + }, + { + "correct entry - single env with value Local env", + Envs{ + Env{ + Name: &name, + Value: &valueLocalEnv, + }, + }, + 0, + }, + { + "correct entry - multiple envs with value Local env", + Envs{ + Env{ + Name: &name, + Value: &valueLocalEnv, + }, + Env{ + Name: &name, + Value: &valueLocalEnv2, + }, + Env{ + Name: &name, + Value: &valueLocalEnv3, + }, + }, + 0, + }, + { + "incorrect entry - multiple envs with value Local env", + Envs{ + Env{ + Name: &name, + Value: &valueLocalEnv, + }, + Env{ + Name: &name, + Value: &valueLocalEnvIncorrect, + }, + Env{ + Name: &name, + Value: &valueLocalEnvIncorrect2, + }, + Env{ + Name: &name, + Value: &valueLocalEnvIncorrect3, + }, + }, + 3, + }, + { + "correct entry - single secret with key", + Envs{ + Env{ + Name: &name, + Value: &valueSecretKey, + }, + }, + 0, + }, + { + "correct entry - multiple secrets with key", + Envs{ + Env{ + Name: &name, + Value: &valueSecretKey, + }, + Env{ + Name: &name, + Value: &valueSecretKey2, + }, + Env{ + Name: &name, + Value: &valueSecretKey3, + }, + }, + 0, + }, + { + "incorrect entry - single secret with key", + Envs{ + Env{ + Name: &name, + Value: &valueSecretKeyIncorrect, + }, + }, + 1, + }, + { + "incorrect entry - mutliple secrets with key", + Envs{ + Env{ + Name: &name, + Value: &valueSecretKey, + }, + Env{ + Name: &name, + Value: &valueSecretKeyIncorrect, + }, + Env{ + Name: &name, + Value: &valueSecretKeyIncorrect2, + }, + Env{ + Name: &name, + Value: &valueSecretKeyIncorrect3, + }, + }, + 3, + }, + { + "correct entry - single whole secret", + Envs{ + Env{ + Value: &valueSecret, + }, + }, + 0, + }, + { + "correct entry - multiple whole secret2", + Envs{ + Env{ + Value: &valueSecret, + }, + Env{ + Value: &valueSecret2, + }, + Env{ + Value: &valueSecret3, + }, + }, + 0, + }, + { + "incorrect entry - single whole secret", + Envs{ + Env{ + Value: &value, + }, + }, + 1, + }, + { + "incorrect entry - multiple whole secret2", + Envs{ + Env{ + Value: &valueSecretIncorrect, + }, + Env{ + Value: &valueSecretIncorrect2, + }, + Env{ + Value: &valueSecretIncorrect3, + }, + Env{ + Value: &value, + }, + Env{ + Value: &valueLocalEnv, + }, + Env{ + Value: &valueLocalEnv2, + }, + Env{ + Value: &valueLocalEnv3, + }, + Env{ + Value: &valueSecret, + }, + }, + 7, + }, + { + "correct entry - all combinations", + Envs{ + Env{ + Name: &name, + Value: &value, + }, + Env{ + Name: &name2, + Value: &value2, + }, + Env{ + Name: &name, + Value: &valueLocalEnv, + }, + Env{ + Name: &name, + Value: &valueLocalEnv2, + }, + Env{ + Name: &name, + Value: &valueLocalEnv3, + }, + Env{ + Value: &valueSecret, + }, + Env{ + Value: &valueSecret2, + }, + Env{ + Value: &valueSecret3, + }, + Env{ + Name: &name, + Value: &valueSecretKey, + }, + Env{ + Name: &name, + Value: &valueSecretKey2, + }, + Env{ + Name: &name, + Value: &valueSecretKey3, + }, + }, + 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ValidateEnvs(tt.envs); len(got) != tt.errs { + t.Errorf("validateEnvs() = %v\n got %d errors but want %d", got, len(got), tt.errs) + } + }) + } + +} diff --git a/docker/runner.go b/docker/runner.go index 50f10fca5d..d2d78aa4ad 100644 --- a/docker/runner.go +++ b/docker/runner.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "regexp" "strings" "time" @@ -43,10 +44,16 @@ func (n *Runner) Run(ctx context.Context, f bosonFunc.Function) error { return errors.New("Function has no associated Image. Has it been built?") } - envs := make([]string, 0, len(f.Env)+1) - for name, value := range f.Env { - if !strings.HasSuffix(name, "-") { - envs = append(envs, fmt.Sprintf("%s=%s", name, value)) + envs := []string{} + for _, env := range f.Envs { + if env.Name != nil && env.Value != nil { + value, set, err := processEnvValue(*env.Value) + if err != nil { + return err + } + if set { + envs = append(envs, *env.Name+"="+value) + } } } if n.Verbose { @@ -136,3 +143,31 @@ func (n *Runner) Run(ctx context.Context, f bosonFunc.Function) error { return nil } + +// run command supports only ENV values in from FOO=bar or FOO={{ env.LOCAL_VALUE }} +var evRegex = regexp.MustCompile(`^{{\s*(\w+)\s*.(\w+)\s*}}$`) + +const ( + ctxIdx = 1 + valIdx = 2 +) + +// processEnvValue returns only value for ENV variable, that is defined in form FOO=bar or FOO={{ env.LOCAL_VALUE }} +// if the value is correct, it is returned and the second return parameter is set to `true` +// otherwise it is set to `false` +// if the specified value is correct, but the required local variable is not set, error is returned as well +func processEnvValue(val string) (string, bool, error) { + if strings.HasPrefix(val, "{{") { + match := evRegex.FindStringSubmatch(val) + if len(match) > valIdx && match[ctxIdx] == "env" { + if v, ok := os.LookupEnv(match[valIdx]); ok { + return v, true, nil + } else { + return "", false, fmt.Errorf("required local environment variable %q is not set", match[valIdx]) + } + } else { + return "", false, nil + } + } + return val, true, nil +} diff --git a/docs/guides/func_yaml.md b/docs/guides/func_yaml.md index 4dbc4c8007..f3b3b9bc58 100644 --- a/docs/guides/func_yaml.md +++ b/docs/guides/func_yaml.md @@ -28,18 +28,33 @@ field will contain all of the available builders for a given runtime. Although it's typically unnecessary to modify the `builder` field, using values from `builderMap` is OK. -### `env` +### `envs` -The `env` field allows you to set environment variables that will be -available to your function at runtime. For example, to set a `MODE` environment -variable to `debug` when the function is deployed, your `func.yaml` file -may look like this. To unset variables use a dash suffix. For example, to unset `MODE`, use `MODE-`. +The `envs` field allows you to set environment variables that will be +available to your function at runtime. +1. Environment variable can be set directly from a value +2. Environment variable can be set from a local environment value. Eg. `'{{ env.LOCAL_ENV_VALUE }}'`, for more details see [Local Environment Variables section](#local-environment-variables). +3. Environment variable can be set from a key in a Kubernetes Secret. This Secret needs to be created before it is referenced in a function. Eg. `'{{ secret.mysecret.key }}'` where `mysecret` is the name of the Secret and `key` is the referenced key. +4. All key-value pairs from a Kubernetes Secret will be set as environment variables. This Secret needs to be created before it is referenced in a function. Eg. `'{{ secret.mysecret2 }}'` where `mysecret2` is the name of the Secret. ```yaml -env: - MODE: debug - API_KEY: {{ env.API_KEY }} - VAR_TO_UNSET-: "" +envs: +- name: EXAMPLE1 # (1) env variable directly from a value + value: value +- name: EXAMPLE2 # (2) env variable from a local environment value + value: '{{ env.LOCAL_ENV_VALUE }}' +- name: EXAMPLE3 # (3) env variable from a key in Secret + value: '{{ secret.mysecret.key }}' +- value: '{{ secret.mysecret2 }}' # (4) all key-value pairs in Secret as env variables +``` + +### `volumes` +Kubernetes Secrets can be mounted to the function as a Kubernetes Volume accessible under specified path. Below you can see an example how to mount the Secret `mysecret` to the path `/workspace/secret`. This Secret needs to be created before it is referenced in a function. + +```yaml +volumes: +- secret: mysecret + path: /workspace/secret ``` ### `image` @@ -73,7 +88,7 @@ The invocation event that triggers your function. Possible values are `http` for plain HTTP requests, and `events` for CloudEvent triggered functions. -## Environment Variables +## Local Environment Variables Any of the fields in `func.yaml` may contain a reference to an environment variable available in the local environment. For example, if I would like @@ -83,6 +98,7 @@ this, prefix the local environment variable with `{{` and `}}` and prefix the name with `env.`. For example: ```yaml -env: - API_KEY: {{ env.API_KEY }} +envs: +- name: API_KEY + value: '{{ env.API_KEY }}' ``` diff --git a/function.go b/function.go index 210ff03052..9ffe28cda1 100644 --- a/function.go +++ b/function.go @@ -52,7 +52,11 @@ type Function struct { // e.g. { "jvm": "docker.io/example/quarkus-jvm-builder" } BuilderMap map[string]string - Env map[string]string + // List of volumes to be mounted to the function + Volumes Volumes + + // Env variables to be set + Envs Envs // Map containing user-supplied annotations // Example: { "division": "finance" } diff --git a/knative/client.go b/knative/client.go index ac26bde7e4..ce72cb7d37 100644 --- a/knative/client.go +++ b/knative/client.go @@ -4,6 +4,7 @@ import ( "fmt" "time" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/clientcmd" clienteventingv1beta1 "knative.dev/client/pkg/eventing/v1beta1" clientservingv1 "knative.dev/client/pkg/serving/v1" @@ -49,6 +50,16 @@ func NewEventingClient(namespace string) (clienteventingv1beta1.KnEventingClient return client, nil } +func NewKubernetesClientset(namespace string) (*kubernetes.Clientset, error) { + + restConfig, err := getClientConfig().ClientConfig() + if err != nil { + return nil, fmt.Errorf("failed to create new kubernetes client: %v", err) + } + + return kubernetes.NewForConfig(restConfig) +} + func GetNamespace(defaultNamespace string) (namespace string, err error) { namespace = defaultNamespace diff --git a/knative/deployer.go b/knative/deployer.go index 7881f7e533..b6a93d8c1b 100644 --- a/knative/deployer.go +++ b/knative/deployer.go @@ -11,6 +11,7 @@ import ( corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/sets" "knative.dev/client/pkg/kn/flags" "knative.dev/client/pkg/wait" servingv1 "knative.dev/serving/pkg/apis/serving/v1" @@ -48,14 +49,24 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (result fn.Deploym if err != nil { if errors.IsNotFound(err) { - service, err := generateNewService(f.Name, f.ImageWithDigest(), f.Runtime, f.Env, f.Annotations) + referencedSecrets := sets.NewString() + referencedConfigMaps := sets.NewString() + + service, err := generateNewService(f.Name, f.ImageWithDigest(), f.Runtime, f.Envs, f.Volumes, f.Annotations) + if err != nil { + err = fmt.Errorf("knative deployer failed to generate the Knative Service: %v", err) + return fn.DeploymentResult{}, err + } + + err = checkSecretsConfigMapsArePresent(ctx, d.Namespace, &referencedSecrets, &referencedConfigMaps) if err != nil { - err = fmt.Errorf("knative deployer failed to generate the service: %v", err) + err = fmt.Errorf("knative deployer failed to generate the Knative Service: %v", err) return fn.DeploymentResult{}, err } + err = client.CreateService(ctx, service) if err != nil { - err = fmt.Errorf("knative deployer failed to deploy the service: %v", err) + err = fmt.Errorf("knative deployer failed to deploy the Knative Service: %v", err) return fn.DeploymentResult{}, err } @@ -64,13 +75,13 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (result fn.Deploym } err, _ = client.WaitForService(ctx, f.Name, DefaultWaitingTimeout, wait.NoopMessageCallback()) if err != nil { - err = fmt.Errorf("knative deployer failed to wait for the service to become ready: %v", err) + err = fmt.Errorf("knative deployer failed to wait for the Knative Service to become ready: %v", err) return fn.DeploymentResult{}, err } route, err := client.GetRoute(ctx, f.Name) if err != nil { - err = fmt.Errorf("knative deployer failed to get the route: %v", err) + err = fmt.Errorf("knative deployer failed to get the Route: %v", err) return fn.DeploymentResult{}, err } @@ -79,23 +90,41 @@ func (d *Deployer) Deploy(ctx context.Context, f fn.Function) (result fn.Deploym Status: fn.Deployed, URL: route.Status.URL.String(), }, nil - // fmt.Sprintf("Function deployed at URL: %v", route.Status.URL.String()), nil } else { - err = fmt.Errorf("knative deployer failed to get the service: %v", err) + err = fmt.Errorf("knative deployer failed to get the Knative Service: %v", err) return fn.DeploymentResult{}, err } } else { // Update the existing Service - _, err = client.UpdateServiceWithRetry(ctx, f.Name, updateService(f.ImageWithDigest(), f.Env, f.Annotations), 3) + referencedSecrets := sets.NewString() + referencedConfigMaps := sets.NewString() + + newEnv, newEnvFrom, err := processEnvs(f.Envs, &referencedSecrets, &referencedConfigMaps) + if err != nil { + return fn.DeploymentResult{}, err + } + + newVolumes, newVolumeMounts, err := processVolumes(f.Volumes, &referencedSecrets, &referencedConfigMaps) + if err != nil { + return fn.DeploymentResult{}, err + } + + err = checkSecretsConfigMapsArePresent(ctx, d.Namespace, &referencedSecrets, &referencedConfigMaps) + if err != nil { + err = fmt.Errorf("knative deployer failed to update the Knative Service: %v", err) + return fn.DeploymentResult{}, err + } + + _, err = client.UpdateServiceWithRetry(ctx, f.Name, updateService(f.ImageWithDigest(), newEnv, newEnvFrom, newVolumes, newVolumeMounts, f.Annotations), 3) if err != nil { - err = fmt.Errorf("knative deployer failed to update the service: %v", err) + err = fmt.Errorf("knative deployer failed to update the Knative Service: %v", err) return fn.DeploymentResult{}, err } route, err := client.GetRoute(ctx, f.Name) if err != nil { - err = fmt.Errorf("knative deployer failed to get the route: %v", err) + err = fmt.Errorf("knative deployer failed to get the Route: %v", err) return fn.DeploymentResult{}, err } @@ -116,7 +145,7 @@ func probeFor(url string) *corev1.Probe { } } -func generateNewService(name, image, runtime string, env map[string]string, annotations map[string]string) (*servingv1.Service, error) { +func generateNewService(name, image, runtime string, envs fn.Envs, volumes fn.Volumes, annotations map[string]string) (*servingv1.Service, error) { containers := []corev1.Container{ { Image: image, @@ -128,6 +157,22 @@ func generateNewService(name, image, runtime string, env map[string]string, anno containers[0].ReadinessProbe = probeFor("/health/readiness") } + referencedSecrets := sets.NewString() + referencedConfigMaps := sets.NewString() + + newEnv, newEnvFrom, err := processEnvs(envs, &referencedSecrets, &referencedConfigMaps) + if err != nil { + return nil, err + } + containers[0].Env = newEnv + containers[0].EnvFrom = newEnvFrom + + newVolumes, newVolumeMounts, err := processVolumes(volumes, &referencedSecrets, &referencedConfigMaps) + if err != nil { + return nil, err + } + containers[0].VolumeMounts = newVolumeMounts + service := &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: name, @@ -143,6 +188,7 @@ func generateNewService(name, image, runtime string, env map[string]string, anno Spec: v1.RevisionSpec{ PodSpec: corev1.PodSpec{ Containers: containers, + Volumes: newVolumes, }, }, }, @@ -150,10 +196,11 @@ func generateNewService(name, image, runtime string, env map[string]string, anno }, } - return setEnv(service, env) + return service, nil } -func updateService(image string, env map[string]string, annotations map[string]string) func(service *servingv1.Service) (*servingv1.Service, error) { +func updateService(image string, newEnv []corev1.EnvVar, newEnvFrom []corev1.EnvFromSource, newVolumes []corev1.Volume, newVolumeMounts []corev1.VolumeMount, + annotations map[string]string) func(service *servingv1.Service) (*servingv1.Service, error) { return func(service *servingv1.Service) (*servingv1.Service, error) { // Removing the name so the k8s server can fill it in with generated name, // this prevents conflicts in Revision name when updating the KService from multiple places. @@ -169,8 +216,176 @@ func updateService(image string, env map[string]string, annotations map[string]s if err != nil { return service, err } - return setEnv(service, env) + + service.Spec.ConfigurationSpec.Template.Spec.Containers[0].Env = newEnv + service.Spec.ConfigurationSpec.Template.Spec.Containers[0].EnvFrom = newEnvFrom + + service.Spec.ConfigurationSpec.Template.Spec.Containers[0].VolumeMounts = newVolumeMounts + service.Spec.ConfigurationSpec.Template.Spec.Volumes = newVolumes + + return service, nil + } +} + +// processEnvs generates array of EnvVars and EnvFromSources from a function config +// envs: +// - name: EXAMPLE1 # ENV directly from a value +// value: value1 +// - name: EXAMPLE2 # ENV from the local ENV var +// value: {{ env.MY_ENV }} +// - name: EXAMPLE3 +// value: {{ secret.example-secret.key }} # ENV from a key in Secret +// - value: {{ secret.example-secret }} # all ENVs from Secret +func processEnvs(envs fn.Envs, referencedSecrets, referencedConfigMaps *sets.String) ([]corev1.EnvVar, []corev1.EnvFromSource, error) { + + envVars := []corev1.EnvVar{{Name: "BUILT", Value: time.Now().Format("20060102T150405")}} + envFrom := []corev1.EnvFromSource{} + + for _, env := range envs { + if env.Name == nil && env.Value != nil { + // all key-pair values from secret are set as ENV, eg. {{ secret.secretName }} + if strings.HasPrefix(*env.Value, "{{") { + envFromSource, err := createEnvFromSource(*env.Value, referencedSecrets, referencedConfigMaps) + if err != nil { + return nil, nil, err + } + envFrom = append(envFrom, *envFromSource) + continue + } + } else if env.Name != nil && env.Value != nil { + if strings.HasPrefix(*env.Value, "{{") { + slices := strings.Split(strings.Trim(*env.Value, "{} "), ".") + if len(slices) == 3 { + // ENV from a key in secret, eg. FOO={{ secret.secretName.key }} + valueFrom, err := createEnvVarSource(slices, referencedSecrets, referencedConfigMaps) + envVars = append(envVars, corev1.EnvVar{Name: *env.Name, ValueFrom: valueFrom}) + if err != nil { + return nil, nil, err + } + continue + } else if len(slices) == 2 { + // ENV from the local ENV var, eg. FOO={{ env.LOCAL_ENV }} + localValue, err := processLocalEnvValue(*env.Value) + if err != nil { + return nil, nil, err + } + envVars = append(envVars, corev1.EnvVar{Name: *env.Name, Value: localValue}) + continue + } + } else { + // a standard ENV with key and value, eg. FOO=bar + envVars = append(envVars, corev1.EnvVar{Name: *env.Name, Value: *env.Value}) + continue + } + } + return nil, nil, fmt.Errorf("unsupported env source entry \"%v\"", env) + } + + return envVars, envFrom, nil +} + +func createEnvFromSource(value string, referencedSecrets, referencedConfigMaps *sets.String) (*corev1.EnvFromSource, error) { + slices := strings.Split(strings.Trim(value, "{} "), ".") + if len(slices) != 2 { + // following error message can be used when we enable ConfigMaps: + // "env requires a value in form \"resourceType.name\" where \"resourceType\" can be one of \"config-map\" or \"secret\""; got %q" + return nil, fmt.Errorf("env requires a value in form \"{{ secret.name }}\"; got %q", slices) + } + + envVarSource := corev1.EnvFromSource{} + + typeString := strings.TrimSpace(slices[0]) + sourceName := strings.TrimSpace(slices[1]) + + var sourceType string + + switch typeString { + case "config-map": + sourceType = "ConfigMap" + envVarSource.ConfigMapRef = &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: sourceName, + }} + + if !referencedConfigMaps.Has(sourceName) { + referencedConfigMaps.Insert(sourceName) + } + case "secret": + sourceType = "Secret" + envVarSource.SecretRef = &corev1.SecretEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: sourceName, + }} + if !referencedSecrets.Has(sourceName) { + referencedSecrets.Insert(sourceName) + } + default: + // following error message can be used when we enable ConfigMaps: + // "unsupported env source type \"%q\"; supported source types are \"config-map\" or \"secret\"" + return nil, fmt.Errorf("unsupported env source type \"%q\"; supported source types is \"secret\"", slices[0]) + } + + if len(sourceName) == 0 { + return nil, fmt.Errorf("the name of %s cannot be an empty string", sourceType) + } + + return &envVarSource, nil +} + +func createEnvVarSource(slices []string, referencedSecrets, referencedConfigMaps *sets.String) (*corev1.EnvVarSource, error) { + + if len(slices) != 3 { + // following error message can be used when we enable ConfigMaps: + // "env requires a value in form \"resourceType.name.key\" where \"resourceType\" can be one of \"configMap\" or \"secret\""; got %q" + return nil, fmt.Errorf("env requires a value in form \"{{ secret.name.key }}\"; got %q", slices) + } + + envVarSource := corev1.EnvVarSource{} + + typeString := strings.TrimSpace(slices[0]) + sourceName := strings.TrimSpace(slices[1]) + sourceKey := strings.TrimSpace(slices[2]) + + var sourceType string + + switch typeString { + case "config-map": + sourceType = "ConfigMap" + envVarSource.ConfigMapKeyRef = &corev1.ConfigMapKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: sourceName, + }, + Key: sourceKey} + + if !referencedConfigMaps.Has(sourceName) { + referencedConfigMaps.Insert(sourceName) + } + case "secret": + sourceType = "Secret" + envVarSource.SecretKeyRef = &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: sourceName, + }, + Key: sourceKey} + + if !referencedSecrets.Has(sourceName) { + referencedSecrets.Insert(sourceName) + } + default: + // following error message can be used when we enable ConfigMaps: + // "unsupported env source type \"%q\"; supported source types are \"configMap\" or \"secret\"" + return nil, fmt.Errorf("unsupported env source type \"%q\"; supported source types is \"secret\"", slices[0]) + } + + if len(sourceName) == 0 { + return nil, fmt.Errorf("the name of %s cannot be an empty string", sourceType) + } + + if len(sourceKey) == 0 { + return nil, fmt.Errorf("the key referenced by resource %s \"%s\" cannot be an empty string", sourceType, sourceName) } + + return &envVarSource, nil } var evRegex = regexp.MustCompile(`^{{\s*(\w+)\s*.(\w+)\s*}}$`) @@ -180,46 +395,122 @@ const ( valIdx = 2 ) -func processValue(val string) (string, error) { +func processLocalEnvValue(val string) (string, error) { match := evRegex.FindStringSubmatch(val) if len(match) > valIdx { if match[ctxIdx] != "env" { - return "", fmt.Errorf("uknown context %q", match[ctxIdx]) + return "", fmt.Errorf("allowed env value entry is \"{{ env.LOCAL_VALUE }}\"1; got: %q", match[ctxIdx]) } if v, ok := os.LookupEnv(match[valIdx]); ok { return v, nil } else { - return "", fmt.Errorf("required environment variable %q is not set", match[valIdx]) + return "", fmt.Errorf("required local environment variable \"%q\" is not set", match[valIdx]) } } else { return val, nil } } -func setEnv(service *servingv1.Service, env map[string]string) (*servingv1.Service, error) { - builtEnvName := "BUILT" - builtEnvValue := time.Now().Format("20060102T150405") +/// processVolumes generates Volumes and VolumeMounts from a function config +// volumes: +// - secret: example-secret # mount Secret as Volume +// path: /etc/secret-volume +// - configMap: example-cm # mount ConfigMap as Volume +// path: /etc/cm-volume +func processVolumes(volumes fn.Volumes, referencedSecrets, referencedConfigMaps *sets.String) ([]corev1.Volume, []corev1.VolumeMount, error) { - toUpdate := make(map[string]string, len(env)+1) - toRemove := make([]string, 0) + createdVolumes := sets.NewString() + usedPaths := sets.NewString() - for name, value := range env { - if strings.HasSuffix(name, "-") { - toRemove = append(toRemove, strings.TrimSuffix(name, "-")) - } else { - toUpdate[name] = value + newVolumes := []corev1.Volume{} + newVolumeMounts := []corev1.VolumeMount{} + + for _, vol := range volumes { + + volumeName := "" + + if vol.Secret != nil { + volumeName = "secret-" + *vol.Secret + + if !createdVolumes.Has(volumeName) { + newVolumes = append(newVolumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: *vol.Secret, + }, + }, + }) + createdVolumes.Insert(volumeName) + + if !referencedSecrets.Has(*vol.Secret) { + referencedSecrets.Insert(*vol.Secret) + } + } + } else if vol.ConfigMap != nil { + volumeName = "config-map-" + *vol.ConfigMap + + if !createdVolumes.Has(volumeName) { + newVolumes = append(newVolumes, corev1.Volume{ + Name: volumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: *vol.ConfigMap, + }, + }, + }, + }) + createdVolumes.Insert(volumeName) + + if !referencedConfigMaps.Has(*vol.ConfigMap) { + referencedConfigMaps.Insert(*vol.ConfigMap) + } + } + } + + if volumeName != "" { + if !usedPaths.Has(*vol.Path) { + newVolumeMounts = append(newVolumeMounts, corev1.VolumeMount{ + Name: volumeName, + MountPath: *vol.Path, + }) + usedPaths.Insert(*vol.Path) + } else { + return nil, nil, fmt.Errorf("mount path %s is defined multiple times", *vol.Path) + } } } - toUpdate[builtEnvName] = builtEnvValue + return newVolumes, newVolumeMounts, nil +} + +// checkSecretsConfigMapsArePresent returns error if Secrets or ConfigMaps +// referenced in input sets are not deployed on the cluster in the specified namespace +func checkSecretsConfigMapsArePresent(ctx context.Context, namespace string, referencedSecrets, referencedConfigMaps *sets.String) error { + client, err := NewKubernetesClientset(namespace) + if err != nil { + return err + } + + errMsg := "" + for s := range *referencedSecrets { + _, err := client.CoreV1().Secrets(namespace).Get(ctx, s, metav1.GetOptions{}) + if err != nil { + errMsg += fmt.Sprintf(" referenced Secret \"%s\" is not present in namespace \"%s\"\n", s, namespace) + } + } - for idx, val := range toUpdate { - v, err := processValue(val) + for cm := range *referencedConfigMaps { + _, err := client.CoreV1().ConfigMaps(namespace).Get(ctx, cm, metav1.GetOptions{}) if err != nil { - return nil, err + errMsg += fmt.Sprintf(" referenced ConfigMap \"%s\" is not present in namespace \"%s\"\n", cm, namespace) } - toUpdate[idx] = v } - return service, flags.UpdateEnvVars(&service.Spec.Template.Spec.PodSpec, toUpdate, toRemove) + if errMsg != "" { + return fmt.Errorf("\n"+errMsg) + } + + return nil } diff --git a/knative/deployer_test.go b/knative/deployer_test.go index 693dc9304c..71a4548a07 100644 --- a/knative/deployer_test.go +++ b/knative/deployer_test.go @@ -37,7 +37,7 @@ func Test_processValue(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := processValue(tt.arg) + got, err := processLocalEnvValue(tt.arg) if (err != nil) != tt.wantErr { t.Errorf("processValue() error = %v, wantErr %v", err, tt.wantErr) return