Add support for using envFrom to load environment variables into Cohe…

…rence Pods (#626)

.github/workflows/coherence-matrix.yaml

@@ -35,8 +35,8 @@ jobs:
         matrixName:
           - "15.1.1-0-SNAPSHOT"
           - "15.1.1-0-SNAPSHOT-Graal"
-          - "23.03"
-          - "23.03-Graal"
+          - "23.09"
+          - "23.09-Graal"
           - "22.06"
           - "14.1.1-2206-SNAPSHOT"
           - "14.1.1-0-SNAPSHOT"
@@ -57,27 +57,27 @@ jobs:
             javaVersion: 17
             baseImage: "gcr.io/distroless/java17-debian11"
 
-          - matrixName: "23.03"
-            coherenceVersion: "23.03.1"
-            coherenceImage: "ghcr.io/oracle/coherence-ce:23.03.1"
+          - matrixName: "23.09"
+            coherenceVersion: "23.09"
+            coherenceImage: "ghcr.io/oracle/coherence-ce:23.09"
             javaVersion: 17
             baseImage: "gcr.io/distroless/java17-debian11"
 
-          - matrixName: "23.03-Graal"
-            coherenceVersion: "23.03.1"
-            coherenceImage: "ghcr.io/oracle/coherence-ce:23.03.1-graal"
+          - matrixName: "23.09-Graal"
+            coherenceVersion: "23.09"
+            coherenceImage: "ghcr.io/oracle/coherence-ce:23.09-graal"
             javaVersion: 17
             baseImage: "gcr.io/distroless/java17-debian11"
 
           - matrixName: "22.06"
-            coherenceVersion: "22.06.5"
-            coherenceImage: "ghcr.io/oracle/coherence-ce:22.06.5"
+            coherenceVersion: "22.06.6"
+            coherenceImage: "ghcr.io/oracle/coherence-ce:22.06.6"
             javaVersion: 11
             baseImage: "gcr.io/distroless/java11-debian11"
 
           - matrixName: "14.1.1-2206-SNAPSHOT"
-            coherenceVersion: "14.1.1-2206-6-SNAPSHOT"
-            coherenceImage: "iad.ocir.io/odx-stateservice/test/coherence:14.1.1-2206-6-SNAPSHOT"
+            coherenceVersion: "14.1.1-2206-7-SNAPSHOT"
+            coherenceImage: "iad.ocir.io/odx-stateservice/test/coherence:14.1.1-2206-7-SNAPSHOT"
             javaVersion: 11
             baseImage: "gcr.io/distroless/java11-debian11"
 
@@ -88,9 +88,9 @@ jobs:
             baseImage: "gcr.io/distroless/java11-debian11"
 
           - matrixName: "14.1.1-0"
-            coherenceVersion: "14.1.1-0-14"
+            coherenceVersion: "14.1.1-0-15"
             javaVersion: 8
-            coherenceImage: "ghcr.io/oracle/coherence-ce:14.1.1-0-14"
+            coherenceImage: "ghcr.io/oracle/coherence-ce:14.1.1-0-15"
             baseImage: "gcr.io/distroless/java11-debian11"
 
           - matrixName: "14.1.1.0.0"
@@ -106,8 +106,8 @@ jobs:
             baseImage: "gcr.io/distroless/java11-debian11"
 
           - matrixName: "12.2.1-4-SNAPSHOT"
-            coherenceVersion: "12.2.1-4-19-SNAPSHOT"
-            coherenceImage: "iad.ocir.io/odx-stateservice/test/coherence:12.2.1-4-19-SNAPSHOT"
+            coherenceVersion: "12.2.1-4-20-SNAPSHOT"
+            coherenceImage: "iad.ocir.io/odx-stateservice/test/coherence:12.2.1-4-20-SNAPSHOT"
             javaVersion: 8
             baseImage: "gcr.io/distroless/java11-debian11"
 

.github/workflows/compatibility-tests.yaml

@@ -37,35 +37,40 @@ jobs:
       fail-fast: false
       matrix:
         compatibilityVersion:
+        - 3.3.0
         - 3.2.11
         - 3.2.10
         - 3.2.9
         - 3.2.7
         - 3.2.6
         - 3.2.5
         include:
+        - compatibilityVersion: 3.3.0
+          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.6"
+          compatibilitySelector: control-plane=coherence
+          k8s: kindest/node:v1.27.3@sha256:3966ac761ae0136263ffdb6cfd4db23ef8a83cba8a463690e98317add2c9ba72
         - compatibilityVersion: 3.2.11
-          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.3"
+          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.6"
           compatibilitySelector: control-plane=coherence
-          k8s: kindest/node:v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f
+          k8s: kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb
         - compatibilityVersion: 3.2.10
-          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.3"
+          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.6"
           compatibilitySelector: control-plane=coherence
-          k8s: kindest/node:v1.26.3@sha256:61b92f38dff6ccc29969e7aa154d34e38b89443af1a2c14e6cfbd2df6419c66f
+          k8s: kindest/node:v1.26.6@sha256:6e2d8b28a5b601defe327b98bd1c2d1930b49e5d8c512e1895099e4504007adb
         - compatibilityVersion: 3.2.9
-          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.3"
+          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.6"
           compatibilitySelector: control-plane=coherence
           k8s: kindest/node:v1.24.12@sha256:1e12918b8bc3d4253bc08f640a231bb0d3b2c5a9b28aa3f2ca1aee93e1e8db16
         - compatibilityVersion: 3.2.7
-          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.3"
+          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.6"
           compatibilitySelector: control-plane=coherence
           k8s: kindest/node:v1.24.12@sha256:1e12918b8bc3d4253bc08f640a231bb0d3b2c5a9b28aa3f2ca1aee93e1e8db16
         - compatibilityVersion: 3.2.6
-          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.3"
+          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.6"
           compatibilitySelector: control-plane=coherence
           k8s: kindest/node:v1.24.12@sha256:1e12918b8bc3d4253bc08f640a231bb0d3b2c5a9b28aa3f2ca1aee93e1e8db16
         - compatibilityVersion: 3.2.5
-          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.3"
+          coherence-image: "ghcr.io/oracle/coherence-ce:22.06.6"
           compatibilitySelector: control-plane=coherence
           k8s: kindest/node:v1.24.12@sha256:1e12918b8bc3d4253bc08f640a231bb0d3b2c5a9b28aa3f2ca1aee93e1e8db16
 

Makefile

@@ -19,13 +19,13 @@ VERSION ?= 3.3.1
 MVN_VERSION ?= $(VERSION)
 
 # The version number to be replaced by this release
-PREV_VERSION ?= 3.2.11
+PREV_VERSION ?= 3.3.0
 
 # The operator version to use to run certification tests against
 CERTIFICATION_VERSION ?= $(VERSION)
 
 # The previous Operator version used to run the compatibility tests.
-COMPATIBLE_VERSION  ?= 3.2.11
+COMPATIBLE_VERSION  ?= 3.3.0
 # The selector to use to find Operator Pods of the COMPATIBLE_VERSION (do not put in double quotes!!)
 COMPATIBLE_SELECTOR = control-plane=coherence
 
@@ -42,7 +42,7 @@ COHERENCE_VERSION ?= 21.12.5
 # The default Coherence image the Operator will run if no image is specified
 COHERENCE_IMAGE_REGISTRY ?= ghcr.io/oracle
 COHERENCE_IMAGE_NAME     ?= coherence-ce
-COHERENCE_IMAGE_TAG      ?= 22.06.5
+COHERENCE_IMAGE_TAG      ?= 22.06.6
 COHERENCE_IMAGE          ?= $(COHERENCE_IMAGE_REGISTRY)/$(COHERENCE_IMAGE_NAME):$(COHERENCE_IMAGE_TAG)
 # The Java version that tests will be compiled to.
 # This should match the version required by the COHERENCE_IMAGE version

api/v1/coherencejobresource_types.go

@@ -62,6 +62,13 @@ func (in *CoherenceJob) IsForceExit() bool {
 	return in.Spec.ForceExit != nil && *in.Spec.ForceExit
 }
 
+func (in *CoherenceJob) GetEnvVarFrom() []corev1.EnvFromSource {
+	if in == nil {
+		return make([]corev1.EnvFromSource, 0)
+	}
+	return in.Spec.EnvFrom
+}
+
 // GetSpec returns this resource's CoherenceResourceSpec
 func (in *CoherenceJob) GetSpec() *CoherenceResourceSpec {
 	return &in.Spec.CoherenceResourceSpec
@@ -384,6 +391,14 @@ type CoherenceJobResourceSpec struct {
 	// ForceExit is a flag to indicate whether the Operator should call System.exit to forcefully exit the process
 	// when the configured main class completes execution.
 	ForceExit *bool `json:"forceExit,omitempty"`
+	// List of sources to populate environment variables in the container.
+	// The keys defined within a source must be a C_IDENTIFIER. All invalid keys
+	// will be reported as an event when the container is starting. When a key exists in multiple
+	// sources, the value associated with the last source will take precedence.
+	// Values defined by an Env with a duplicate key will take precedence.
+	// Cannot be updated.
+	// +optional
+	EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
 }
 
 // GetRestartPolicy returns the name of the application image to use

api/v1/coherenceresource.go

@@ -7,6 +7,7 @@
 package v1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -87,4 +88,6 @@ type CoherenceResource interface {
 	GetAPIVersion() string
 	// IsForceExit is a flag to determine whether the Operator calls System.exit when the main class finishes.
 	IsForceExit() bool
+	// GetEnvVarFrom returns the array of EnvVarSource configurations
+	GetEnvVarFrom() []corev1.EnvFromSource
 }

api/v1/coherenceresource_types.go

@@ -179,6 +179,13 @@ func (in *Coherence) IsForceExit() bool {
 	return false
 }
 
+func (in *Coherence) GetEnvVarFrom() []corev1.EnvFromSource {
+	if in == nil {
+		return make([]corev1.EnvFromSource, 0)
+	}
+	return in.Spec.EnvFrom
+}
+
 // FindFullyQualifiedPortServiceNames returns a map of the exposed ports of this resource mapped to their Service's
 // fully qualified domain name.
 func (in *Coherence) FindFullyQualifiedPortServiceNames() map[string]string {
@@ -416,6 +423,14 @@ type CoherenceStatefulSetResourceSpec struct {
 	// Actions to execute once all the Pods are ready after an initial deployment
 	// +optional
 	Actions []Action `json:"actions,omitempty"`
+	// List of sources to populate environment variables in the container.
+	// The keys defined within a source must be a C_IDENTIFIER. All invalid keys
+	// will be reported as an event when the container is starting. When a key exists in multiple
+	// sources, the value associated with the last source will take precedence.
+	// Values defined by an Env with a duplicate key will take precedence.
+	// Cannot be updated.
+	// +optional
+	EnvFrom []corev1.EnvFromSource `json:"envFrom,omitempty"`
 }
 
 // CreateStatefulSetResource creates the deployment's StatefulSet resource.

api/v1/coherenceresourcespec_types.go

@@ -725,6 +725,8 @@ func (in *CoherenceResourceSpec) CreateCoherenceContainer(deployment CoherenceRe
 		VolumeMounts:    vm,
 	}
 
+	c.EnvFrom = deployment.GetEnvVarFrom()
+
 	if in.ImagePullPolicy != nil {
 		c.ImagePullPolicy = *in.ImagePullPolicy
 	}

api/v1/common_test.go

@@ -600,6 +600,37 @@ func addEnvVarsToContainer(c *corev1.Container, envVars ...corev1.EnvVar) {
 	}
 }
 
+func addEnvVarsFrom(sts *appsv1.StatefulSet, containerName string, envVars ...corev1.EnvFromSource) {
+	if sts != nil {
+		addEnvVarsFromToPodSpec(&sts.Spec.Template, containerName, envVars...)
+	}
+}
+
+func addEnvVarsFromToJob(job *batchv1.Job, containerName string, envVars ...corev1.EnvFromSource) {
+	if job != nil {
+		addEnvVarsFromToPodSpec(&job.Spec.Template, containerName, envVars...)
+	}
+}
+
+func addEnvVarsFromToPodSpec(template *corev1.PodTemplateSpec, containerName string, envVars ...corev1.EnvFromSource) {
+	for i, c := range template.Spec.InitContainers {
+		if c.Name == containerName {
+			addEnvVarsFromToContainer(&c, envVars...)
+			template.Spec.InitContainers[i] = c
+		}
+	}
+	for i, c := range template.Spec.Containers {
+		if c.Name == containerName {
+			addEnvVarsFromToContainer(&c, envVars...)
+			template.Spec.Containers[i] = c
+		}
+	}
+}
+
+func addEnvVarsFromToContainer(c *corev1.Container, envVars ...corev1.EnvFromSource) {
+	c.EnvFrom = append(c.EnvFrom, envVars...)
+}
+
 func addPorts(sts *appsv1.StatefulSet, containerName string, ports ...corev1.ContainerPort) {
 	if sts != nil {
 		addPortsToPodSpec(&sts.Spec.Template, containerName, ports...)

api/v1/create_job_test.go

@@ -8,6 +8,7 @@ package v1_test
 
 import (
 	coh "github.com/oracle/coherence-operator/api/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/pointer"
 	"testing"
 )
@@ -103,3 +104,35 @@ func TestCreateJobWithReplicasAndSyncedCompletionsOverride(t *testing.T) {
 	// assert that the Job is as expected
 	assertJobCreation(t, deployment, expected)
 }
+
+func TestCreateJobWithEnvVarsFrom(t *testing.T) {
+	cm := corev1.ConfigMapEnvSource{
+		LocalObjectReference: corev1.LocalObjectReference{
+			Name: "test-vars",
+		},
+		Optional: pointer.Bool(true),
+	}
+
+	var from []corev1.EnvFromSource
+	from = append(from, corev1.EnvFromSource{
+		Prefix:       "foo",
+		ConfigMapRef: &cm,
+	})
+
+	spec := coh.CoherenceJobResourceSpec{
+		CoherenceResourceSpec: coh.CoherenceResourceSpec{
+			Env: []corev1.EnvVar{},
+		},
+		EnvFrom: from,
+	}
+
+	// Create the test deployment
+	deployment := createTestCoherenceJobDeployment(spec)
+	// Create expected StatefulSet
+	expected := createMinimalExpectedJob(deployment)
+
+	addEnvVarsFromToJob(expected, coh.ContainerNameCoherence, from...)
+
+	// assert that the StatefulSet is as expected
+	assertJobCreation(t, deployment, expected)
+}

api/v1/create_statefulset_test.go

@@ -128,6 +128,38 @@ func TestCreateStatefulSetWithEmptyEnvVars(t *testing.T) {
 	assertStatefulSetCreation(t, deployment, stsExpected)
 }
 
+func TestCreateStatefulSetWithEnvVarsFrom(t *testing.T) {
+	cm := corev1.ConfigMapEnvSource{
+		LocalObjectReference: corev1.LocalObjectReference{
+			Name: "test-vars",
+		},
+		Optional: pointer.Bool(true),
+	}
+
+	var from []corev1.EnvFromSource
+	from = append(from, corev1.EnvFromSource{
+		Prefix:       "foo",
+		ConfigMapRef: &cm,
+	})
+
+	spec := coh.CoherenceStatefulSetResourceSpec{
+		CoherenceResourceSpec: coh.CoherenceResourceSpec{
+			Env: []corev1.EnvVar{},
+		},
+		EnvFrom: from,
+	}
+
+	// Create the test deployment
+	deployment := createTestCoherenceDeployment(spec)
+	// Create expected StatefulSet
+	stsExpected := createMinimalExpectedStatefulSet(deployment)
+
+	addEnvVarsFrom(stsExpected, coh.ContainerNameCoherence, from...)
+
+	// assert that the StatefulSet is as expected
+	assertStatefulSetCreation(t, deployment, stsExpected)
+}
+
 func TestCreateStatefulSetWithHealthPort(t *testing.T) {
 	// create a spec with a custom health port
 	spec := coh.CoherenceResourceSpec{

api/v1/hasher_test.go

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates.
+ * Licensed under the Universal Permissive License v 1.0 as shown at
+ * http://oss.oracle.com/licenses/upl.
+ */
+
+package v1_test
+
+import (
+	. "github.com/onsi/gomega"
+	coh "github.com/oracle/coherence-operator/api/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"testing"
+)
+
+func TestHash(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	spec := coh.CoherenceStatefulSetResourceSpec{}
+
+	deployment := &coh.Coherence{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "test-ns",
+			Name:      "test",
+		},
+		Spec: spec,
+	}
+
+	coh.EnsureHashLabel(deployment)
+
+	g.Expect(deployment.GetLabels()["coherence-hash"]).To(Equal("5cb9fd9f96"))
+}