[#4167] Kafka certificate authentication (#4168)

airyhq · May 6, 2024 · 2efde81 · 2efde81
1 parent 2f052cd
commit 2efde81
Show file tree

Hide file tree

Showing 23 changed files with 270 additions and 7 deletions.
diff --git a/backend/components/chat-plugin/helm/templates/backend/deployment.yaml b/backend/components/chat-plugin/helm/templates/backend/deployment.yaml
@@ -51,6 +51,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.backend.resources | indent 10 }}
       initContainers:
@@ -68,3 +73,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/contacts/helm/templates/deployment.yaml b/backend/components/contacts/helm/templates/deployment.yaml
@@ -45,6 +45,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
       initContainers:
@@ -62,3 +67,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/facebook/helm/templates/deployments.yaml b/backend/components/facebook/helm/templates/deployments.yaml
@@ -59,6 +59,11 @@ spec:
               - name: Health-Check
                 value: health-check
             initialDelaySeconds: 120
+{{ if .Values.global.kafkaCertAuth }}
+          volumeMounts:
+          - name: kafka-config-certs
+            mountPath: /opt/kafka/certs
+{{ end }}
           resources:
 {{ toYaml .Values.connector.resources | indent 12 }}
       initContainers:
@@ -76,6 +81,11 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -124,6 +134,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.eventsRouter.resources | indent 10 }}
       initContainers:
@@ -141,3 +156,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/google/helm/templates/deployments.yaml b/backend/components/google/helm/templates/deployments.yaml
@@ -54,6 +54,11 @@ spec:
               - name: Health-Check
                 value: health-check
             initialDelaySeconds: 120
+{{ if .Values.global.kafkaCertAuth }}
+          volumeMounts:
+          - name: kafka-config-certs
+            mountPath: /opt/kafka/certs
+{{ end }}
           resources:
 {{ toYaml .Values.connector.resources | indent 12 }}
       initContainers:
@@ -71,6 +76,11 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -122,6 +132,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.eventsRouter.resources | indent 10 }}
       initContainers:
@@ -139,3 +154,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/media-resolver/helm/templates/deployment.yaml b/backend/components/media-resolver/helm/templates/deployment.yaml
@@ -45,6 +45,11 @@ spec:
             initialDelaySeconds: 120
             periodSeconds: 10
             failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+          volumeMounts:
+          - name: kafka-config-certs
+            mountPath: /opt/kafka/certs
+{{ end }}
           resources:
 {{ toYaml .Values.resources | indent 12 }}
       initContainers:
@@ -62,3 +67,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/sources-api/helm/templates/deployment.yaml b/backend/components/sources-api/helm/templates/deployment.yaml
@@ -50,6 +50,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
       initContainers:
@@ -67,3 +72,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/streams/helm/templates/deployment.yaml b/backend/components/streams/helm/templates/deployment.yaml
@@ -48,6 +48,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.resources | indent 10 }}
       initContainers:
@@ -67,3 +72,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/twilio/helm/templates/deployments.yaml b/backend/components/twilio/helm/templates/deployments.yaml
@@ -54,6 +54,11 @@ spec:
               - name: Health-Check
                 value: health-check
             initialDelaySeconds: 120
+{{ if .Values.global.kafkaCertAuth }}
+          volumeMounts:
+          - name: kafka-config-certs
+            mountPath: /opt/kafka/certs
+{{ end }}
           resources:
 {{ toYaml .Values.connector.resources | indent 12 }}
       initContainers:
@@ -141,3 +146,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/viber/helm/templates/deployments.yaml b/backend/components/viber/helm/templates/deployments.yaml
@@ -49,6 +49,11 @@ spec:
                 - name: Health-Check
                   value: health-check
             initialDelaySeconds: 120
+{{ if .Values.global.kafkaCertAuth }}
+          volumeMounts:
+          - name: kafka-config-certs
+            mountPath: /opt/kafka/certs
+{{ end }}
           resources:
 {{ toYaml .Values.connector.resources | indent 12 }}
       initContainers:
@@ -66,3 +71,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/webhook/helm/templates/deployments.yaml b/backend/components/webhook/helm/templates/deployments.yaml
@@ -55,6 +55,11 @@ spec:
               - name: Health-Check
                 value: health-check
           initialDelaySeconds: 120
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.consumer.resources | indent 10 }}
       initContainers:
@@ -157,3 +162,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/backend/components/whatsapp/helm/templates/deployments.yaml b/backend/components/whatsapp/helm/templates/deployments.yaml
@@ -113,6 +113,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.eventsRouter.resources | indent 10 }}
       initContainers:
@@ -130,3 +135,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/docs/docs/getting-started/installation/helm.md b/docs/docs/getting-started/installation/helm.md
@@ -290,6 +290,39 @@ Run the following command to create the `Airy` platform without the bundled inst
 helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --values ./airy.yaml
 ```
 
+#### Confluent
+
+To connect to a Kafka instance in Confluent cloud, settings the `config.kafka.brokers` and `config.kafka.aurhJaas` is enough, prior to deploying the Helm chart.
+
+#### Aiven
+
+Aiven cloud uses a keystore and truststore certificates that need to be loaded on the workloads that are connecting to Kafka. Get the necessary certificates and connection files from Aiven using the `avn` CLI and place them in a separate directory.
+
+```
+avn service user-kafka-java-creds {KAFKA_INSTANCE} --username {USERNAME} -d ./aiven/ --password {PASSWORD}
+```
+
+Create a Kubernetes ConfigMap that contains the contents of the created directory:
+
+```
+kubectl create configmap kafka-config-certs --from-file aiven/
+```
+
+Set the connection appropriate parameters in your `airy.yaml` file:
+
+```yaml
+config:
+  kafka:
+    brokers: "the-aiven-kafka-broker-url"
+    keyTrustSecret: "the-key-trust-secret"
+```
+
+Then install Airy with the following command:
+
+```sh
+helm install airy airy/airy --timeout 10m --set prerequisites.kafka.enabled=false --set global.kafkaCertAuth=true --values ./airy.yaml
+```
+
 ### Kafka partitions per topic
 
 Currently all the default topics in the Airy instance are created with 10 partitions. To create these topics with a different number of partitions, add the following to your `airy.yaml` file before running `helm install` (before the initial creation of the topics):

diff --git a/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml b/infrastructure/helm-chart/templates/components/api-admin/deployment.yaml
@@ -60,6 +60,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.components.api.admin.resources | indent 10 }}
       initContainers:
@@ -77,3 +82,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml b/infrastructure/helm-chart/templates/components/api-communication/deployment.yaml
@@ -45,6 +45,11 @@ spec:
           initialDelaySeconds: 120
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.components.api.communication.resources | indent 10 }}
       initContainers:
@@ -62,3 +67,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}
diff --git a/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml b/infrastructure/helm-chart/templates/components/api-components-installer/deployment.yaml
@@ -81,6 +81,11 @@ spec:
           initialDelaySeconds: 60
           periodSeconds: 10
           failureThreshold: 3
+{{ if .Values.global.kafkaCertAuth }}
+        volumeMounts:
+        - name: kafka-config-certs
+          mountPath: /opt/kafka/certs
+{{ end }}
         resources:
 {{ toYaml .Values.components.api.components.installer.resources | indent 10 }}
       initContainers:
@@ -102,4 +107,8 @@ spec:
         - name: provisioning-scripts
           configMap:
             name: provisioning-scripts
-
+{{ if .Values.global.kafkaCertAuth }}
+        - name: kafka-config-certs
+          configMap:
+            name: kafka-config-certs
+{{ end }}