feat: Edge Processor

ansible/playbooks/microk8s.yml

@@ -12,5 +12,6 @@
       openebs: true
   tasks:
     - include_tasks: ../tasks/mk8s/install_mk8s.yml
+    - include_tasks: ../tasks/mk8s/deploy_secrets.yml
     - include_tasks: ../tasks/mk8s/install_helm_repo.yml
     - include_tasks: ../tasks/mk8s/deploy_app.yml

ansible/playbooks/microk8s_ha.yml

@@ -12,6 +12,7 @@
       openebs: true
   tasks:
     - include_tasks: ../tasks/mk8s/install_mk8s.yml
+    - include_tasks: ../tasks/mk8s/deploy_secrets.yml
     - include_tasks: ../tasks/mk8s/install_helm_repo.yml
     - include_tasks: ../tasks/mk8s/deploy_app.yml
     - include_tasks: ../tasks/mk8s/get_registration_token.yml

ansible/tasks/mk8s/deploy_secrets.yml

@@ -0,0 +1,25 @@
+---
+- name: Load k8s secrets
+  include_vars:
+    file: "{{ item }}"
+  with_first_found:
+    - files:
+      - /opt/ansible/resources/k8s_secrets.yaml
+      - /opt/charts/splunk-connect-for-syslog/secrets.yaml
+
+- name: Create mTLS secret
+  ansible.builtin.shell: |
+    microk8s kubectl apply -f - <<EOF
+    apiVersion: v1
+    kind: Secret
+    metadata:
+      name: {{ hec_tls.secret }}
+    type: Opaque
+    data:
+      key.pem: {{ hec_tls.value.key | b64encode }}
+      cert.pem: {{ hec_tls.value.cert | b64encode }}
+      ca_cert.pem: {{ hec_tls.value.ca | b64encode }}
+    EOF
+  when:
+    - hec_tls is defined
+    - ('secret' in hec_tls) and ('value' in hec_tls)

charts/splunk-connect-for-syslog/templates/statefulset.yaml

@@ -65,6 +65,10 @@ spec:
           - name: SC4S_DEST_SPLUNK_HEC_DEFAULT_DISKBUFF_ENABLE
             value: "no"
           {{- end }}
+          {{- if .Values.splunk.hec_tls }}
+          - name: SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT
+            value: "/etc/syslog-ng/tls/hec"
+          {{- end }}
           {{- if .Values.sc4s.existingCert }}
           - name: SC4S_SOURCE_TLS_ENABLE
             value: "yes"
@@ -186,6 +190,11 @@ spec:
           - name: data
             mountPath: /var/lib/syslog-ng/
           {{- end }}
+          {{- if .Values.splunk.hec_tls }}
+          - name: hec-tls
+            mountPath: /etc/syslog-ng/tls/hec/
+            readOnly: true
+          {{- end }}
           {{- if .Values.sc4s.existingCert }}
           - name: tls
             mountPath: /etc/syslog-ng/tls/
@@ -224,6 +233,11 @@ spec:
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       volumes:
+        {{- if .Values.splunk.hec_tls }}
+        - name: hec-tls
+          secret:
+            secretName: {{ .Values.splunk.hec_tls }}
+        {{- end }}
         {{- if .Values.sc4s.existingCert }}
         - name: tls
           secret:

docs/edge_processor.md

@@ -0,0 +1,113 @@
+# Edge Processor integration guide (Experimental)
+
+## Intro
+
+You can use the `Edge Processor` to:
+
+* Enrich log messages with extra data, such as adding a new field or overriding an index using `SPL2`.
+* Filter log messages using `SPL2`.
+* Send log messages to alternative destinations, for example, `AWS S3` or `Apache Kafka`.
+
+## How it works
+
+```mermaid
+stateDiagram
+    direction LR
+
+    SC4S: SC4S
+    EP: Edge Processor
+    Dest: Another destination
+    Device: Your device
+    S3: AWS S3
+    Instance: Instance
+    Pipeline: Pipeline with SPL2
+
+    Device --> SC4S: Syslog protocol
+    SC4S --> EP: HEC
+    state EP {
+      direction LR
+      Instance --> Pipeline
+    }
+    EP --> Splunk
+    EP --> S3
+    EP --> Dest
+```
+
+## Set up the Edge Processor for SC4S
+
+### Set up on Docker / Podman
+
+1. On the `env_file`, configure the HEC URL as IP of *managed instance*, that you registered on Edge Processor.
+2.  Add your HEC token. You can find your token in the Edge Processor "global settings" page. 
+
+```
+SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=http://x.x.x.x:8088
+SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx
+```
+
+### Kubernetes
+
+1. Set up the Edge Processor on your `values.yaml` HEC URL using the IP of *managed instance*, that you registered on Edge Processor.
+
+2. Provide the hec_token. You can find this token on the Edge Processor's "global settings" page.
+
+```
+splunk:
+  hec_url: "http://x.x.x.x:8088"
+  hec_token: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+```
+
+## mTLS encryption
+
+### Prepare your certificates
+
+Before setup, [generate mTLS certificates](https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/EdgeProcessor/SecureForwarders). Server mTLS certificates should be uploaded to `Edge Processor` and client certifcates should be used with `SC4S`.
+
+Rename the certificate files. SC4S requires the following names:
+
+  * `key.pem` - client certificate key
+  * `cert.pem` - client certificate
+  * `ca_cert.pem` - certificate authority
+
+### Docker / Podman
+
+  1. Use HTTPS in HEC url: `SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://x.x.x.x:8088`.
+  2. Move your clients mTLS certificates to `/opt/sc4s/tls/hec`.
+  3. Mount `/opt/sc4s/tls/hec` to `/etc/syslog-ng/tls/hec` using docker/podman volumes.
+  4. Define mounting mTLS point for HEC: `SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT=/etc/syslog-ng/tls/hec`.
+  5. Start or restart SC4S.
+
+### Kubernetes
+
+  1. Add the secret name of the mTLS certificates to the `values.yaml` file:
+
+```
+splunk:
+  hec_url: "https://x.x.x.x:8088"
+  hec_token: "xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+  hec_tls: "hec-tls-secret"
+```
+
+  2. Add your mTLS certificates to the `charts/splunk-connect-for-syslog/secrets.yaml` file:
+
+```
+hec_tls:
+  secret: "hec-tls-secret"
+  value:
+    key: |
+      -----BEGIN PRIVATE KEY-----
+      Exmaple key
+      -----END PRIVATE KEY-----
+    cert: |
+      -----BEGIN CERTIFICATE-----
+      Exmaple cert
+      -----END CERTIFICATE-----
+    ca: |
+      -----BEGIN CERTIFICATE-----
+      Example ca
+      -----END CERTIFICATE-----
+```
+
+  3. Encrypt your `secrets.yaml` using `ansible-vault encrypt charts/splunk-connect-for-syslog/secrets.yaml`.
+  4. Add the IP address for your cluster nodes to the inventory file `ansible/inventory/inventory_microk8s_ha.yaml`.
+  5. Deploy the Ansible playbook `ansible-playbook -i ansible/inventory/inventory_microk8s_ha.yaml ansible/playbooks/microk8s_ha.yml --ask-vault-pass`

mkdocs.yml

@@ -14,6 +14,11 @@ markdown_extensions:
   - sane_lists
   - codehilite
   - pymdownx.snippets
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
 
 
 theme:
@@ -57,6 +62,7 @@ nav:
   - SC4S Lite (Experimental):
       - Intro: "lite.md"
       - Pluggable modules: "pluggable_modules.md"
+  - Edge Processor (Experimental): "edge_processor.md"
   - Troubleshooting:
       - SC4S Startup and Validation: "troubleshooting/troubleshoot_SC4S_server.md"
       - SC4S Logging and Troubleshooting Resources: "troubleshooting/troubleshoot_resources.md"

package/etc/conf.d/conflib/_common/t_templates.conf

@@ -138,7 +138,7 @@ template t_JSON_5424_SDATA {
 
 template t_splunk_hec {
     template('$(format-json
-        time=$(if ("${.netsource.sc4s_use_recv_time}" eq "yes") "$R_UNIXTIME" "$S_UNIXTIME")
+        time=$(if ("${.netsource.sc4s_use_recv_time}" eq "yes") ${R_UNIXTIME} ${S_UNIXTIME})
         host=$(lowercase ${HOST})
         source=${.splunk.source:-SC4S}
         sourcetype=${.splunk.sourcetype:-sc4s:fallback}

package/etc/conf.d/destinations/dest_hec/plugin.jinja

@@ -34,6 +34,11 @@ destination d_hec{{ dest_mode }}{{ altname }}{
     {%- endif %}
         tls(
             peer-verify({{ peer_verify }})
+        {% if tls_mount %}
+            key-file("{{ tls_mount }}/key.pem")
+            cert-file("{{ tls_mount }}/cert.pem")
+            ca-dir("{{ tls_mount }}")
+        {% endif %}
         {%- if cipher_suite %}
             cipher-suite("{{ cipher_suite }}")
         {%- endif %}

package/etc/conf.d/destinations/dest_hec/plugin.py

@@ -144,6 +144,7 @@ def hec_endpoint_collector(hec_path, url_hec):
             f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_DISKBUFSIZE",
             int(disk_space / workers),
         ),
+        tls_mount=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TLS_MOUNT"),
         peer_verify=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TLS_VERIFY", "yes"),
         cipher_suite=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_CIPHER_SUITE"),
         ssl_version=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_SSL_VERSION"),

package/sbin/entrypoint.sh

@@ -170,18 +170,25 @@ if [ "$SC4S_DEST_SPLUNK_HEC_GLOBAL" != "no" ]
 then
   HEC=$(echo $SC4S_DEST_SPLUNK_HEC_DEFAULT_URL | cut -d' ' -f 1)
   if [ "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY}" == "no" ]; then export NO_VERIFY=-k ; fi
+
+  if [ -n "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}" ]; then
+    export HEC_TLS_OPTS="--cert ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/cert.pem  --key ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/key.pem --cacert ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_MOUNT}/ca_cert.pem";
+  else
+    export HEC_TLS_OPTS="";
+  fi
+
   SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=$(grep -Po '(?<=^splunk_sc4s_fallback,index,).*' -m1 $SC4S_ETC/conf.d/local/context/splunk_metadata.csv )
   export SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX:=main}
   SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=$(cat $SC4S_ETC/conf.d/local/context/splunk_metadata.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3)
   export SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX:=main}
 
-  if curl -s -S ${NO_VERIFY} "${HEC}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d "{\"event\": \"HEC TEST EVENT\", \"sourcetype\": \"sc4s:probe\", \"index\": \"${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}\"}" 2>&1 | grep -v '{"text":"Success"'
+  if curl -s -S ${NO_VERIFY} "${HEC}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d "{\"event\": \"HEC TEST EVENT\", \"sourcetype\": \"sc4s:probe\", \"index\": \"${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}\"}" ${HEC_TLS_OPTS} 2>&1 | grep -v -e '{"text":"Success"' -e '{"text": "Success"'
   then
     echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}. sourcetype=sc4s:fallback\nStartup will continue to prevent data loss if this is a transient failure."
     echo ""
   else
     echo -e "SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX} for sourcetype=sc4s:fallback..."
-    if curl -s -S ${NO_VERIFY} "${HEC}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d "{\"event\": \"HEC TEST EVENT\", \"sourcetype\": \"sc4s:probe\", \"index\": \"${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}\"}" 2>&1 | grep -v '{"text":"Success"'
+    if curl -s -S ${NO_VERIFY} "${HEC}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d "{\"event\": \"HEC TEST EVENT\", \"sourcetype\": \"sc4s:probe\", \"index\": \"${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}\"}" ${HEC_TLS_OPTS} 2>&1 | grep -v -e '{"text":"Success"' -e '{"text": "Success"'
       then
         echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue for index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}. sourcetype=sc4s:events \nStartup will continue to prevent data loss if this is a transient failure."
         echo ""