feat: Edge Processor MVP

charts/splunk-connect-for-syslog/templates/statefulset.yaml

@@ -72,6 +72,14 @@ spec:
           - name: SC4S_SOURCE_TLS_ENABLE
             value: "no"
           {{- end }}
+          {{- if .Values.splunk.hec_tls }}
+          - name: SC4S_DEST_TLS_MOUNT
+            value: "/etc/syslog-ng/tls/hec/"
+          {{- end }}
+          {{- if .Values.splunk.hec_template }}
+          - name: SC4S_HEC_TEMPLATE
+            value: "{{ .Values.splunk.hec_template }}"
+          {{- end }}
           {{- if .Values.sc4s }}
           {{- if .Values.sc4s.vendor_product }}
             {{- range $vp := .Values.sc4s.vendor_product }}
@@ -191,6 +199,11 @@ spec:
             mountPath: /etc/syslog-ng/tls/
             readOnly: true
           {{- end }}
+          {{- if .Values.splunk.hec_tls }}
+          - name: hec-tls
+            mountPath: /etc/syslog-ng/tls/hec/
+            readOnly: true
+          {{- end }}
           {{- if .Values.sc4s.context_files }}
           - name: context
             mountPath: /etc/syslog-ng/conf.d/configmap/context/
@@ -228,6 +241,11 @@ spec:
         - name: tls
           secret:
             secretName: {{ .Values.sc4s.existingCert }}
+        {{- end }}
+        {{- if .Values.splunk.hec_tls }}
+        - name: hec-tls
+          secret:
+            secretName: {{ .Values.splunk.hec_tls }}
         {{- end }}               
         {{- if .Values.sc4s.context_files }}
         - name: context

charts/splunk-connect-for-syslog/values.yaml

@@ -9,6 +9,8 @@ splunk:
 #   hec_url: ""
 #   hec_token: ""
 #   hec_verify_tls: "yes"
+#   hec_tls: "hec_tls"
+#   hec_template: "t_splunk_hec"
 
 sc4s: 
   {}

docs/edge_processor.md

@@ -0,0 +1,18 @@
+# SC4S + EP guide (Experimental)
+
+## Basic Setup:
+
+* Use IP of EP instance as HEC URL: `SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=http://x.x.x.x:8088`
+* Use token from EP Global Settings: `SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=secret`
+* Use EP API format: `SC4S_HEC_TEMPLATE=t_edge_hec`
+
+## TLS:
+
+* Switch to HTTPS at HEC: `SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=https://x.x.x.x:8088`
+* [Generate certificates](https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/EdgeProcessor/SecureForwarders)
+* Upload certs at Edge Processor TLS settings
+* Rename `Client cert` to `cert.pem`
+* Rename `Client key` to `key.pem`
+* Rename `CA Cert` to `ca_cert.pem`
+* Mount dir with certs to `/etc/syslog-ng/tls/hec`
+* Set path for TLS dir: `SC4S_DEST_TLS_MOUNT=/etc/syslog-ng/tls/hec`

mkdocs.yml

@@ -57,6 +57,7 @@ nav:
   - SC4S Lite (Experimental):
       - Intro: "lite.md"
       - Pluggable modules: "pluggable_modules.md"
+  - Edge Processor (Experimental): "edge_processor.md"
   - Troubleshooting:
       - SC4S Startup and Validation: "troubleshooting/troubleshoot_SC4S_server.md"
       - SC4S Logging and Troubleshooting Resources: "troubleshooting/troubleshoot_resources.md"

package/etc/conf.d/conflib/_common/t_templates.conf

@@ -149,6 +149,19 @@ template t_splunk_hec {
  )');
 };
 
+template t_edge_hec {
+    template('$(format-json
+        time=$(if ("${.netsource.sc4s_use_recv_time}" eq "yes") ${R_UNIXTIME} ${S_UNIXTIME})
+        host=$(lowercase ${HOST})
+        source=${.splunk.source:-SC4S}
+        sourcetype=${.splunk.sourcetype:-sc4s:fallback}
+        index=${.splunk.index:-main}
+        event="$(template ${.splunk.sc4s_template:-t_hdr_msg})"
+        fields.sc4s_tags="$(implode "|" $(explode "," "$TAGS"))"
+        fields.*
+ )');
+};
+
 
 
 template t_fallback_kv {

package/etc/conf.d/destinations/dest_hec/plugin.jinja

@@ -34,6 +34,11 @@ destination d_hec{{ dest_mode }}{{ altname }}{
     {%- endif %}
         tls(
             peer-verify({{ peer_verify }})
+        {% if tls_mount %}
+            key-file("{{ tls_mount }}/key.pem")
+            cert-file("{{ tls_mount }}/cert.pem")
+            ca-dir("{{ tls_mount }}")
+        {% endif %}
         {%- if cipher_suite %}
             cipher-suite("{{ cipher_suite }}")
         {%- endif %}

package/etc/conf.d/destinations/dest_hec/plugin.py

@@ -21,7 +21,8 @@ def hec_endpoint_collector(hec_path, url_hec):
 )
 tm = templateEnv.get_template("plugin.jinja")
 
-msg_template = "$(template ${.splunk.sc4s_hec_template} $(template t_splunk_hec))"
+t_hec_template = os.getenv("SC4S_HEC_TEMPLATE", "t_splunk_hec")
+msg_template = "$(template ${.splunk.sc4s_hec_template} $(template %s))" % t_hec_template
 dest_mode = "_fmt"
 dests = []
 
@@ -144,6 +145,7 @@ def hec_endpoint_collector(hec_path, url_hec):
             f"SC4S_DEST_SPLUNK_HEC_{group}_DISKBUFF_DISKBUFSIZE",
             int(disk_space / workers),
         ),
+        tls_mount=os.getenv("SC4S_DEST_TLS_MOUNT"),
         peer_verify=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_TLS_VERIFY", "yes"),
         cipher_suite=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_CIPHER_SUITE"),
         ssl_version=os.getenv(f"SC4S_DEST_SPLUNK_HEC_{group}_SSL_VERSION"),

package/sbin/entrypoint.sh

@@ -170,18 +170,25 @@ if [ "$SC4S_DEST_SPLUNK_HEC_GLOBAL" != "no" ]
 then
   HEC=$(echo $SC4S_DEST_SPLUNK_HEC_DEFAULT_URL | cut -d' ' -f 1)
   if [ "${SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY}" == "no" ]; then export NO_VERIFY=-k ; fi
+
+  if [ -n "${SC4S_DEST_TLS_MOUNT}" ]; then
+    export HEC_TLS_OPTS="--cert ${SC4S_DEST_TLS_MOUNT}/cert.pem  --key ${SC4S_DEST_TLS_MOUNT}/key.pem --cacert ${SC4S_DEST_TLS_MOUNT}/ca_cert.pem";
+  else
+    export HEC_TLS_OPTS="";
+  fi
+
   SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=$(grep -Po '(?<=^splunk_sc4s_fallback,index,).*' -m1 $SC4S_ETC/conf.d/local/context/splunk_metadata.csv )
   export SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX:=main}
   SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=$(cat $SC4S_ETC/conf.d/local/context/splunk_metadata.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3)
   export SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX:=main}
 
-  if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v '{"text":"Success"'
+  if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' ${HEC_TLS_OPTS} 2>&1 | grep -v -e '{"text":"Success"' -e '{"text": "Success"'
   then
     echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}. sourcetype=sc4s:fallback\nStartup will continue to prevent data loss if this is a transient failure."
     echo ""
   else
     echo -e "SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX} for sourcetype=sc4s:fallback..."
-    if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v '{"text":"Success"'
+    if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' ${HEC_TLS_OPTS} 2>&1 | grep -v -e '{"text":"Success"' -e '{"text": "Success"'
       then
         echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue for index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}. sourcetype=sc4s:events \nStartup will continue to prevent data loss if this is a transient failure."
         echo ""