feat: Edge Processor - Alpha version

charts/splunk-connect-for-syslog/templates/statefulset.yaml

@@ -56,6 +56,10 @@ spec:
                 key: hec_token
                 name: splunk-connection
           {{- end }}
+          {{- if .Values.splunk.hec_template }}
+          - name: SC4S_HEC_TEMPLATE
+            value: "{{ .Values.splunk.hec_template }}"
+          {{- end }}
           - name: SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY
             value: "{{ .Values.splunk.hec_verify_tls }}"
           {{- if .Values.persistence.enabled }}              

docs/edge_processor.md

@@ -0,0 +1,63 @@
+# Edge Processor integration guide (Experimental)
+
+## Intro
+
+`Edge Processor` can be used on that usecases:
+
+* Enrich log message extra data (for example add some field or override index) using `SPL2`
+* Filter log message using `SPL2`
+* Send log messages to alternative destanations (like `AWS S3`, `Apache Kafka`, etc.)
+
+## How it's working
+
+```mermaid
+stateDiagram
+    direction LR
+
+    SC4S: SC4S
+    EP: Edge Processor
+    Dest: Another destination
+    Device: Your device
+    S3: AWS S3
+    Instance: Instance
+    Pipeline: Pipeline with SPL2
+
+    Device --> SC4S: Syslog protocol
+    SC4S --> EP: HEC
+    state EP {
+      direction LR
+      Instance --> Pipeline
+    }
+    EP --> Splunk
+    EP --> S3
+    EP --> Dest
+```
+
+## Basic Setup
+
+ 1. Use IP of EP instance as HEC URL
+ 2. Use token from EP Global Settings
+ 3. Use EP API format: t_edge_hec
+
+### Docker:
+
+Update `env_file`:
+```
+SC4S_DEST_SPLUNK_HEC_DEFAULT_URL=http://x.x.x.x:8088
+SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=secret
+SC4S_HEC_TEMPLATE=t_edge_hec
+```
+
+### Kubernetes:
+
+Update `values.yaml`:
+```
+splunk:
+    hec_url: "http://x.x.x.x:8088"
+    hec_token: "secret"
+    hec_template: "t_edge_hec"
+```
+
+## TLS
+
+ Coming soon...

mkdocs.yml

@@ -14,6 +14,11 @@ markdown_extensions:
   - sane_lists
   - codehilite
   - pymdownx.snippets
+  - pymdownx.superfences:
+      custom_fences:
+        - name: mermaid
+          class: mermaid
+          format: !!python/name:pymdownx.superfences.fence_code_format
 
 
 theme:
@@ -57,6 +62,7 @@ nav:
   - SC4S Lite (Experimental):
       - Intro: "lite.md"
       - Pluggable modules: "pluggable_modules.md"
+  - Edge Processor (Experimental): "edge_processor.md"
   - Troubleshooting:
       - SC4S Startup and Validation: "troubleshooting/troubleshoot_SC4S_server.md"
       - SC4S Logging and Troubleshooting Resources: "troubleshooting/troubleshoot_resources.md"

package/etc/conf.d/conflib/_common/t_templates.conf

@@ -149,6 +149,19 @@ template t_splunk_hec {
  )');
 };
 
+template t_edge_hec {
+    template('$(format-json
+        time=${S_UNIXTIME}
+        host=$(lowercase ${HOST})
+        source=${.splunk.source:-SC4S}
+        sourcetype=${.splunk.sourcetype:-sc4s:fallback}
+        index=${.splunk.index:-main}
+        event="$(template ${.splunk.sc4s_template:-t_hdr_msg})"
+        fields.sc4s_tags="$(implode "|" $(explode "," "$TAGS"))"
+        fields.*
+ )');
+};
+
 
 
 template t_fallback_kv {

package/etc/conf.d/destinations/dest_hec/plugin.py

@@ -21,7 +21,8 @@ def hec_endpoint_collector(hec_path, url_hec):
 )
 tm = templateEnv.get_template("plugin.jinja")
 
-msg_template = "$(template ${.splunk.sc4s_hec_template} $(template t_splunk_hec))"
+t_hec_template = os.getenv("SC4S_HEC_TEMPLATE", "t_splunk_hec")
+msg_template = "$(template ${.splunk.sc4s_hec_template} $(template %s))" % t_hec_template
 dest_mode = "_fmt"
 dests = []
 

package/sbin/entrypoint.sh

@@ -175,13 +175,13 @@ then
   SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=$(cat $SC4S_ETC/conf.d/local/context/splunk_metadata.csv | grep ',index,' | grep sc4s_events | cut -d, -f 3)
   export SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX:=main}
 
-  if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v '{"text":"Success"'
+  if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v -e '{"text":"Success"' -e '{"text": "Success"'
   then
     echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX}. sourcetype=sc4s:fallback\nStartup will continue to prevent data loss if this is a transient failure."
     echo ""
   else
     echo -e "SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=${SC4S_DEST_SPLUNK_HEC_FALLBACK_INDEX} for sourcetype=sc4s:fallback..."
-    if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v '{"text":"Success"'
+    if curl -s -S ${NO_VERIFY} "${HEC}?/index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}" -H "Authorization: Splunk ${SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN}" -d '{"event": "HEC TEST EVENT", "sourcetype": "sc4s:probe"}' 2>&1 | grep -v -e '{"text":"Success"' -e '{"text": "Success"'
       then
         echo -e "SC4S_ENV_CHECK_HEC: Invalid Splunk HEC URL, invalid token, or other HEC connectivity issue for index=${SC4S_DEST_SPLUNK_HEC_EVENTS_INDEX}. sourcetype=sc4s:events \nStartup will continue to prevent data loss if this is a transient failure."
         echo ""