feat: mTLS for Splunk 10 (#1197)

* feat: mTLS support for Splunk 10

* docs: fix typo

---------

Co-authored-by: Olga <[email protected]>
Co-authored-by: ajasnosz <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 ### Changed
+- implemented mTLS for Splunk 10
 
 ### Fixed
 

charts/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml

@@ -46,6 +46,14 @@ spec:
           env:
             {{- include "environmental-variables" . | nindent 12 }}
             {{- include "environmental-variables-sender" . | nindent 12 }}
+            {{- if .Values.splunk.mtls.enabled }}
+            - name: SPLUNK_HEC_MTLS_CLIENT_CERT
+              value: /app/hec-mtls/tls.crt
+            - name: SPLUNK_HEC_MTLS_CLIENT_KEY
+              value: /app/hec-mtls/tls.key
+            - name: SPLUNK_HEC_MTLS_CA_CERT
+              value: /app/hec-mtls/cacert.pem
+            {{- end }}
           {{- if .Values.worker.livenessProbe.enabled }}
           livenessProbe:
             exec:
@@ -63,6 +71,11 @@ spec:
             periodSeconds: {{ .Values.worker.readinessProbe.periodSeconds }}
           {{- end }}
           volumeMounts:
+            {{- if .Values.splunk.mtls.enabled }}
+            - name: hec-mtls
+              mountPath: "/app/hec-mtls"
+              readOnly: true
+            {{- end }}
             - name: config
               mountPath: "/app/config"
               readOnly: true
@@ -102,6 +115,11 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
+        {{- if .Values.splunk.mtls.enabled }}
+        - name: hec-mtls
+          secret:
+            secretName: {{ .Values.splunk.mtls.secretRef }}
+        {{- end }}
         # You set volumes at the Pod level, then mount them into containers inside that Pod
         - name: config
           configMap:

charts/splunk-connect-for-snmp/values.schema.json

@@ -185,6 +185,18 @@
         },
         "metricsIndex": {
           "type": "string"
+        },
+        "mtls": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "enabled": {
+              "type": "boolean"
+            },
+            "secretRef": {
+              "type": "string"
+            }
+          }
         }
       }
     },

charts/splunk-connect-for-snmp/values.yaml

@@ -93,6 +93,12 @@ splunk:
   # name of the metrics index
   metricsIndex: "netmetrics"
 
+  # new encryption appeared in Splunk 10
+  mtls:
+    enabled: false
+    # name of existing secret (should be created manully), that will store certs for mTLS
+    secretRef: ""
+
 ################################################################################
 # Splunk Observability configuration
 ################################################################################

docs/mtls.md

@@ -0,0 +1,127 @@
+# mTLS
+
+
+## Intro
+
+!!! info
+    mTLS encryption support is available beginning with Splunk 10
+
+Mutual TLS (mTLS) is an extension of the standard TLS protocol that provides mutual authentication between a client and a server. While TLS typically ensures that the client can verify the server’s identity, mTLS requires both parties to verify each other’s identities using digital certificates. In our case client is SC4SNMP and server is Splunk.
+
+
+## How to setup Splunk
+
+!!! info
+    If you are using Splunk Cloud, reach out to your administrator to configure mutual TLS (mTLS).
+
+1. Ensure that client and server mTLS certificates are already prepared
+
+2. Update `$SPLUNK_HOME/etc/system/local/server.conf`:
+
+```
+[sslConfig]
+requireClientCert = true
+[kvstore]
+hostnameOption = fullyqualifiedname
+```
+
+3. Update `$SPLUNK_HOME/etc/system/local/web.conf`:
+
+```
+[settings]
+sslPassword = password
+sslRootCAPath = cacert.pem
+enableSplunkWebSSL = true
+```
+
+4. Restart Splunk:
+
+```
+$SPLUNK_HOME/bin/splunk restart
+```
+
+## How to setup SC4SNMP
+
+/// tab | microk8s
+1. Add your **client** mTLS certificates to secrets:
+
+```
+microk8s kubectl create secret generic mtls -n sc4snmp \
+  --from-file=client.crt=./client.crt \
+  --from-file=client.key=./client.key \
+  --from-file=cacert.pem=./cacert.pem
+```
+
+2. Use https protocol to communicate with Splunk. To enforce this, set the `splunk.protocol` variable in the configuration file values.yaml:
+
+```
+splunk:
+    protocol: "https"
+```
+
+3. Add `mtls` section and provide your secret with certificates inside. To do this, update the `values.yaml` file under the splunk section as shown below:
+
+```
+splunk:
+    mtls:
+        enabled: true
+        secretRef: "mtls"
+```
+
+4. Redeploy SC4SNMP
+///
+
+/// tab | docker-compose
+1. Add your **client** mTLS certificates to secrets. To do this, update the docker-compose.yaml file by adding the following section at the end:
+
+```
+secrets:
+  cert:
+    file: client.crt
+  key:
+    file: client.key
+  ca:
+    file: cacert.pem
+```
+
+2. To provide the certificates to the `worker-sender` service, update its definition in the `docker-compose.yaml` file as shown below:
+
+```
+worker-sender:
+    environment:
+        SPLUNK_HEC_MTLS_CLIENT_CERT: /run/secrets/cert
+        SPLUNK_HEC_MTLS_CLIENT_KEY: /run/secrets/key
+        SPLUNK_HEC_MTLS_CA_CERT: /run/secrets/ca
+    secrets:
+        - cert
+        - key
+        - ca
+```
+3. Use https protocol to communicate with Splunk. To enforce this, set the `SPLUNK_HEC_PROTOCOL` variable in the configuration file `.env`:
+
+```
+SPLUNK_HEC_PROTOCOL=https
+```
+
+4. Redeploy SC4SNMP
+///
+
+
+## Troubleshooting
+
+1. Double-check that the mTLS certificates you are using are valid. To do this, send a test log message using `curl` in verbose mode, which can help identify any issues with the certificates:
+
+```
+curl -k https://${HEC_URL} \
+  -H "Authorization: Splunk ${HEC_TOKEN}" \
+  -H "Content-Type: application/json" \
+  -d '{"event": "Hello", "sourcetype": "manual", "host": "myhost", "source": "myapp"}' \
+  --cert client.crt \
+  --key client.key \
+  --cacert cacert.pem \
+  -vvv
+```
+
+2. Check logs of `worker-sender`. Refer to the instructions on how to configure logs for `kubernetes` or `docker` deployment.
+
+

mkdocs.yml

@@ -17,6 +17,8 @@ markdown_extensions:
   - admonition
   - pymdownx.details
   - pymdownx.superfences
+  - pymdownx.blocks.tab:
+      alternate_style: true
 
 plugins:
   - search:
@@ -96,6 +98,7 @@ nav:
   - Releases: "releases.md"
   - Request MIB: "mib-request.md"
   - Security: "security.md"
+  - mTLS: "mtls.md"
   - Troubleshooting:
       - Accessing and configuring logs: "troubleshooting/configuring-logs.md"
       - Docker commands: "troubleshooting/docker-commands.md"

splunk_connect_for_snmp/splunk/tasks.py

@@ -37,6 +37,9 @@
 SPLUNK_HEC_HOST = os.getenv("SPLUNK_HEC_HOST", "127.0.0.1")
 SPLUNK_HEC_PORT = os.getenv("SPLUNK_HEC_PORT", None)
 SPLUNK_HEC_PATH = os.getenv("SPLUNK_HEC_PATH", "services/collector")
+SPLUNK_HEC_MTLS_CLIENT_CERT = os.getenv("SPLUNK_HEC_MTLS_CLIENT_CERT", None)
+SPLUNK_HEC_MTLS_CLIENT_KEY = os.getenv("SPLUNK_HEC_MTLS_CLIENT_KEY", None)
+SPLUNK_HEC_MTLS_CA_CERT = os.getenv("SPLUNK_HEC_MTLS_CA_CERT", None)
 METRICS_INDEXING_ENABLED = human_bool(os.getenv("METRICS_INDEXING_ENABLED", "false"))
 
 url = {
@@ -101,10 +104,31 @@
 class HECTask(Task):
     def __init__(self):
         self.session = Session()
-        self.session.verify = SPLUNK_HEC_TLSVERIFY
+
+        # if we have CA cert for verifacation and we are not under insecureSSL mode
+        if (
+            SPLUNK_HEC_MTLS_CA_CERT is not None
+            and os.path.exists(SPLUNK_HEC_MTLS_CA_CERT)
+            and SPLUNK_HEC_TLSVERIFY
+        ):
+            self.session.verify = SPLUNK_HEC_MTLS_CA_CERT
+        else:
+            self.session.verify = SPLUNK_HEC_TLSVERIFY
+
         self.session.headers = SPLUNK_HEC_HEADERS
         self.session.logger = logger
 
+        if (
+            SPLUNK_HEC_MTLS_CLIENT_CERT is not None
+            and SPLUNK_HEC_MTLS_CLIENT_KEY is not None
+            and os.path.exists(SPLUNK_HEC_MTLS_CLIENT_CERT)
+            and os.path.exists(SPLUNK_HEC_MTLS_CLIENT_KEY)
+        ):
+            self.session.cert = (
+                SPLUNK_HEC_MTLS_CLIENT_CERT,
+                SPLUNK_HEC_MTLS_CLIENT_KEY,
+            )
+
 
 class PrepareTask(Task):
     def __init__(self):