Elastic, kibana and APM

README.md

@@ -83,3 +83,16 @@ task sandbox:destroy                  # Tear down sandbox
 ## Secret Management
 
 Secrets are managed with SOPS and AGE encryption. Templates in the infrastructure overlays use `@secret-gen:random:XX` annotations for automatic password generation. The sandbox uses a per-setup AGE key (`security/sandbox-key.txt`), while shared secrets (TLS certificates) use a developer AGE key distributed out-of-band.
+
+### APM Setup & Testing 
+
+1. Create a new APM user with their own dedicated APM Server, unique Secret Token, and isolated Kibana space.
+    `python3 provision_apm_user.py testprojectname`
+   This will deploy a new Kubernetes ApmServer resource named `apm-server-<username>` and provide a unique URL (e.g., `http://testprojectname-apm.sandbox.rijksapp.dev`) and Secret Token.
+
+2.
+   One-liner to run in an isolated environment:
+   ```bash
+   python3 -m venv venv && ./venv/bin/pip install elastic-apm && ELASTIC_APM_SERVER_URL=[url, see output from above] ELASTIC_APM_SECRET_TOKEN=[token, see output from above] ./venv/bin/python3 test_apm_lib.py
+   ```
+   User can then go to https://kibana.sandbox.rijksapp.dev/login and login with their username and password to see their data

Taskfile.yaml

@@ -1872,6 +1872,15 @@ tasks:
       - task: sandbox:sync
       # Phase 4: Bootstrap ArgoCD + Operations Manager (images pulled from GHCR)
       - task: prepare-argocd-operator
+      - |
+        kubectl create -f bootstrap/crd/elastic/crds.yaml
+        kubectl apply -f infrastructure/bootstrap/infrastructure/elastic/controller/operator.yaml
+
+        kustomize build infrastructure/bootstrap/infrastructure/elastic/controller/overlays/sandboxed-local | kubectl apply -f -
+      # hierna kan je inloggen op https://kibana.sandbox.rijksapp.dev met username elastic voor password zie volgende regel
+      # kubectl get secret elasticsearch-es-elastic-user -o=jsonpath='{.data.elastic}' | base64 --decode; echo
+
+
       - task: bootstrap-argo-system
         vars:
           KEY_FILE: security/sandbox-key.txt
@@ -2205,7 +2214,7 @@ tasks:
           grep -v '^[[:space:]]*#' "{{.ENV_FILE}}" | grep -v '^[[:space:]]*$' | while IFS='=' read -r key value; do
             value=$(echo "$value" | sed 's/^["'\'']//' | sed 's/["'\'']$//')
             # Replace the empty value in the template with the actual value
-            sed -i '' "s|$key: \"\"|$key: \"$value\"|" "$SECRET_FILE"
+            sed "s|$key: \"\"|$key: \"$value\"|" "$SECRET_FILE" > "$SECRET_FILE.tmp" && mv "$SECRET_FILE.tmp" "$SECRET_FILE"
             echo "  Set: $key"
           done
         else

infrastructure/bootstrap/infrastructure/elastic/controller/base/apm_server_template.yaml.tmp

@@ -0,0 +1,65 @@
+apiVersion: apm.k8s.elastic.co/v1
+kind: ApmServer
+metadata:
+  name: apm-server-{username}
+  namespace: rig-system
+spec:
+  version: 9.3.2
+  count: 1
+  elasticsearchRef:
+    name: elasticsearch
+  kibanaRef:
+    name: kibana
+  config:
+    output.elasticsearch:
+      username: "elastic"
+      password: "{elastic_password}"
+      indices:
+        - index: "traces-apm-{username}"
+          when.contains:
+            data_stream.type: "traces"
+        - index: "metrics-apm-{username}"
+          when.contains:
+            data_stream.type: "metrics"
+        - index: "logs-apm-{username}"
+          when.contains:
+            data_stream.type: "logs"
+    apm-server:
+      auth.secret_token: "{token}"
+      host: "0.0.0.0:8200"
+      data_streams.namespace: "{username}"
+  http:
+    service:
+      spec:
+        type: ClusterIP
+    tls:
+      selfSignedCertificate:
+        disabled: true
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: apm-server-{username}-token
+  namespace: rig-system
+stringData:
+  secret-token: "{token}"
+---
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: apm-server-{username}-ingress
+  namespace: rig-system
+  annotations:
+    nginx.ingress.kubernetes.io/ssl-redirect: "false"
+spec:
+  rules:
+  - host: {username}-apm.sandbox.rijksapp.dev
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: apm-server-{username}-apm-http
+            port:
+              number: 8200

infrastructure/bootstrap/infrastructure/elastic/controller/base/elasticsearch.yaml

@@ -0,0 +1,12 @@
+apiVersion: elasticsearch.k8s.elastic.co/v1
+kind: Elasticsearch
+metadata:
+  name: elasticsearch
+  namespace: rig-system
+spec:
+  version: 9.3.2
+  nodeSets:
+    - name: default
+      count: 1
+      config:
+        node.store.allow_mmap: false

infrastructure/bootstrap/infrastructure/elastic/controller/base/kibana.yaml

@@ -0,0 +1,14 @@
+apiVersion: kibana.k8s.elastic.co/v1
+kind: Kibana
+metadata:
+  name: kibana
+  namespace: rig-system
+spec:
+  version: 9.3.2
+  count: 1
+  elasticsearchRef:
+    name: elasticsearch
+  config:
+    xpack.fleet.packages:
+      - name: apm
+        version: latest

infrastructure/bootstrap/infrastructure/elastic/controller/base/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - elasticsearch.yaml
+  - kibana.yaml