Wire bundled otel-collector to monoscope via in-cluster agent

- monoscope-k8s/values-agent.yaml: enable ClusterIP service for the
  daemonset (internalTrafficPolicy: Local) so other in-cluster
  collectors can forward OTLP to a stable DNS name; drop the explicit
  k8s_attributes block in favour of the chart's preset (richer
  metadata, per-node filtering); add resources block so GOMEMLIMIT
  activates.
- monoscope-k8s/values-cluster.yaml: drop the explicit k8s_attributes
  block (same reason); add resources block.
- monoscope-k8s/otel-demo-overlay.yaml: minimal additive overlay
  (mode: deployment + one otlp/monoscope-agent forward exporter).
  No secrets, no x-api-key — auth lives entirely in monoscope-agent.
  Logs intentionally NOT forwarded here to avoid double-counting
  filelog.
- Makefile: add k8s-apply-monoscope, k8s-apply-otel-demo-overlay,
  k8s-delete-monoscope targets.

Author: tonyalaribe

Makefile

@@ -255,3 +255,59 @@ endif
 .PHONY: build-react-native-android
 build-react-native-android:
 	$(DOCKER_CMD) build -f src/react-native-app/android.Dockerfile --platform=linux/amd64 --output=. src/react-native-app
+
+# --- Monoscope Kubernetes deployment ---------------------------------------
+# Apply the Monoscope OTel collector Helm releases (DaemonSet agent + cluster
+# Deployment) defined in monoscope-k8s/. The MONOSCOPE_API_KEY must be set in
+# the environment — it's stored in a k8s Secret, never written to disk.
+#
+# Example:
+#   MONOSCOPE_API_KEY=xxx make k8s-apply-monoscope
+
+K8S_NAMESPACE ?= default
+
+.PHONY: k8s-apply-monoscope
+k8s-apply-monoscope:
+ifndef MONOSCOPE_API_KEY
+	$(error MONOSCOPE_API_KEY is not set. Export it or pass MONOSCOPE_API_KEY=... to make)
+endif
+	@echo "→ Adding/updating open-telemetry helm repo"
+	@helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts >/dev/null 2>&1 || true
+	@helm repo update open-telemetry >/dev/null
+	@echo "→ Upserting monoscope-secrets in namespace $(K8S_NAMESPACE)"
+	@kubectl create secret generic monoscope-secrets \
+	  --namespace $(K8S_NAMESPACE) \
+	  --from-literal=api-key='$(MONOSCOPE_API_KEY)' \
+	  --dry-run=client -o yaml | kubectl apply -f -
+	@echo "→ Installing/upgrading agent (DaemonSet)"
+	helm upgrade --install monoscope-agent open-telemetry/opentelemetry-collector \
+	  --namespace $(K8S_NAMESPACE) --values monoscope-k8s/values-agent.yaml
+	@echo "→ Installing/upgrading cluster collector (Deployment)"
+	helm upgrade --install monoscope-cluster open-telemetry/opentelemetry-collector \
+	  --namespace $(K8S_NAMESPACE) --values monoscope-k8s/values-cluster.yaml
+	@echo ""
+	@echo "Monoscope collectors deployed. Verify with:"
+	@echo "  kubectl get pods -l 'app.kubernetes.io/instance in (monoscope-agent,monoscope-cluster)'"
+	@echo "  monoscope events search 'resource.k8s.cluster.uid != \"\"' --since 5m --limit 1"
+
+.PHONY: k8s-delete-monoscope
+k8s-delete-monoscope:
+	-helm uninstall monoscope-agent --namespace $(K8S_NAMESPACE)
+	-helm uninstall monoscope-cluster --namespace $(K8S_NAMESPACE)
+	-kubectl delete secret monoscope-secrets --namespace $(K8S_NAMESPACE)
+
+# Patch the otel-demo helm release so its bundled otel-collector fans
+# traces/metrics/logs out to monoscope alongside the in-cluster sinks.
+# Requires the monoscope-secrets secret (run `make k8s-apply-monoscope` first).
+.PHONY: k8s-apply-otel-demo-overlay
+k8s-apply-otel-demo-overlay:
+	@helm get values otel-demo --namespace $(K8S_NAMESPACE) >/dev/null 2>&1 || \
+	  (echo "otel-demo release not found in $(K8S_NAMESPACE); install it first" && exit 1)
+	@kubectl get secret monoscope-secrets --namespace $(K8S_NAMESPACE) >/dev/null 2>&1 || \
+	  (echo "monoscope-secrets not found; run 'make k8s-apply-monoscope' first" && exit 1)
+	@echo "Note: --reset-values clears any stale monoscope-related values"
+	@echo "from prior overlay revisions; the chart's vanilla defaults plus"
+	@echo "monoscope-k8s/otel-demo-overlay.yaml become the new state."
+	helm upgrade otel-demo open-telemetry/opentelemetry-demo \
+	  --namespace $(K8S_NAMESPACE) --reset-values \
+	  --values monoscope-k8s/otel-demo-overlay.yaml

monoscope-k8s/otel-demo-overlay.yaml

@@ -0,0 +1,47 @@
+# Overlay applied on top of the otel-demo chart values to fan the bundled
+# otel-collector's traces / metrics / logs out to the in-cluster
+# monoscope-agent collector — which is the component that holds the API
+# key secret and forwards to monoscope.tech. The bundled collector itself
+# stays vanilla: no secrets, no x-api-key, no auth concerns.
+#
+# Apply:
+#   helm upgrade otel-demo open-telemetry/opentelemetry-demo \
+#     --reset-values --values monoscope-k8s/otel-demo-overlay.yaml
+#
+# (--reset-values clears any previous user-supplied values so the bundled
+# collector returns to chart defaults plus only what's added here.)
+
+opentelemetry-collector:
+  # Run the bundled collector as a singleton Deployment instead of the chart's
+  # default DaemonSet. The chart-default daemonset binds host ports (jaeger
+  # receivers on 6831/14250/14268) that would collide with the monoscope-agent
+  # daemonset. A singleton has no host-level needs for this demo.
+  mode: deployment
+
+  config:
+    exporters:
+      # In-cluster forward to the monoscope-agent DaemonSet. The agent has
+      # internalTrafficPolicy: Local, so OTLP requests stay on-node when
+      # possible. The agent attaches x-api-key and exports to monoscope.tech.
+      otlp/monoscope-agent:
+        endpoint: monoscope-agent-opentelemetry-collector.default.svc.cluster.local:4317
+        tls:
+          insecure: true
+
+    service:
+      pipelines:
+        # Append otlp/monoscope-agent to each pipeline's existing exporters.
+        # The chart merges receivers/processors but replaces pipeline arrays,
+        # so we re-list the chart-default exporters here verbatim. If the
+        # chart updates its defaults, this list must be re-synced (check
+        # `helm show values open-telemetry/opentelemetry-demo`).
+        traces:
+          exporters: [otlp/jaeger, debug, spanmetrics, otlp/monoscope-agent]
+        metrics:
+          exporters: [otlphttp/prometheus, debug, otlp/monoscope-agent]
+        # Logs intentionally NOT forwarded to monoscope-agent here — the agent
+        # already captures every container's stdout via filelog from
+        # /var/log/pods. Forwarding OTLP logs too would double-count any app
+        # that uses both stdout and the OTel logger SDK (e.g. cart).
+        logs:
+          exporters: [opensearch, debug]

monoscope-k8s/values-agent.yaml

@@ -1,16 +1,37 @@
 mode: daemonset
 
+# Daemonset mode skips a ClusterIP Service by default (each node runs its own
+# pod). Enable it so other in-cluster collectors / apps can forward OTLP to a
+# stable DNS name. internalTrafficPolicy: Local keeps requests on the same
+# node when possible (saves cross-node hops for app-to-agent traffic).
+service:
+  enabled: true
+  internalTrafficPolicy: Local
+
 image:
   repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s
   tag: "0.149.0"
 
+# Pod-level limits — required so the chart's GOMEMLIMIT helper activates.
+# Without limits.memory, GOMEMLIMIT is silently disabled and the Go runtime
+# can spike past memory_limiter under burst load. Tune for your workload.
+resources:
+  requests:
+    cpu: 100m
+    memory: 256Mi
+  limits:
+    memory: 512Mi
+
 presets:
   logsCollection:
     enabled: true
   kubeletMetrics:
     enabled: true
   kubernetesAttributes:
-    enabled: true
+    enabled: true          # Provides RBAC + a rich metadata extractor (workload kinds,
+                           # container.image.*, k8s.cluster.uid, service.* from annotations).
+                           # Pipelines reference it as `k8sattributes` (the chart still uses
+                           # the deprecated alias internally — known chart noise, harmless).
 
 extraEnvs:
   - name: MONOSCOPE_API_KEY
@@ -38,17 +59,6 @@ config:
       check_interval: 1s
       limit_mib: 4000
       spike_limit_mib: 800
-    k8s_attributes:
-      auth_type: serviceAccount
-      passthrough: false
-      extract:
-        metadata:
-          - k8s.pod.name
-          - k8s.pod.uid
-          - k8s.deployment.name
-          - k8s.namespace.name
-          - k8s.node.name
-          - k8s.container.name
     resource:
       attributes:
         - key: x-api-key
@@ -66,13 +76,13 @@ config:
     pipelines:
       traces:
         receivers: [otlp]
-        processors: [k8s_attributes, memory_limiter, batch, resource]
+        processors: [k8sattributes, memory_limiter, batch, resource]
         exporters: [otlp_grpc]
       metrics:
         receivers: [otlp, kubeletstats]
-        processors: [k8s_attributes, memory_limiter, batch, resource]
+        processors: [k8sattributes, memory_limiter, batch, resource]
         exporters: [otlp_grpc]
       logs:
         receivers: [otlp, filelog]
-        processors: [k8s_attributes, memory_limiter, batch, resource]
+        processors: [k8sattributes, memory_limiter, batch, resource]
         exporters: [otlp_grpc]

monoscope-k8s/values-cluster.yaml

@@ -1,17 +1,27 @@
 mode: deployment
+# Must stay 1 — k8s_events and k8s_cluster don't leader-elect, so >1 replica
+# produces duplicate events and cluster metrics.
 replicaCount: 1
 
 image:
   repository: ghcr.io/open-telemetry/opentelemetry-collector-releases/opentelemetry-collector-k8s
   tag: "0.149.0"
 
+# See agent values for the GOMEMLIMIT linkage.
+resources:
+  requests:
+    cpu: 50m
+    memory: 128Mi
+  limits:
+    memory: 256Mi
+
 presets:
   clusterMetrics:
     enabled: true
   kubernetesEvents:
     enabled: true
   kubernetesAttributes:
-    enabled: true
+    enabled: true          # See agent values for the rationale.
 
 extraEnvs:
   - name: MONOSCOPE_API_KEY
@@ -37,9 +47,6 @@ config:
       check_interval: 1s
       limit_mib: 1000
       spike_limit_mib: 200
-    k8s_attributes:
-      auth_type: serviceAccount
-      passthrough: false
     resource:
       attributes:
         - key: x-api-key
@@ -57,9 +64,9 @@ config:
     pipelines:
       metrics:
         receivers: [k8s_cluster]
-        processors: [k8s_attributes, memory_limiter, batch, resource]
+        processors: [k8sattributes, memory_limiter, batch, resource]
         exporters: [otlp_grpc]
       logs:
         receivers: [k8s_events]
-        processors: [k8s_attributes, memory_limiter, batch, resource]
+        processors: [k8sattributes, memory_limiter, batch, resource]
         exporters: [otlp_grpc]