otel-agent: support Deployment kind, routing directly to existing otel-collector (#863)

ref sourcegraph/sourcegraph#12299
ref https://sourcegraph.slack.com/archives/C05MW2TMYAV/p1778179723106999

## Summary

Added `openTelemetry.agent.kind` value (`"DaemonSet"` | `"Deployment"`,
default `"DaemonSet"`) for environments where DaemonSets are restricted
(e.g. some customer limitation)

When `kind: "Deployment"`, the otel-agent DaemonSet is not created and
application pods send traces directly to the existing `otel-collector`
Deployment via `http://otel-collector:4317` (the pre-existing ClusterIP
service with gRPC OTLP on port 4317)

## Test plan

kind cluster with below override. tested and it worked

```yml
storageClass:
  create: false
  name: standard

sourcegraph: 
  localDevMode: true
  image:
    repository: us-docker.pkg.dev/sourcegraph-images/external
    defaultTag: docker-images-notest-michaellz_370952_2026-05-08_7.2-48f70aff43d4 #370941_2026-05-07_7.2-2c8911ef83a9
    useGlobalTagAsDefault: true

openTelemetry:
  enabled: true
  agent:
    kind: Deployment

jaeger:
  enabled: true
```

there's no more `otel-agent` ds:

<img width="740" height="92" alt="CleanShot 2026-05-07 at 18 51 42"
src="https://github.com/user-attachments/assets/544ee8e1-5353-433e-835a-caecb0286cd4"
/>

can view the trace in jager:

<img width="1400" height="976" alt="CleanShot 2026-05-07 at 18 49 50"
src="https://github.com/user-attachments/assets/42de66af-62b4-4ce4-ae27-7a3635d94e1c"
/>


<img width="1448" height="833" alt="CleanShot 2026-05-07 at 18 49 31"
src="https://github.com/user-attachments/assets/d0652be5-13ca-4495-bd0f-3dab7664a4b1"
/>

---
_Generated by [Claude
Code](https://claude.ai/code/session_01K2AnXLduzruntPtUsC7xiA)_

---------

Co-authored-by: Claude <noreply@anthropic.com>

Author: michaellzc

charts/sourcegraph/README.md

@@ -203,6 +203,7 @@ In addition to the documented values, all services also support the following va
 | openTelemetry.agent.containerSecurityContext.runAsGroup | int | `101` |  |
 | openTelemetry.agent.containerSecurityContext.runAsUser | int | `100` |  |
 | openTelemetry.agent.hostPorts | object | `{"grpcOtlp":4317,"httpOtlp":4318,"httpZpages":55679}` | Resource requests & limits for the `otel-agent` container, learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/) |
+| openTelemetry.agent.kind | string | `"DaemonSet"` | The workload kind for the otel-agent. Valid values are `DaemonSet` and `Deployment`. Use `Deployment` in environments where DaemonSets are restricted (e.g. some managed Kubernetes offerings). When set to `Deployment`, the otel-agent DaemonSet is not created and application pods send traces directly to the existing otel-collector Deployment (otel-collector:4317) instead. |
 | openTelemetry.agent.name | string | `"otel-agent"` | Name used by resources. Does not affect service names or PVCs. |
 | openTelemetry.agent.resources.limits.cpu | string | `"500m"` |  |
 | openTelemetry.agent.resources.limits.memory | string | `"500Mi"` |  |

charts/sourcegraph/templates/_helpers.tpl

@@ -169,6 +169,10 @@ app.kubernetes.io/name: jaeger
 
 {{- define "sourcegraph.openTelemetryEnv" -}}
 {{- if .Values.openTelemetry.enabled -}}
+{{- if eq (.Values.openTelemetry.agent.kind | default "DaemonSet") "Deployment" -}}
+- name: OTEL_EXPORTER_OTLP_ENDPOINT
+  value: http://otel-collector:4317
+{{- else -}}
 # OTEL_AGENT_HOST must be defined before OTEL_EXPORTER_OTLP_ENDPOINT to substitute the node IP on which the DaemonSet pod instance runs in the latter variable
 - name: OTEL_AGENT_HOST
   valueFrom:
@@ -178,6 +182,7 @@ app.kubernetes.io/name: jaeger
   value: http://$(OTEL_AGENT_HOST):{{ toYaml .Values.openTelemetry.agent.hostPorts.grpcOtlp }}
 {{- end }}
 {{- end }}
+{{- end }}
 
 {{- define "sourcegraph.databaseAuth" -}}
 {{- $top := index . 0 -}}

charts/sourcegraph/templates/otel-collector/otel-agent.DaemonSet.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.openTelemetry.enabled -}}
+{{- if and .Values.openTelemetry.enabled (eq (.Values.openTelemetry.agent.kind | default "DaemonSet") "DaemonSet") -}}
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:

charts/sourcegraph/tests/otelAgentKind_test.yaml

@@ -0,0 +1,42 @@
+---
+suite: otelAgentKind
+tests:
+  - it: should render a DaemonSet by default
+    template: otel-collector/otel-agent.DaemonSet.yaml
+    asserts:
+      - isKind:
+          of: DaemonSet
+  - it: should render a DaemonSet when kind is DaemonSet
+    template: otel-collector/otel-agent.DaemonSet.yaml
+    set:
+      openTelemetry:
+        agent:
+          kind: "DaemonSet"
+    asserts:
+      - isKind:
+          of: DaemonSet
+  - it: should not render a DaemonSet when kind is Deployment
+    template: otel-collector/otel-agent.DaemonSet.yaml
+    set:
+      openTelemetry:
+        agent:
+          kind: "Deployment"
+    asserts:
+      - hasDocuments:
+          count: 0
+  - it: should set OTEL_EXPORTER_OTLP_ENDPOINT to otel-collector when kind is Deployment
+    template: searcher/searcher.StatefulSet.yaml
+    set:
+      openTelemetry:
+        agent:
+          kind: "Deployment"
+    asserts:
+      - contains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: OTEL_EXPORTER_OTLP_ENDPOINT
+            value: "http://otel-collector:4317"
+      - notContains:
+          path: spec.template.spec.containers[0].env
+          content:
+            name: OTEL_AGENT_HOST

charts/sourcegraph/values.yaml

@@ -658,6 +658,11 @@ openTelemetry:
   agent:
     # -- Name used by resources. Does not affect service names or PVCs.
     name: "otel-agent"
+    # -- The workload kind for the otel-agent. Valid values are `DaemonSet` and `Deployment`.
+    # Use `Deployment` in environments where DaemonSets are restricted (e.g. some managed Kubernetes offerings).
+    # When set to `Deployment`, the otel-agent DaemonSet is not created and application pods send
+    # traces directly to the existing otel-collector Deployment (otel-collector:4317) instead.
+    kind: "DaemonSet"
     # -- Resource requests & limits for the `otel-agent` container,
     # learn more from the [Kubernetes documentation](https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/)
     hostPorts: