Local monitoring

charts/v2/airbyte/templates/airbyte-connector-builder-server/deployment.yaml

@@ -66,6 +66,7 @@ spec:
         {{- include "airbyte.connectorBuilder.envs" . | nindent 8 }}
         {{- include "airbyte.metrics.envs" . | nindent 8 }}
         {{- include "airbyte.tracking.envs" . | nindent 8 }}
+        {{- include "airbyte.datadog.envs" . | nindent 8 }}  # Add this line
 
         {{- if (eq .Values.global.edition "enterprise") }}
         {{- include "airbyte.enterprise.envs" . | nindent 8 }}
@@ -123,7 +124,7 @@ spec:
           containerPort: 8080
           protocol: TCP
         {{- if .Values.connectorBuilderServer.debug.enabled }}
-        - name: debug 
+        - name: debug
           containerPort: {{ .Values.connectorBuilderServer.debug.remoteDebugPort }}
           protocol: TCP
         {{- end}}

charts/v2/airbyte/templates/airbyte-connector-rollout-worker/deployment.yaml

@@ -73,6 +73,7 @@ spec:
 
         {{- include "airbyte.common.envs" . | nindent 8 }}
         {{- include "airbyte.logging.envs" . | nindent 8 }}
+        {{- include "airbyte.datadog.envs" . | nindent 8 }}
         {{- include "airbyte.metrics.envs" . | nindent 8 }}
         {{- include "airbyte.tracking.envs" . | nindent 8 }}
         {{- include "airbyte.temporal.envs" . | nindent 8 }}

charts/v2/airbyte/templates/airbyte-cron/deployment.yaml

@@ -59,13 +59,15 @@ spec:
           image: {{ include "imageUrl" (list .Values.cron.image $) }}
           imagePullPolicy: "{{ .Values.cron.image.pullPolicy }}"
           env:
+          # Add Datadog envs before other environment variables
+          {{- include "airbyte.datadog.envs" . | nindent 10 }}
+          {{- include "airbyte.metrics.envs" . | nindent 10 }}
 
           {{- include "airbyte.common.envs" . | nindent 10 }}
           {{- include "airbyte.cron.envs" . | nindent 10 }}
           {{- include "airbyte.database.envs" . | nindent 10 }}
           {{- include "airbyte.database.migrations.envs" . | nindent 10 }}
           {{- include "airbyte.featureFlags.envs" . | nindent 10 }}
-          {{- include "airbyte.metrics.envs" . | nindent 10 }}
           {{- include "airbyte.micronaut.envs" . | nindent 10 }}
           {{- include "airbyte.temporal.envs" . | nindent 10 }}
           {{- include "airbyte.tracking.envs" . | nindent 10 }}

charts/v2/airbyte/templates/airbyte-metrics/deployment.yaml

@@ -33,7 +33,7 @@ spec:
       {{- end }}
     spec:
       serviceAccountName: {{ .Values.global.serviceAccountName }}
-      {{- if .Values.global.imagePullSecrets }} 
+      {{- if .Values.global.imagePullSecrets }}
       imagePullSecrets:
         {{- range .Values.global.imagePullSecrets }}
           {{- printf "- name: %s" .name | nindent 8 }}
@@ -57,6 +57,7 @@ spec:
         {{- include "airbyte.database.envs" . | nindent 8 }}
         {{- include "airbyte.database.migrations.envs" . | nindent 8 }}
         {{- include "airbyte.metrics.envs" . | nindent 8 }}
+        {{- include "airbyte.datadog.envs" . | nindent 8 }}  # Add this line
 
         # Values from secret
         {{- if .Values.metrics.secrets }}

charts/v2/airbyte/templates/airbyte-server/deployment.yaml

@@ -71,14 +71,15 @@ spec:
           value: "Internal server"
         {{- end}}
 
-        {{- include "airbyte.auth.security.envs" . | nindent 8 }} 
-        {{- include "airbyte.common.envs" . | nindent 8 }} 
+        {{- include "airbyte.auth.security.envs" . | nindent 8 }}
+        {{- include "airbyte.common.envs" . | nindent 8 }}
+        {{- include "airbyte.metrics.envs" . | nindent 8 }}  # Add this line
+        {{- include "airbyte.datadog.envs" . | nindent 8 }}  # Add this line
         {{- include "airbyte.cluster.envs" . | nindent 8 }}
         {{- include "airbyte.connector.envs" . | nindent 8 }}
         {{- include "airbyte.database.envs" . | nindent 8 }}
         {{- include "airbyte.database.migrations.envs" . | nindent 8 }}
-        {{- include "airbyte.logging.envs" . | nindent 8 }} 
-        {{- include "airbyte.metrics.envs" . | nindent 8 }}
+        {{- include "airbyte.logging.envs" . | nindent 8 }}
         {{- include "airbyte.micronaut.envs" . | nindent 8 }}
         {{- include "airbyte.storage.envs" . | nindent 8 }}
         {{- include "airbyte.secretsManager.envs" . | nindent 8 }}
@@ -88,15 +89,15 @@ spec:
         {{- include "airbyte.workloads.resources.envs" . | nindent 8 }}
 
         {{- if or (eq .Values.global.edition "enterprise") (eq .Values.global.edition "pro") }}
-        {{- include "airbyte.auth.instanceAdmin.enterprise.envs" . | nindent 8 }} 
-        {{- include "airbyte.auth.identityProvider.envs" . | nindent 8 }} 
+        {{- include "airbyte.auth.instanceAdmin.enterprise.envs" . | nindent 8 }}
+        {{- include "airbyte.auth.identityProvider.envs" . | nindent 8 }}
         {{- include "airbyte.enterprise.envs" . | nindent 8 }}
         {{- include "airbyte.keycloak.envs" . | nindent 8 }}
         {{- include "airbyte.keycloak.admin.client.envs" . | nindent 8 }}
         {{- include "airbyte.keycloak.admin.user.envs" . | nindent 8 }}
         {{- else if (eq .Values.global.deploymentMode "oss") }}
-        {{- include "airbyte.auth.envs" . | nindent 8 }} 
-        {{- include "airbyte.auth.jwt.envs" . | nindent 8 }} 
+        {{- include "airbyte.auth.envs" . | nindent 8 }}
+        {{- include "airbyte.auth.jwt.envs" . | nindent 8 }}
         {{- else }}
         {{- include "airbyte.cloudApi.envs" . | nindent 8 }}
         {{- include "airbyte.keycloak.admin.client.envs" . | nindent 8 }}

charts/v2/airbyte/templates/airbyte-temporal/deployment.yaml

@@ -83,7 +83,11 @@ spec:
         {{- end }}
         {{- end }}
         ports:
-        - containerPort: 7233
+        - name: temporal  # Added name for the main temporal port
+          containerPort: 7233
+        # Add metrics port
+        - name: metrics
+          containerPort: 9090
         {{- if .Values.temporal.containerSecurityContext }}
         securityContext: {{- toYaml .Values.temporal.containerSecurityContext | nindent 10 }}
         {{- end }}

charts/v2/airbyte/templates/airbyte-temporal/service.yaml

@@ -11,6 +11,11 @@ spec:
   - port: {{ .Values.temporal.service.port }}
     protocol: TCP
     targetPort: 7233
+    name: temporal
+  - port: {{ .Values.temporal.service.metricsPort }}
+    name: metrics
+    targetPort: 9090
+    protocol: TCP
   selector:
     {{- include "airbyte.selectorLabels" . | nindent 4 }}
     {{- if .Values.temporal.extraSelectorLabels }}

charts/v2/airbyte/templates/airbyte-webapp/deployment.yaml

@@ -63,6 +63,9 @@ spec:
         {{ include "airbyte.keycloak.admin.client.internalHost.env" . | nindent 8 }}
         {{ include "airbyte.tracking.envs" . | nindent 8 }}
         {{ include "airbyte.webapp.envs" . | nindent 8 }}
+        {{- include "airbyte.datadog.envs" . | nindent 8 }}
+        {{- include "airbyte.metrics.envs" . | nindent 8 }}
+
 
         # Values from secret
         {{- if .Values.webapp.secrets }}

charts/v2/airbyte/templates/airbyte-worker/deployment.yaml

@@ -70,6 +70,7 @@ spec:
 
         {{- include "airbyte.common.envs" . | nindent 8 }}
         {{- include "airbyte.database.envs" . | nindent 8 }}
+        {{- include "airbyte.datadog.envs" . | nindent 8 }}  # Add this line
         {{- include "airbyte.database.migrations.envs" . | nindent 8 }}
         {{- include "airbyte.jobs.envs" . | nindent 8 }}
         {{- include "airbyte.jobs.errors.envs" . | nindent 8 }}

charts/v2/airbyte/templates/airbyte-workload-api-server/deployment.yaml

@@ -74,6 +74,7 @@ spec:
         {{- include "airbyte.database.envs" . | nindent 8 }}
         {{- include "airbyte.keycloak.admin.client.realmIssuer.env" . | nindent 8 }}
         {{- include "airbyte.metrics.envs" . | nindent 8 }}
+        {{- include "airbyte.datadog.envs" . | nindent 8 }}
         {{- include "airbyte.micronaut.envs" . | nindent 8 }}
         {{- include "airbyte.temporal.envs" . | nindent 8 }}
         {{- include "airbyte.workloadApiServer.envs" . | nindent 8 }}

charts/v2/airbyte/templates/airbyte-workload-launcher/deployment.yaml

@@ -72,13 +72,14 @@ spec:
 
         {{- include "airbyte.cluster.envs" . | nindent 8 }}
         {{- include "airbyte.common.envs" . | nindent 8 }}
+        {{- include "airbyte.metrics.envs" . | nindent 8 }}  # Add this line
+        {{- include "airbyte.datadog.envs" . | nindent 8 }}  # Add this line
         {{- include "airbyte.database.envs" . | nindent 8 }}
         {{- include "airbyte.database.migrations.envs" . | nindent 8 }}
         {{- include "airbyte.jobs.envs" . | nindent 8 }}
         {{- include "airbyte.jobs.errors.envs" . | nindent 8 }}
         {{- include "airbyte.keycloak.admin.client.realmIssuer.env" . | nindent 8 }}
         {{- include "airbyte.logging.envs" . | nindent 8 }}
-        {{- include "airbyte.metrics.envs" . | nindent 8 }}
         {{- include "airbyte.micronaut.envs" . | nindent 8 }}
         {{- include "airbyte.storage.envs" . | nindent 8 }}
         {{- include "airbyte.secretsManager.envs" . | nindent 8 }}

charts/v2/airbyte/templates/config/_datadog.tpl

@@ -23,17 +23,6 @@ Renders the global.datadog.agentHost value
     {{- .Values.global.datadog.agentHost }}
 {{- end }}
 
-{{/*
-Renders the datadog.agentHost environment variable
-*/}}
-{{- define "airbyte.datadog.agentHost.env" }}
-- name: DD_AGENT_HOST
-  valueFrom:
-    fieldRef:
-      fieldPath: status.hostIP
-
-{{- end }}
-
 {{/*
 Renders the global.datadog.enabled value
 */}}
@@ -311,6 +300,14 @@ Renders the global.datadog.integrations.urlConnection.enabled value
     {{- .Values.global.datadog.integrations.urlConnection.enabled | default false }}
 {{- end }}
 
+{{- define "airbyte.datadog.agentHost.env" }}
+- name: DD_AGENT_HOST
+  valueFrom:
+    configMapKeyRef:
+      name: {{ .Release.Name }}-airbyte-env
+      key: DD_AGENT_HOST
+{{- end }}
+
 {{/*
 Renders the datadog.integrations.urlConnection.enabled environment variable
 */}}

charts/v2/airbyte/templates/config/_metrics.tpl

@@ -95,17 +95,6 @@ Renders the global.metrics.statsd.host value
     {{- .Values.global.metrics.statsd.host }}
 {{- end }}
 
-{{/*
-Renders the metrics.statsd.host environment variable
-*/}}
-{{- define "airbyte.metrics.statsd.host.env" }}
-- name: STATSD_HOST
-  valueFrom:
-    fieldRef:
-      fieldPath: status.hostIP
-
-{{- end }}
-
 {{/*
 Renders the global.metrics.statsd.port value
 */}}
@@ -124,6 +113,14 @@ Renders the metrics.statsd.port environment variable
       key: STATSD_PORT
 {{- end }}
 
+{{- define "airbyte.metrics.statsd.host.env" }}
+- name: STATSD_HOST
+  valueFrom:
+    configMapKeyRef:
+      name: {{ .Release.Name }}-airbyte-env
+      key: STATSD_HOST
+{{- end }}
+
 {{/*
 Renders the set of all metrics environment variables
 */}}