Remove worker_version label_replace workaround for Temporal Cloud backlog metrics

Temporal Cloud now emits temporal_worker_deployment_name and temporal_worker_build_id
as separate labels, so the label_replace recording rule and adapter name alias are no
longer needed. The adapter queries temporal_cloud_v1_approximate_backlog_count directly.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: carlydf

api/v1alpha1/workerresourcetemplate_webhook_test.go

@@ -253,7 +253,7 @@ func TestWorkerResourceTemplate_ValidateCreate(t *testing.T) {
 							"type": "External",
 							"external": map[string]interface{}{
 								"metric": map[string]interface{}{
-									"name": "temporal_backlog_count_by_version",
+									"name": "temporal_cloud_v1_approximate_backlog_count",
 									"selector": map[string]interface{}{
 										"matchLabels": map[string]interface{}{
 											"task_type": "Activity",
@@ -282,7 +282,7 @@ func TestWorkerResourceTemplate_ValidateCreate(t *testing.T) {
 							"type": "External",
 							"external": map[string]interface{}{
 								"metric": map[string]interface{}{
-									"name": "temporal_backlog_count_by_version",
+									"name": "temporal_cloud_v1_approximate_backlog_count",
 									"selector": map[string]interface{}{
 										"matchLabels": map[string]interface{}{
 											"temporal_worker_deployment_name": "default_my-worker",
@@ -309,7 +309,7 @@ func TestWorkerResourceTemplate_ValidateCreate(t *testing.T) {
 							"type": "External",
 							"external": map[string]interface{}{
 								"metric": map[string]interface{}{
-									"name": "temporal_backlog_count_by_version",
+									"name": "temporal_cloud_v1_approximate_backlog_count",
 									"selector": map[string]interface{}{
 										"matchLabels": map[string]interface{}{
 											"temporal_worker_build_id": "abc123",
@@ -336,7 +336,7 @@ func TestWorkerResourceTemplate_ValidateCreate(t *testing.T) {
 							"type": "External",
 							"external": map[string]interface{}{
 								"metric": map[string]interface{}{
-									"name": "temporal_backlog_count_by_version",
+									"name": "temporal_cloud_v1_approximate_backlog_count",
 									"selector": map[string]interface{}{
 										"matchLabels": map[string]interface{}{
 											"temporal_namespace": "my-ns",
@@ -363,7 +363,7 @@ func TestWorkerResourceTemplate_ValidateCreate(t *testing.T) {
 							"type": "External",
 							"external": map[string]interface{}{
 								"metric": map[string]interface{}{
-									"name": "temporal_backlog_count_by_version",
+									"name": "temporal_cloud_v1_approximate_backlog_count",
 									"selector": map[string]interface{}{
 										"matchLabels": map[string]interface{}{},
 									},
@@ -390,7 +390,7 @@ func TestWorkerResourceTemplate_ValidateCreate(t *testing.T) {
 							"type": "External",
 							"external": map[string]interface{}{
 								"metric": map[string]interface{}{
-									"name":     "temporal_backlog_count_by_version",
+									"name":     "temporal_cloud_v1_approximate_backlog_count",
 									"selector": map[string]interface{}{},
 								},
 								"target": map[string]interface{}{

docs/worker-resource-templates.md

@@ -130,7 +130,7 @@ spec:
         - type: External
           external:
             metric:
-              name: temporal_backlog_count_by_version
+              name: temporal_cloud_v1_approximate_backlog_count
               selector:
                 matchLabels:
                   task_type: "Activity"

examples/wrt-hpa-backlog.yaml

@@ -18,7 +18,7 @@
 #        helm upgrade prometheus-adapter prometheus-community/prometheus-adapter \
 #          -n monitoring -f internal/demo/k8s/prometheus-adapter-values.yaml
 #   3. Verify the backlog metric is flowing:
-#        # Port-forward Prometheus and query: temporal_backlog_count_by_version
+#        # Port-forward Prometheus and query: temporal_cloud_v1_approximate_backlog_count
 #
 # Apply:
 #   kubectl apply -f examples/wrt-hpa-backlog.yaml
@@ -61,12 +61,15 @@ spec:
               value: "750m"
 
         # Metric: backlog count — scale up when tasks are queued but not yet picked up.
+        # Temporal Cloud emits temporal_cloud_v1_approximate_backlog_count with
+        # temporal_worker_deployment_name and temporal_worker_build_id as separate labels;
+        # prometheus-adapter exposes it as temporal_cloud_v1_approximate_backlog_count.
         # temporal_worker_deployment_name, temporal_worker_build_id, and temporal_namespace are
         # appended automatically by the controller — do not set them here.
         - type: External
           external:
             metric:
-              name: temporal_backlog_count_by_version
+              name: temporal_cloud_v1_approximate_backlog_count
               selector:
                 matchLabels:
                   task_type: "Activity"

internal/demo/README.md

@@ -262,14 +262,6 @@ Stop the load generator (`Ctrl-C`) and watch the HPA scale back down as in-fligh
 
 `approximate_backlog_count` measures tasks queued in Temporal but not yet started on a worker. Adding it as a second HPA metric means the HPA scales up on *arriving* work even before slots are full — important for bursty traffic.
 
-> **Note:** Temporal Cloud emits `temporal_approximate_backlog_count` with a combined
-> `worker_version="<worker-deployment-name>_<build-id>"` label that easily exceeds Kubernetes max label
-> length of 63 characters. The recording rule in `prometheus-stack-values.yaml` uses `label_replace` 
-> to extract `temporal_worker_deployment_name` and `temporal_worker_build_id` as separate k8s-compatible 
-> labels, producing `temporal_backlog_count_by_version`. The HPA then selects on those labels — the same 
-> pair used by Phase 1. Temporal Cloud is in the process of rolling out the new separate labels, so this
-> workaround is required until then.
-
 **Step 1 — Create the Temporal Cloud credentials secret.**
 
 Create a Temporal Cloud metrics API key (separate from the namespace API key) at Cloud UI → Settings → Observability → Generate API Key. Save it to `certs/metrics-api-key.txt`, then create the secret in the `monitoring` namespace:
@@ -294,11 +286,11 @@ helm upgrade prometheus-adapter prometheus-community/prometheus-adapter \
 
 ```bash
 kubectl -n monitoring port-forward svc/prometheus-kube-prometheus-prometheus 9092:9090 &
-curl -s 'http://localhost:9092/api/v1/query?query=temporal_backlog_count_by_version' \
+curl -s 'http://localhost:9092/api/v1/query?query=temporal_cloud_v1_approximate_backlog_count' \
   | jq '.data.result'
 ```
 
-You should see a result with `twd_name` and `build_id` labels. If the result is empty, wait 15–30s for the recording rule to evaluate.
+You should see results with `temporal_worker_deployment_name` and `temporal_worker_build_id` labels. If the result is empty, wait 10–15s for the first scrape to complete.
 
 **Step 4 — Apply the combined WRT.**
 ```bash

internal/demo/k8s/grafana-dashboard.json

@@ -127,7 +127,7 @@
       },
       "targets": [
         {
-          "expr": "temporal_backlog_count_by_version{task_type=\"Workflow\", temporal_worker_deployment_name=\"default_helloworld\"}",
+          "expr": "temporal_cloud_v1_approximate_backlog_count{task_type=\"Workflow\", temporal_worker_deployment_name=\"default_helloworld\"}",
           "legendFormat": "{{temporal_worker_deployment_name}} / {{temporal_worker_build_id}}",
           "refId": "A"
         }
@@ -152,7 +152,7 @@
       },
       "targets": [
         {
-          "expr": "temporal_backlog_count_by_version{task_type=\"Activity\", temporal_worker_deployment_name=\"default_helloworld\"}",
+          "expr": "temporal_cloud_v1_approximate_backlog_count{task_type=\"Activity\", temporal_worker_deployment_name=\"default_helloworld\"}",
           "legendFormat": "{{temporal_worker_deployment_name}} / {{temporal_worker_build_id}}",
           "refId": "A"
         }

internal/demo/k8s/prometheus-adapter-values.yaml

@@ -33,14 +33,11 @@ rules:
         namespaced: false  # cluster-scoped: HPAs in any k8s namespace can consume this metric
 
     # Phase 2: approximate backlog count per worker version (from Temporal Cloud).
-    # Uses the temporal_backlog_count_by_version recording rule.
-    # cluster-scoped so HPAs in any namespace can consume it; temporal_worker_deployment_name
-    # + temporal_worker_build_id + temporal_namespace matchLabels in the HPA are sufficient to
-    # select the right series.
-    - seriesQuery: 'temporal_backlog_count_by_version{}'
+    # Temporal Cloud emits temporal_worker_deployment_name and temporal_worker_build_id as
+    # separate labels, so the raw metric can be used directly — no recording rule needed.
+    # cluster-scoped so HPAs in any namespace can consume it.
+    - seriesQuery: 'temporal_cloud_v1_approximate_backlog_count{}'
       metricsQuery: 'sum(<<.Series>>{<<.LabelMatchers>>})'
-      name:
-        as: "temporal_backlog_count_by_version"
       resources:
         namespaced: false  # cluster-scoped: HPAs in any namespace can consume this metric
 

internal/demo/k8s/prometheus-stack-values.yaml

@@ -48,7 +48,10 @@ prometheus:
         metrics_path: /v1/metrics
         params:
           labels:
-            - worker_version
+            - temporal_worker_deployment_name
+            - temporal_worker_build_id
+            - temporal_namespace
+            - task_type
 
 # ─── 3. Recording Rules ─────────────────────────────────────────────────────
 
@@ -80,43 +83,4 @@ additionalPrometheusRulesMap:
                 1
               )
 
-      - name: temporal_cloud_backlog
-        interval: 10s
-        rules:
-          # Backlog count per worker version, shaped to match the label format that
-          # Temporal Cloud will emit natively in a future release. This recording rule
-          # is a temporary shim: once Temporal Cloud emits temporal_worker_deployment_name and
-          # temporal_worker_build_id as separate labels, this rule can be deleted with no
-          # other changes. Note: this rule only works with Build IDs that don't have underscores.
-          #
-          # Current Temporal Cloud label:
-          #   worker_version="{k8s-namespace}_{workerdeployment-name}_{build-id}"
-          #
-          # k8s namespaces and WorkerDeployment names follow DNS naming rules (no underscores), so _
-          # is an unambiguous separator and label_replace is reliable.
-          #
-          # Extracted labels (matching future Temporal Cloud output):
-          #   temporal_worker_deployment_name      — "{k8s-namespace}_{workerdeployment-name}" (all but last segment)
-          #   temporal_worker_build_id  — last segment; ≤63 chars, valid k8s label value
-          #   temporal_namespace          — preserved from source metric
-          #
-          # Produced labels:
-          #   temporal_worker_deployment_name      — inner label_replace: all but the last _ segment
-          #   temporal_worker_build_id  — outer label_replace: last _ segment
-          #   temporal_namespace          — preserved from source metric
-          #   task_type                   — preserved as a dimension; HPAs can filter by
-          #                                 task_type="Activity", task_type="Workflow", etc.
-          # The prometheus-adapter serves this as a cluster-scoped external metric
-          # (namespaced: false), so HPAs in any namespace can consume it.
-          - record: temporal_backlog_count_by_version
-            expr: |
-              label_replace(
-                label_replace(
-                  sum by (worker_version, task_type, temporal_namespace) (
-                    temporal_cloud_v1_approximate_backlog_count{worker_version=~".+_.+_.+"}
-                  ),
-                  "temporal_worker_deployment_name", "$1", "worker_version", "^(.+)_[^_]+$"
-                ),
-                "temporal_worker_build_id", "$1", "worker_version", ".*_([^_]+)$"
-              )
 

internal/k8s/workerresourcetemplates_test.go

@@ -180,7 +180,7 @@ func TestAutoInjectFields_MatchLabels(t *testing.T) {
 					"type": "External",
 					"external": map[string]interface{}{
 						"metric": map[string]interface{}{
-							"name": "temporal_backlog_count_by_version",
+							"name": "temporal_cloud_v1_approximate_backlog_count",
 							"selector": map[string]interface{}{
 								"matchLabels": map[string]interface{}{}, // opt-in for metric selector
 							},
@@ -223,7 +223,7 @@ func TestAutoInjectFields_MetricSelector(t *testing.T) {
 					"type": "External",
 					"external": map[string]interface{}{
 						"metric": map[string]interface{}{
-							"name": "temporal_backlog_count_by_version",
+							"name": "temporal_cloud_v1_approximate_backlog_count",
 							"selector": map[string]interface{}{
 								"matchLabels": matchLabels,
 							},
@@ -259,7 +259,7 @@ func TestAutoInjectFields_MetricSelector(t *testing.T) {
 					"type": "External",
 					"external": map[string]interface{}{
 						"metric": map[string]interface{}{
-							"name":     "temporal_backlog_count_by_version",
+							"name":     "temporal_cloud_v1_approximate_backlog_count",
 							"selector": map[string]interface{}{},
 						},
 					},

internal/tests/internal/wrt_integration_test.go

@@ -192,7 +192,7 @@ func wrtTestCases() []testCase {
 									"type": "External",
 									"external": {
 										"metric": {
-											"name": "temporal_backlog_count_by_version",
+											"name": "temporal_cloud_v1_approximate_backlog_count",
 											"selector": {
 												"matchLabels": {
 													"task_type": "Activity"