[AGTHEAL-223] http-sd: apply check template rename_labels to SD target tags (#53774)

### What does this PR do?

This PR makes the OpenMetrics `rename_labels` configuration apply consistently to labels coming from Prometheus HTTP Service Discovery.

The HTTP SD provider converts labels returned by the service discovery endpoint into instance tags on the generated OpenMetrics check. Because those tags are added outside the OpenMetrics scraping pipeline, they previously bypassed rename_labels, which only handled labels found in the scraped metrics payload.

This change extracts the existing rename_labels map from the check template and applies it to HTTP SD labels before converting them into tags. No new configuration is introduced.

For example:
```
  prometheus_http_sd:
    configs:
      - url: "http://my-sd-service:8080/service_instances"
        check_template: |
          {
            "name": "openmetrics",
            "init_config": {},
            "instances": [
              {
                "openmetrics_endpoint": "http://%%host%%:%%port%%/metrics",
                "namespace": "app",
                "metrics": [".*"],
                "rename_labels": { "experiment": "appXYZ.experiment" }
              }
            ]
          }
```

An HTTP SD response containing:

```
labels:
  experiment: test123
```

now produces tag: `appXYZ.experiment:test123`

### Motivation
Previously, `rename_labels` in OM check was only applied to labels parsed from the scraped `/metrics` payload. Labels provided by the HTTP Service Discovery endpoint were converted directly into instance tags by the provider and therefore bypassed the same renaming logic.

The difference came from where the labels entered the generated check:

scraped metric labels were processed by the OpenMetrics check
HTTP SD labels were converted directly into instance tags by the provider

This PR closes that gap so a single `rename_labels` configuration applies consistently to both scraped metric labels and HTTP SD target labels.


### Describe how you validated your changes
```
clusterAgent:
  enabled: true
  env:
    - name: DD_PROMETHEUS_HTTP_SD_URL
      value: "http://my-sd-service:8080/service_instances"
    - name: DD_PROMETHEUS_HTTP_SD_CHECK_TEMPLATE
      value: '{"name":"openmetrics","init_config":{},"instances":[{"openmetrics_endpoint":"http://%%host%%:%%port%%/metrics","namespace":"app","metrics":[".*"],"rename_labels":{"experiment":"appXYZ.experiment"}}]}'
```

With the SD service returning:
```
[
    {
        "targets": ["10.0.10.2:9100", "10.0.10.3:9100", "10.0.10.4:9100", "10.0.10.5:9100"],
        "labels": {
               "experiment":"test123"
        }
    }
...
]
```
the generated OM check gets tag `appXYZ.experiment:test123` (renamed).

### Tag de-duplication behavior
`rename_labels` in the provider can rename an SD label to a key that also arrives from scraped /metrics endpoint.The [de-duplication process](https://github.com/DataDog/datadog-agent/blob/9b52f96ec484f74e39d4020c57b77d1c04972e52/pkg/tagset/hash_generator.go#L10-L16) is done by the aggregator when it builds the metric's context key. For instance, there are two identical renamed tags from SD and scraped metrics endpoint,  the aggregator will collapse the identical strings before sending to backend. 

Co-authored-by: minyi.zhu <minyi.zhu@datadoghq.com>

Author: zhuminyi

comp/core/autodiscovery/providers/prometheus_http_sd.go

@@ -69,6 +69,11 @@ type httpSDEntry struct {
 	client        *http.Client
 	checkTemplate httpSDCheckTemplate
 	filterProgram cel.Program // compiled exclude_filter CEL program; nil if no filter
+	// renameLabels mirrors the check template's openmetrics "rename_labels" map.
+	// It is applied to the SD target labels before they are injected as instance
+	// tags, so the same rename_labels the user configures for scraped metrics also
+	// covers the tags derived from the SD response. nil if the template has none.
+	renameLabels map[string]string
 }
 
 // httpSDConfigEntry mirrors a single entry under prometheus_http_sd.configs in
@@ -113,6 +118,7 @@ func buildEntries(rawConfigs []httpSDConfigEntry, sharedClient *http.Client) ([]
 			client:        sharedClient,
 			checkTemplate: tmpl,
 			filterProgram: filterProg,
+			renameLabels:  extractRenameLabels(tmpl),
 		})
 	}
 	return entries, errs
@@ -269,7 +275,7 @@ func (e *httpSDEntry) collect() ([]integration.Config, error) {
 
 	var configs []integration.Config
 	for _, tg := range targetGroups {
-		tags := labelsToTags(tg.Labels)
+		tags := labelsToTags(tg.Labels, e.renameLabels)
 
 		for _, target := range tg.Targets {
 			host, port, splitErr := net.SplitHostPort(target)
@@ -359,8 +365,13 @@ func (e *httpSDEntry) buildConfig(host, port string, tags []string) (integration
 
 // labelsToTags converts HTTP SD labels to Datadog tags.
 // Internal labels (prefixed with __) except __meta_ are skipped.
+// When renameLabels is non-empty, a tag key matching one of its entries is
+// renamed, mirroring the openmetrics check's rename_labels behavior so the same
+// mapping applies to both scraped labels and SD-derived tags. The lookup uses
+// the effective tag key (after the __meta_ prefix is stripped), which is the
+// name the user sees as a tag.
 // Tags are sorted for stable config digests across polls.
-func labelsToTags(labels map[string]string) []string {
+func labelsToTags(labels, renameLabels map[string]string) []string {
 	var tags []string
 	for k, v := range labels {
 		tagKey := k
@@ -369,12 +380,40 @@ func labelsToTags(labels map[string]string) []string {
 		} else if strings.HasPrefix(k, "__") {
 			continue
 		}
+		if renamed, ok := renameLabels[tagKey]; ok {
+			tagKey = renamed
+		}
 		tags = append(tags, tagKey+":"+v)
 	}
 	sort.Strings(tags)
 	return tags
 }
 
+// extractRenameLabels reads the openmetrics "rename_labels" map from the first
+// instance of the check template, if present. The provider is otherwise
+// check-agnostic; this reaches into the openmetrics schema intentionally so the
+// SD-derived tags honor the same rename_labels the user already configures for
+// scraped metrics. Returns nil when the field is absent or malformed.
+func extractRenameLabels(tmpl httpSDCheckTemplate) map[string]string {
+	if len(tmpl.Instances) == 0 {
+		return nil
+	}
+	raw, ok := tmpl.Instances[0]["rename_labels"].(map[string]interface{})
+	if !ok {
+		return nil
+	}
+	renameLabels := make(map[string]string, len(raw))
+	for k, v := range raw {
+		if s, ok := v.(string); ok {
+			renameLabels[k] = s
+		}
+	}
+	if len(renameLabels) == 0 {
+		return nil
+	}
+	return renameLabels
+}
+
 // substituteTemplateVars replaces %%host%% and %%port%% placeholders in a value.
 // Only string values are substituted; all other types are returned as-is.
 func substituteTemplateVars(v interface{}, host, port string) interface{} {

comp/core/autodiscovery/providers/prometheus_http_sd_test.go

@@ -47,6 +47,7 @@ func makeTestProviderWithEntries(t *testing.T, specs []entrySpec) *PrometheusHTT
 			client:        http.DefaultClient,
 			checkTemplate: tmpl,
 			filterProgram: filterProg,
+			renameLabels:  extractRenameLabels(tmpl),
 		}
 	}
 
@@ -180,7 +181,7 @@ func TestLabelsToTags(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			tags := labelsToTags(tt.labels)
+			tags := labelsToTags(tt.labels, nil)
 			assert.Equal(t, tt.expected, tags)
 		})
 	}
@@ -195,7 +196,7 @@ func TestLabelsToTagsStableOrder(t *testing.T) {
 
 	// Run multiple times to verify deterministic ordering
 	for i := 0; i < 10; i++ {
-		tags := labelsToTags(labels)
+		tags := labelsToTags(labels, nil)
 		assert.Equal(t, []string{"a_label:first", "m_label:middle", "z_label:last"}, tags)
 	}
 }
@@ -792,3 +793,129 @@ func TestNewPrometheusHTTPSDConfigProviderFromConfig(t *testing.T) {
 	require.NoError(t, yaml.Unmarshal(configs[0].Instances[0], &instance))
 	assert.Equal(t, "http://host2:9100/metrics", instance["openmetrics_endpoint"])
 }
+
+func TestLabelsToTagsRename(t *testing.T) {
+	tests := []struct {
+		name         string
+		labels       map[string]string
+		renameLabels map[string]string
+		expected     []string
+	}{
+		{
+			name:         "plain label is renamed",
+			labels:       map[string]string{"experiment": "test123", "service": "node"},
+			renameLabels: map[string]string{"experiment": "appXYZ.experiment"},
+			expected:     []string{"appXYZ.experiment:test123", "service:node"},
+		},
+		{
+			name:         "rename applies to __meta_ label after prefix is stripped",
+			labels:       map[string]string{"__meta_service_type": "worker"},
+			renameLabels: map[string]string{"service_type": "appXYZ.service_type"},
+			expected:     []string{"appXYZ.service_type:worker"},
+		},
+		{
+			name:         "label not in rename map is untouched",
+			labels:       map[string]string{"user": "alice"},
+			renameLabels: map[string]string{"experiment": "appXYZ.experiment"},
+			expected:     []string{"user:alice"},
+		},
+		{
+			name:         "nil rename map is a no-op",
+			labels:       map[string]string{"experiment": "test123"},
+			renameLabels: nil,
+			expected:     []string{"experiment:test123"},
+		},
+		{
+			name:         "__ internal labels stay skipped even if named in the map",
+			labels:       map[string]string{"__address__": "10.0.0.1:9090", "experiment": "test123"},
+			renameLabels: map[string]string{"experiment": "appXYZ.experiment"},
+			expected:     []string{"appXYZ.experiment:test123"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			tags := labelsToTags(tt.labels, tt.renameLabels)
+			assert.Equal(t, tt.expected, tags)
+		})
+	}
+}
+
+func TestExtractRenameLabels(t *testing.T) {
+	tests := []struct {
+		name     string
+		template string
+		expected map[string]string
+	}{
+		{
+			name:     "rename_labels present",
+			template: `{"name":"openmetrics","instances":[{"rename_labels":{"experiment":"appXYZ.experiment","pool":"appXYZ.pool"}}]}`,
+			expected: map[string]string{"experiment": "appXYZ.experiment", "pool": "appXYZ.pool"},
+		},
+		{
+			name:     "no rename_labels field",
+			template: `{"name":"openmetrics","instances":[{"openmetrics_endpoint":"http://x/metrics"}]}`,
+			expected: nil,
+		},
+		{
+			name:     "empty rename_labels map",
+			template: `{"name":"openmetrics","instances":[{"rename_labels":{}}]}`,
+			expected: nil,
+		},
+		{
+			name:     "non-string values are skipped",
+			template: `{"name":"openmetrics","instances":[{"rename_labels":{"experiment":"appXYZ.experiment","bad":123}}]}`,
+			expected: map[string]string{"experiment": "appXYZ.experiment"},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			var tmpl httpSDCheckTemplate
+			require.NoError(t, json.Unmarshal([]byte(tt.template), &tmpl))
+			assert.Equal(t, tt.expected, extractRenameLabels(tmpl))
+		})
+	}
+}
+
+// TestRenameLabelsAppliedToSDTags is the end-to-end case behind CONTP-1801:
+// the check template's rename_labels must also rename the tags derived from the
+// SD target labels, not just scraped metrics. Here the SD response labels the
+// target with `experiment`, which the OpenMetrics check alone could never rename
+// (it's an injected instance tag, not a scraped label).
+func TestRenameLabelsAppliedToSDTags(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		json.NewEncoder(w).Encode([]httpSDTargetGroup{
+			{
+				Targets: []string{"10.0.0.7:8000"},
+				Labels:  map[string]string{"experiment": "test123", "service": "node"},
+			},
+		})
+	}))
+	defer server.Close()
+
+	provider := makeTestProviderWithEntries(t, []entrySpec{{
+		url:      server.URL,
+		template: `{"name":"openmetrics","init_config":{},"instances":[{"openmetrics_endpoint":"http://%%host%%:%%port%%/metrics","rename_labels":{"experiment":"appXYZ.experiment"}}]}`,
+	}})
+
+	configs, err := provider.Collect(context.Background())
+	require.NoError(t, err)
+	require.Len(t, configs, 1)
+
+	var instance map[string]interface{}
+	require.NoError(t, yaml.Unmarshal(configs[0].Instances[0], &instance))
+
+	tags := make([]string, 0)
+	for _, tag := range instance["tags"].([]interface{}) {
+		tags = append(tags, tag.(string))
+	}
+	assert.Contains(t, tags, "appXYZ.experiment:test123", "SD `experiment` label should be renamed via rename_labels")
+	assert.NotContains(t, tags, "experiment:test123", "raw `experiment` tag should not remain after rename")
+	assert.Contains(t, tags, "service:node", "unmapped SD labels should be untouched")
+
+	// rename_labels stays in the instance so the OpenMetrics check still renames
+	// scraped labels of the same name. (YAML v2 decodes nested maps with
+	// interface{} keys.)
+	assert.Equal(t, map[interface{}]interface{}{"experiment": "appXYZ.experiment"}, instance["rename_labels"])
+}

releasenotes-dca/notes/http-sd-rename-labels-sd-tags-6f3a9c1d2e4b5a70.yaml

@@ -0,0 +1,10 @@
+---
+enhancements:
+  - |
+    The Cluster Agent's Prometheus HTTP Service Discovery provider now applies
+    the OpenMetrics check template's ``rename_labels`` mapping to the tags
+    derived from the SD target labels, in addition to the labels scraped from
+    each target. Previously ``rename_labels`` only affected scraped metric
+    labels, so a label supplied by the SD endpoint could not be renamed.
+    No configuration change is required: the existing
+    ``rename_labels`` in the ``check_template`` now covers both sources.