ScrapeConfig targetAllocator bad password using BasicAuth/OAuth2

[ChiragR7]

Component(s)

receiver/prometheus

What happened?

Description

Basic Auth and OAuth2 have the same issue: the password or client secret is not being decoded correctly, which results in an unauthorized error.

Steps to Reproduce

Cluster Role/Role/Role Bindings

# 1. ServiceAccount
---
apiVersion: v1
kind: ServiceAccount
metadata:
  name: otel-target-allocator-sa
  namespace: infra
---
# 2. Role
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
  name: otel-target-allocator-role
  namespace: infra
rules:
  - apiGroups:
      - ""
    resources:
      - pods
      - services
      - endpoints
      - configmaps
      - secrets
      - namespaces
    verbs:
      - get
      - watch
      - list
  - apiGroups:
      - apps
    resources:
      - statefulsets
    verbs:
      - get
      - watch
      - list
  - apiGroups:
      - discovery.k8s.io
    resources:
      - endpointslices
    verbs:
      - get
      - watch
      - list
  - apiGroups:
      - networking.k8s.io
    resources:
      - ingresses
    verbs:
      - get
      - watch
      - list
  - apiGroups:
      - monitoring.coreos.com
    resources:
      - servicemonitors
      - podmonitors
      - scrapeconfigs
      - probes
    verbs:
      - get
      - watch
      - list
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: otel-target-allocator
rules:
  # Core resources
  - apiGroups: [""]
    resources:
      - pods
      - services
      - endpoints
      - namespaces
      - nodes
      - configmaps
    verbs: [get, list, watch]

  # Node metrics
  - apiGroups: [""]
    resources:
      - nodes/metrics
    verbs: [get, list, watch]

  # EndpointSlices
  - apiGroups: ["discovery.k8s.io"]
    resources:
      - endpointslices
    verbs: [get, list, watch]

  # Ingresses
  - apiGroups: ["networking.k8s.io"]
    resources:
      - ingresses
    verbs: [get, list, watch]

  # Prometheus Operator CRDs
  - apiGroups: ["monitoring.coreos.com"]
    resources:
      - servicemonitors
      - podmonitors
      - scrapeconfigs
      - probes
    verbs: ["*"]

  # Non-resource URLs
  - nonResourceURLs:
      - /metrics
      - /api
      - /api/*
      - /apis
      - /apis/*
    verbs: [get]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: otel-target-allocator
subjects:
  - kind: ServiceAccount
    name: otel-target-allocator-sa
    namespace: infra
roleRef:
  kind: ClusterRole
  name: otel-target-allocator
  apiGroup: rbac.authorization.k8s.io

---
# 3. RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
  name: otel-target-allocator-rb
  namespace: infra
subjects:
  - kind: ServiceAccount
    name: otel-target-allocator-sa
    namespace: infra
roleRef:
  kind: Role
  name: otel-target-allocator-role
  apiGroup: rbac.authorization.k8s.io

Secret

apiVersion: v1
kind: Secret
metadata:
  name: Test-secret
  namespace: infra
type: Opaque
stringData:
  username: 'admin'
  password: 'Test@123'

ScrapeConfig

apiVersion: monitoring.coreos.com/v1alpha1
kind: ScrapeConfig
metadata:
  namespace: infra
  name: target-scrape-config
  labels:
    prometheus.target.allocator: "enabled"
spec:
  jobName: test
  metricsPath: '/v1/metrics'
  tlsConfig:
    insecureSkipVerify: true
    serverName: '10.87.218.92'
  scheme: HTTPS
  basicAuth:
    username:
        name: Test-secret
        key: username
    password:
        key: password
        name: Test-secret
  staticConfigs:
    - targets: ['10.87.218.92']
  params:
    id: ['e83134ff-89fb-45eb-97ae-920b35f8fde5']

Prometheus Receiver

apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
  name: prometheus-receiver
  namespace: infra
spec:
  mode: statefulset
  targetAllocator:
    enabled: true
    serviceAccount: otel-target-allocator-sa
    replicas: 1
    prometheusCR:
      enabled: true
      allowNamespaces:
        - infra
      scrapeConfigSelector:
        matchLabels:
          prometheus.target.allocator: "enabled"
      podMonitorSelector:
        matchLabels:
          non-existent-label: "true"
      serviceMonitorSelector:
        matchLabels:
          non-existent-label: "true"
  config:
    receivers:
      prometheus:
        config:
          scrape_configs: []

    processors:
      batch:
        timeout: 10s
        send_batch_size: 1000
        send_batch_max_size: 1000

      memory_limiter:
        check_interval: 1s
        limit_mib: 60
        spike_limit_mib: 40

    exporters:
      debug:
        verbosity: normal

    service:
      pipelines:
        metrics:
          receivers: [prometheus]
          processors: [memory_limiter,batch]
          exporters: [debug]
      telemetry:
        logs:
          level: debug
  managementState: 'managed'
  resources:
    limits:
      cpu: 500m
      memory: 512Mi
    requests:
      cpu: 250m
      memory: 256Mi

Expected Result

It should scrape the target

Actual Result

{"target": "https://10.87.218.92/v1/metrics?id=e83134ff-89fb-45eb-97ae-920b35f8fde5", "err": "server returned HTTP status 401 Unauthorized"}

Collector version

0.143.0

Environment information

Environment

OS: (e.g., "Ubuntu 20.04")
Compiler(if manually compiled): (e.g., "go 14.2")

OpenTelemetry Collector configuration

It worked fine without targetallocator when I give config in place

apiVersion: opentelemetry.io/v1beta1
kind: OpenTelemetryCollector
metadata:
  name: prometheus-receiver
  namespace: infra
spec:
  mode: statefulset
  config:
    receivers:
      prometheus:
        config:
          scrape_configs:
            - job_name: 'my-service'
              metrics_path: '/v1/metrics'
              scheme: https
              static_configs:
                - targets: ['10.87.218.92']
              tls_config:
                insecure_skip_verify: true
              basic_auth:
                username: 'admin'
                password: 'Test@123'
              params:
                id: ['e83134ff-89fb-45eb-97ae-920b35f8fde5']


    processors:
      batch:
        timeout: 10s
        send_batch_size: 1000
        send_batch_max_size: 1000
      memory_limiter:
        check_interval: 1s
        limit_mib: 60
        spike_limit_mib: 40

    exporters:
      debug:
        verbosity: normal

    service:
      pipelines:
        metrics:
          receivers: [prometheus]
          processors: [memory_limiter, batch]
          exporters: [debug]
  managementState: 'managed'
  resources:
    limits:
      cpu: 500m
      memory: 512Mi
    requests:
      cpu: 250m
      memory: 256Mi

Log output

Additional context

No response

Tip

_{React with 👍 to help prioritize this issue. Please use comments to provide useful context, avoiding +1 or me too, to help us triage it. Learn more here.}

[github-actions]

Pinging code owners:

• receiver/prometheus: @Aneurysm9 @dashpole @ArthurSens @krajorama

See Adding Labels via Comments if you do not have permissions to add labels yourself.

[ChiragR7]

Do you think we should change below method in receiver/prometheusreceiver/internal/targetallocator/config.go ???

func unmarshalConf(conf *confmap.Conf, cb func(map[string]any), out any) error {
	in := conf.ToStringMap()
	if len(in) == 0 {
		return nil
	}

	if cb != nil {
		cb(in)
	}

	// Convert any Secret types in the map to their string values before marshaling.
	// This prevents the Secret type's default MarshalYAML from redacting values to "<secret>".
	// This is critical for authentication to work properly when using configurations with
	// basic auth, bearer tokens, OAuth2 credentials, etc.
	convertedMap := convertSecretsToStrings(in)

	yamlOut, err := yaml.Marshal(convertedMap)
	if err != nil {
		return fmt.Errorf("prometheus receiver: failed to marshal config to yaml: %w", err)
	}

	err = yaml.Unmarshal(yamlOut, out)
	if err != nil {
		return fmt.Errorf("prometheus receiver: failed to unmarshal yaml to prometheus config object: %w", err)
	}
	return nil
}


// convertSecretsToStrings recursively converts any commonconfig.Secret types in a map to strings.
// This prevents the Secret type's default MarshalYAML from redacting values to "<secret>".
func convertSecretsToStrings(in map[string]any) map[string]any {
	out := make(map[string]any, len(in))
	for k, v := range in {
		out[k] = convertSecretValue(v)
	}
	return out
}

// convertSecretValue converts a value, handling Secret types and recursing into maps and slices.
func convertSecretValue(v any) any {
	switch val := v.(type) {
	case commonconfig.Secret:
		// Convert Secret to its string value
		return string(val)
	case map[string]any:
		// Recursively convert maps
		return convertSecretsToStrings(val)
	case []any:
		// Recursively convert slices
		result := make([]any, len(val))
		for i, item := range val {
			result[i] = convertSecretValue(item)
		}
		return result
	default:
		// Return other types as-is
		return val
	}
}

[ChiragR7]

It handles both

		inputMap := map[string]any{
			"authorization": map[string]any{
				"type":        "Bearer",
				"credentials": promConfig.Secret("secretBearerToken888"),
			},
		}
		
		var resultConfig promConfig.HTTPClientConfig
		err := unmarshalConf(confmap.NewFromStringMap(inputMap), nil, &resultConfig)

		var cfg promConfig.HTTPClientConfig
		input := map[string]any{
			"basic_auth": map[string]any{
				"username": "original-user",
				"password": "secretPassword123",
			},
			"follow_redirects": false,
		}
		// Callback that mutates the config but should not affect secrets
		cb := func(m map[string]any) {
			m["basic_auth"].(map[string]any)["username"] = "mutated-user"
			m["follow_redirects"] = true
		}
		err := unmarshalConf(confmap.NewFromStringMap(input), cb, &cfg)
		require.NoError(t, err)