Allow defining extra resource labels in prometheus (#1742)

* Allow defining extra resource labels in prometheus

* reformat

Author: mariomac

docs/sources/configure/export-data.md

@@ -180,6 +180,23 @@ no Prometheus endpoint is open.
 
 Specifies the HTTP query path to fetch the list of Prometheus metrics.
 
+| YAML                        | Environment variable                         | Type            | Default |
+|-----------------------------|----------------------------------------------|-----------------|---------|
+| `extra_resource_attributes` | `BEYLA_PROMETHEUS_EXTRA_RESOURCE_ATTRIBUTES` | list of strings | (empty) |
+
+A list of additional resource attributes to be added to the reported `target_info` metric.
+
+Due to internal limitations of the Prometheus API client, Beyla needs to know beforehand which attributes are exposed
+for each metric. This would cause that some attributes that are discovered at runtime, during instrumentation, won't
+be visible by default. For example, attributes defined on each application via Kubernetes annotations, or in the
+target application's `OTEL_RESOURCE_ATTRIBUTES` environment variable.
+
+For example, an application defining the `OTEL_RESOURCE_ATTRIBUTES=deployment.environment=production` as environment
+variable, the `target_info{deployment.environment="production"}` attribute would be visible by default if the metrics
+are exported via OpenTelemetry but not if they are exported via Prometheus.
+
+To make `deployment_environment` visible in Prometheus, you need to add it to the `extra_resource_attributes` list.
+
 | YAML  | Environment variable   | Type     | Default |
 | ----- | ---------------------- | -------- | ------- |
 | `ttl` | `BEYLA_PROMETHEUS_TTL` | Duration | `5m`    |

pkg/export/prom/prom.go

@@ -6,6 +6,7 @@ import (
 	"runtime"
 	"slices"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/golang-lru/v2/expirable"
@@ -121,6 +122,11 @@ type PrometheusConfig struct {
 	// Registry is only used for embedding Beyla within the Grafana Agent.
 	// It must be nil when Beyla runs as standalone
 	Registry *prometheus.Registry `yaml:"-"`
+
+	// ExtraResourceLabels adds extra metadata labels to Prometheus metrics from sources whose availability can't be known
+	// beforehand. For example, to add the OTEL deployment.environment resource attribute as a Prometheus resource attribute,
+	// you should add `deployment.environment`.
+	ExtraResourceLabels []string `yaml:"extra_resource_attributes" env:"BEYLA_PROMETHEUS_EXTRA_RESOURCE_ATTRIBUTES" envSeparator:","`
 }
 
 func (p *PrometheusConfig) SpanMetricsEnabled() bool {
@@ -161,7 +167,8 @@ func (p *PrometheusConfig) Enabled() bool {
 }
 
 type metricsReporter struct {
-	cfg *PrometheusConfig
+	cfg                 *PrometheusConfig
+	extraMetadataLabels []attr.Name
 
 	beylaInfo             *Expirer[prometheus.Gauge]
 	httpDuration          *Expirer[prometheus.Histogram]
@@ -308,11 +315,13 @@ func newReporter(
 	kubeEnabled := ctxInfo.K8sInformer.IsKubeEnabled()
 	// If service name is not explicitly set, we take the service name as set by the
 	// executable inspector
+	extraMetadataLabels := parseExtraMetadata(cfg.ExtraResourceLabels)
 	mr := &metricsReporter{
 		bgCtx:                     ctx,
 		ctxInfo:                   ctxInfo,
 		cfg:                       cfg,
 		kubeEnabled:               kubeEnabled,
+		extraMetadataLabels:       extraMetadataLabels,
 		hostID:                    ctxInfo.HostID,
 		clock:                     clock,
 		is:                        is,
@@ -457,7 +466,7 @@ func newReporter(
 			return NewExpirer[prometheus.Gauge](prometheus.NewGaugeVec(prometheus.GaugeOpts{
 				Name: TracesTargetInfo,
 				Help: "target service information in trace span metric format",
-			}, labelNamesTargetInfo(kubeEnabled)).MetricVec, clock.Time, cfg.TTL)
+			}, labelNamesTargetInfo(kubeEnabled, extraMetadataLabels)).MetricVec, clock.Time, cfg.TTL)
 		}),
 		tracesHostInfo: optionalGaugeProvider(cfg.SpanMetricsEnabled() || cfg.ServiceGraphMetricsEnabled(), func() *Expirer[prometheus.Gauge] {
 			return NewExpirer[prometheus.Gauge](prometheus.NewGaugeVec(prometheus.GaugeOpts{
@@ -500,7 +509,7 @@ func newReporter(
 		targetInfo: NewExpirer[prometheus.Gauge](prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: TargetInfo,
 			Help: "attributes associated to a given monitored entity",
-		}, labelNamesTargetInfo(kubeEnabled)).MetricVec, clock.Time, cfg.TTL),
+		}, labelNamesTargetInfo(kubeEnabled, extraMetadataLabels)).MetricVec, clock.Time, cfg.TTL),
 		gpuKernelCallsTotal: optionalCounterProvider(is.GPUEnabled(), func() *Expirer[prometheus.Counter] {
 			return NewExpirer[prometheus.Counter](prometheus.NewCounterVec(prometheus.CounterOpts{
 				Name: attributes.GPUKernelLaunchCalls.Prom,
@@ -597,6 +606,16 @@ func newReporter(
 	return mr, nil
 }
 
+func parseExtraMetadata(labels []string) []attr.Name {
+	// first, we convert any metric in snake_format to dotted.format,
+	// as it is the internal representation of metadata labels
+	attrNames := make([]attr.Name, len(labels))
+	for i, label := range labels {
+		attrNames[i] = attr.Name(strings.ReplaceAll(label, "_", "."))
+	}
+	return attrNames
+}
+
 func optionalHistogramProvider(enable bool, provider func() *Expirer[prometheus.Histogram]) *Expirer[prometheus.Histogram] {
 	if !enable {
 		return nil
@@ -796,13 +815,17 @@ func (r *metricsReporter) labelValuesSpans(span *request.Span) []string {
 	}
 }
 
-func labelNamesTargetInfo(kubeEnabled bool) []string {
+func labelNamesTargetInfo(kubeEnabled bool, extraMetadataLabelNames []attr.Name) []string {
 	names := []string{hostIDKey, hostNameKey, serviceKey, serviceNamespaceKey, serviceInstanceKey, serviceJobKey, telemetryLanguageKey, telemetrySDKKey, sourceKey}
 
 	if kubeEnabled {
 		names = appendK8sLabelNames(names)
 	}
 
+	for _, mdn := range extraMetadataLabelNames {
+		names = append(names, mdn.Prom())
+	}
+
 	return names
 }
 
@@ -823,6 +846,10 @@ func (r *metricsReporter) labelValuesTargetInfo(service svc.Attrs) []string {
 		values = appendK8sLabelValuesService(values, service)
 	}
 
+	for _, k := range r.extraMetadataLabels {
+		values = append(values, service.Metadata[k])
+	}
+
 	return values
 }
 

test/integration/configs/instrumenter-config-promscrape.yml

@@ -7,6 +7,7 @@ prometheus_export:
   features:
     - application
     - application_process
+  extra_resource_attributes: ["deployment_environment"]
 attributes:
   select:
     process_cpu_utilization:

test/integration/k8s/common/k8s_metrics_testfuncs.go

@@ -229,9 +229,10 @@ func FeatureGRPCMetricsDecoration(manifest string, overrideAttrs map[string]stri
 				attributeMap(allAttributes, overrideAttrs))).
 		Assess("target_info metrics exist",
 			testMetricsDecoration([]string{"target_info"}, `{job=~".*testserver"}`, map[string]string{
-				"host_name": "testserver",
-				"host_id":   HostIDRegex,
-				"instance":  targetInfoInstance,
+				"host_name":              "testserver",
+				"host_id":                HostIDRegex,
+				"instance":               targetInfoInstance,
+				"deployment_environment": "integration-test",
 			}),
 		).Feature()
 }