[nr-k8s-otel-collector] Adding ksmSelector for prometheus receiver (#1966)

tviaud · dbudziwojskiNR · web-flow · commit 1066784c7788 · 2025-11-05T09:59:26.000-08:00
#### Is this a new chart No #### What this PR does / why we need it: This pull request adds support for customising which `kube-state-metrics` instance is targeted by the Prometheus receiver in the otel deployment. This is useful for clusters running multiple kube-state-metrics instances, allowing users to specify a label selector to avoid scraping duplicate metrics. This is mostly mirroring what's done for the `ksm` component in the `newrelic-infrastructure` chart: https://github.com/newrelic/nri-kubernetes/blob/main/charts/newrelic-infrastructure/values.yaml#L255. This would be useful in migration cases to keep `nri-bundle` and `nr-k8s-otel-collector` up at the same time for example. #### Which issue this PR fixes *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close that issue when PR gets merged)* - fixes # #### Special notes for your reviewer: #### Checklist [Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields.] - [x] Chart Version bumped - [x] Variables are documented in the README.md - [x] Title of the PR starts with chart name (e.g. `[mychartname]`) # Release Notes to Publish (nr-k8s-otel-collector) If this PR contains changes in `nr-k8s-otel-collector`, please complete the following section. All other charts should ignore this section.  ## 🚀 What's Changed * Added configurable label selector for kube-state-metrics discovery via `receivers.prometheus.ksmSelector` to prevent duplicate metric scraping when running multiple kube-state-metrics instances in the same cluster (e.g., during migration from nri-bundle to nr-k8s-otel-collector).  --------- Co-authored-by: Daniel Budziwojski <dbudziwojski@newrelic.com>
diff --git a/charts/nr-k8s-otel-collector/Chart.yaml b/charts/nr-k8s-otel-collector/Chart.yaml
@@ -17,7 +17,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.9.0
+version: 0.9.1
 
 
 dependencies:
diff --git a/charts/nr-k8s-otel-collector/README.md b/charts/nr-k8s-otel-collector/README.md
@@ -198,6 +198,7 @@ to export data to this connector which can then be connected to the New Relic ma
 | receivers.kubeletstats.enabled | bool | `true` | Specifies whether the `kubeletstats` receiver is enabled |
 | receivers.kubeletstats.scrapeInterval | string | `1m` | Sets the scrape interval for the `kubeletstats` receiver |
 | receivers.prometheus.enabled | bool | `true` | Specifies whether the `prometheus` receiver is enabled |
+| receivers.prometheus.ksmSelector | string | `app.kubernetes.io/name=kube-state-metrics` | Label selector that will be used to automatically discover an instance of kube-state-metrics running in the cluster. |
 | receivers.prometheus.scrapeInterval | string | `1m` | Sets the scrape interval for the `prometheus` receiver |
 | serviceAccount | object | See `values.yaml` | Settings controlling ServiceAccount creation |
 | serviceAccount.create | bool | `true` | Specifies whether a ServiceAccount should be created |
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/clusterrole.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/clusterrole.yaml
@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
 rules:
   - apiGroups:
     - ""
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/clusterrolebinding.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/clusterrolebinding.yaml
@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
 subjects:
   - kind: ServiceAccount
     name: nr-k8s-otel-collector
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/daemonset-configmap.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/daemonset-configmap.yaml
@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
 data:
   daemonset-config.yaml: |
     receivers:
@@ -592,7 +592,7 @@ data:
             value: <cluser_name>
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.9.0
+            value: 0.9.1
           - key: newrelic.entity.type
             action: upsert
             value: "k8s"
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/daemonset.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/daemonset.yaml
@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
 spec:
   selector:
     matchLabels:
@@ -24,7 +24,7 @@ spec:
         app.kubernetes.io/name: nr-k8s-otel-collector
         component: daemonset
       annotations:
-        checksum/config: 8642440e20482bfac3b36b4627568fe20e39b868f724f4eac1c0870ca11ed6e8
+        checksum/config: a56de7c912a17330a02ac7dccd064585f4462d0988af358ecf102eb7a021fac0
     spec:
       serviceAccountName: nr-k8s-otel-collector
       containers:
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/deployment-configmap.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/deployment-configmap.yaml
@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
 data:
   deployment-config.yaml: |
     receivers:
@@ -507,7 +507,7 @@ data:
             value: <cluser_name>
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.9.0
+            value: 0.9.1
           - key: newrelic.entity.type
             action: upsert
             value: "k8s"
@@ -525,7 +525,7 @@ data:
             value: <cluser_name>
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.9.0
+            value: 0.9.1
 
       transform/events:
         log_statements:
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/deployment.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/deployment.yaml
@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
 spec:
   replicas: 1
   minReadySeconds: 5
@@ -26,7 +26,7 @@ spec:
         app.kubernetes.io/name: nr-k8s-otel-collector
         component: deployment
       annotations:
-        checksum/config: 5615ec42ac95ea7c1a64be8653ec13b1435be77de1f1a2f9bbdeef476a6b63ae
+        checksum/config: ee1dfc4c58f891f8358838b0c5795afe1e42d8f2230d0158a4a8792da8828099
     spec:
       serviceAccountName: nr-k8s-otel-collector
       containers:
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/secret.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/secret.yaml
@@ -10,6 +10,6 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
 data:
   licenseKey: PE5SX2xpY2Vuc2VLZXk+
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/service.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/service.yaml
@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
 spec:
   type: ClusterIP
   ports:
diff --git a/charts/nr-k8s-otel-collector/examples/k8s/rendered/serviceaccount.yaml b/charts/nr-k8s-otel-collector/examples/k8s/rendered/serviceaccount.yaml
@@ -10,5 +10,5 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.9.0
+    helm.sh/chart: nr-k8s-otel-collector-0.9.1
   annotations:
diff --git a/charts/nr-k8s-otel-collector/templates/_config_map.tpl b/charts/nr-k8s-otel-collector/templates/_config_map.tpl
@@ -242,3 +242,17 @@
       {{- end -}}
     {{- end -}}
 {{- end -}}
+
+{{- /* Parse KSM selector and return the Prometheus label key */ -}}
+{{- define "nrKubernetesOtel.receivers.prometheus.ksmSelector.labelKey" -}}
+    {{- $ksmSelector := .Values.receivers.prometheus.ksmSelector | default "app.kubernetes.io/name=kube-state-metrics" -}}
+    {{- $parts := split "=" $ksmSelector -}}
+    {{- mustRegexReplaceAll "[^a-zA-Z0-9_-]" $parts._0 "_" -}}
+{{- end -}}
+
+{{- /* Parse KSM selector and return the label value */ -}}
+{{- define "nrKubernetesOtel.receivers.prometheus.ksmSelector.labelValue" -}}
+    {{- $ksmSelector := .Values.receivers.prometheus.ksmSelector | default "app.kubernetes.io/name=kube-state-metrics" -}}
+    {{- $parts := split "=" $ksmSelector -}}
+    {{- $parts._1 -}}
+{{- end -}}
diff --git a/charts/nr-k8s-otel-collector/templates/deployment-configmap.yaml b/charts/nr-k8s-otel-collector/templates/deployment-configmap.yaml
@@ -28,9 +28,9 @@ data:
                 - role: pod
               relabel_configs:
                 - action: keep
-                  regex: kube-state-metrics
+                  regex: {{ include "nrKubernetesOtel.receivers.prometheus.ksmSelector.labelValue" . }}
                   source_labels:
-                  - __meta_kubernetes_pod_label_app_kubernetes_io_name
+                  - __meta_kubernetes_pod_label_{{ include "nrKubernetesOtel.receivers.prometheus.ksmSelector.labelKey" . }}
                 {{- if include "newrelic.common.openShift" . }}
                 # this is targeting our ksm deployment label app_kubernetes_io_name=kube-state-metrics
                 - action: keep
diff --git a/charts/nr-k8s-otel-collector/tests/ksm_selector_test.yaml b/charts/nr-k8s-otel-collector/tests/ksm_selector_test.yaml
@@ -0,0 +1,87 @@
+suite: ksmSelector
+templates:
+  - templates/deployment-configmap.yaml
+release:
+  name: my-release
+  namespace: my-namespace
+tests:
+  - it: uses default KSM selector when not specified
+    set:
+      cluster: my-cluster
+      licenseKey: us-whatever
+    asserts:
+      - template: templates/deployment-configmap.yaml
+        matchRegex:
+          path: data["deployment-config.yaml"]
+          pattern: |
+            ^\s*receivers:[\S\s]*prometheus/ksm:\s+config:\s+scrape_configs:\s+- job_name: kube-state-metrics\s+scrape_interval: 1m\s+kubernetes_sd_configs:\s+- role: pod\s+relabel_configs:\s+- action: keep\s+regex: kube-state-metrics\s+source_labels:\s+- __meta_kubernetes_pod_label_app_kubernetes_io_name\s+- action: replace\s+target_label: job_label\s+replacement: kube-state-metrics\s+
+
+  - it: uses custom KSM selector when specified
+    set:
+      cluster: my-cluster
+      licenseKey: us-whatever
+      receivers:
+        prometheus:
+          ksmSelector: "app.kubernetes.io/instance=my-custom-ksm"
+    asserts:
+      - template: templates/deployment-configmap.yaml
+        matchRegex:
+          path: data["deployment-config.yaml"]
+          pattern: |
+            ^\s*receivers:[\S\s]*prometheus/ksm:\s+config:\s+scrape_configs:\s+- job_name: kube-state-metrics\s+scrape_interval: 1m\s+kubernetes_sd_configs:\s+- role: pod\s+relabel_configs:\s+- action: keep\s+regex: my-custom-ksm\s+source_labels:\s+- __meta_kubernetes_pod_label_app_kubernetes_io_instance\s+- action: replace\s+target_label: job_label\s+replacement: kube-state-metrics\s+
+
+  - it: handles custom selector with different label key
+    set:
+      cluster: my-cluster
+      licenseKey: us-whatever
+      receivers:
+        prometheus:
+          ksmSelector: "custom.label/name=test-value"
+    asserts:
+      - template: templates/deployment-configmap.yaml
+        matchRegex:
+          path: data["deployment-config.yaml"]
+          pattern: |
+            ^\s*receivers:[\S\s]*prometheus/ksm:\s+config:\s+scrape_configs:\s+- job_name: kube-state-metrics\s+scrape_interval: 1m\s+kubernetes_sd_configs:\s+- role: pod\s+relabel_configs:\s+- action: keep\s+regex: test-value\s+source_labels:\s+- __meta_kubernetes_pod_label_custom_label_name\s+- action: replace\s+target_label: job_label\s+replacement: kube-state-metrics\s+
+
+  - it: handles selector with hyphens in label name
+    set:
+      cluster: my-cluster
+      licenseKey: us-whatever
+      receivers:
+        prometheus:
+          ksmSelector: "app-name=ksm-prod"
+    asserts:
+      - template: templates/deployment-configmap.yaml
+        matchRegex:
+          path: data["deployment-config.yaml"]
+          pattern: |
+            ^\s*receivers:[\S\s]*prometheus/ksm:\s+config:\s+scrape_configs:\s+- job_name: kube-state-metrics\s+scrape_interval: 1m\s+kubernetes_sd_configs:\s+- role: pod\s+relabel_configs:\s+- action: keep\s+regex: ksm-prod\s+source_labels:\s+- __meta_kubernetes_pod_label_app-name\s+- action: replace\s+target_label: job_label\s+replacement: kube-state-metrics\s+
+
+  - it: uses default selector when receivers.prometheus is set but ksmSelector is not
+    set:
+      cluster: my-cluster
+      licenseKey: us-whatever
+      receivers:
+        prometheus:
+          enabled: true
+    asserts:
+      - template: templates/deployment-configmap.yaml
+        matchRegex:
+          path: data["deployment-config.yaml"]
+          pattern: |
+            ^\s*receivers:[\S\s]*prometheus/ksm:\s+config:\s+scrape_configs:\s+- job_name: kube-state-metrics\s+scrape_interval: 1m\s+kubernetes_sd_configs:\s+- role: pod\s+relabel_configs:\s+- action: keep\s+regex: kube-state-metrics\s+source_labels:\s+- __meta_kubernetes_pod_label_app_kubernetes_io_name\s+- action: replace\s+target_label: job_label\s+replacement: kube-state-metrics\s+
+
+  - it: correctly formats label key with dots and slashes
+    set:
+      cluster: my-cluster
+      licenseKey: us-whatever
+      receivers:
+        prometheus:
+          ksmSelector: "io.kubernetes.pod.name/instance=test"
+    asserts:
+      - template: templates/deployment-configmap.yaml
+        matchRegex:
+          path: data["deployment-config.yaml"]
+          pattern: |
+            ^\s*receivers:[\S\s]*prometheus/ksm:\s+config:\s+scrape_configs:\s+- job_name: kube-state-metrics\s+scrape_interval: 1m\s+kubernetes_sd_configs:\s+- role: pod\s+relabel_configs:\s+- action: keep\s+regex: test\s+source_labels:\s+- __meta_kubernetes_pod_label_io_kubernetes_pod_name_instance\s+- action: replace\s+target_label: job_label\s+replacement: kube-state-metrics\s+
diff --git a/charts/nr-k8s-otel-collector/values.yaml b/charts/nr-k8s-otel-collector/values.yaml
@@ -256,6 +256,9 @@ receivers:
     # -- Sets the scrape interval for the `prometheus` receiver
     # @default -- `1m`
     scrapeInterval: 1m
+    # -- Label selector that will be used to automatically discover an instance of kube-state-metrics running in the cluster.
+    # @default -- `app.kubernetes.io/name=kube-state-metrics`
+    ksmSelector: "app.kubernetes.io/name=kube-state-metrics"
   k8sEvents:
     # -- (bool) Specifies whether the `k8s_events` receiver is enabled
     # @default -- `true`
