fix: Filter generic client-go metrics from KEDA pipeline

Drop rest_client_*, apiserver_*, workqueue_*, process_*, and go_* metrics
from the KEDA scrape pipeline. These are generic Go runtime/client-go
metrics that conflict with the control plane (apiserver) pipeline.

Also removes Karpenter config from this branch (handled in separate PR)
and removes conditional guard on cloud.resource_id ARN construction.

Author: vaishnavi-30-beep

charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-cluster-scraper-config.tpl

@@ -54,6 +54,34 @@ receivers:
                 - {{ include "kube-state-metrics.name" . }}.{{ .Release.Namespace }}.svc:{{ .Values.kubeStateMetrics.service.port }}
 {{- end }}
 
+{{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.karpenter.enabled }}
+  prometheus/cw_k8s_ci_v0_karpenter:
+    config:
+      scrape_configs:
+        - job_name: karpenter
+          scrape_interval: {{ .Values.otelContainerInsights.metricResolution }}
+          scrape_timeout: {{ include "otel-container-insights.scrapeTimeout" . }}
+          metrics_path: /metrics
+          kubernetes_sd_configs:
+            - role: pod
+              namespaces:
+                names:
+                  - {{ .Values.otelContainerInsights.solutions.karpenter.namespace }}
+          relabel_configs:
+            - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
+              regex: karpenter
+              action: keep
+            - source_labels: [__meta_kubernetes_pod_container_port_name]
+              regex: http-metrics
+              action: keep
+            - source_labels: [__meta_kubernetes_pod_name]
+              target_label: pod
+            - source_labels: [__meta_kubernetes_namespace]
+              target_label: namespace
+            - source_labels: [__meta_kubernetes_pod_node_name]
+              target_label: node
+{{- end }}
+
 {{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.keda.enabled }}
   prometheus/cw_k8s_ci_v0_keda:
     config:
@@ -168,7 +196,70 @@ processors:
           - set(attributes["cloudwatch.pipeline"], "kube-state-metrics")
 {{- end }}
 
+{{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.karpenter.enabled }}
+  transform/cw_k8s_ci_v0_set_scope_karpenter:
+    error_mode: ignore
+    metric_statements:
+      - context: scope
+        statements:
+          - set(scope.name, "github.com/aws/karpenter")
+          - set(scope.schema_url, "")
+          - set(attributes["cloudwatch.source"], "cloudwatch-agent")
+          - set(attributes["cloudwatch.solution"], "k8s-otel-container-insights")
+          - set(attributes["cloudwatch.pipeline"], "karpenter")
+
+  # Promote scraped pod/namespace/node labels to OTel K8s resource attributes.
+  # This allows the shared k8sattributes processor to enrich with deployment/workload info.
+  groupbyattrs/cw_k8s_ci_v0_karpenter:
+    keys:
+      - pod
+      - namespace
+      - node
+
+  transform/cw_k8s_ci_v0_karpenter_promote:
+    error_mode: ignore
+    metric_statements:
+      - context: resource
+        statements:
+          - set(attributes["k8s.pod.name"], attributes["pod"]) where attributes["pod"] != nil
+          - set(attributes["k8s.namespace.name"], attributes["namespace"]) where attributes["namespace"] != nil
+          - set(attributes["k8s.node.name"], attributes["node"]) where attributes["node"] != nil
+          # Remove deprecated/unwanted attributes auto-injected by the Prometheus receiver.
+          - delete_key(attributes, "net.host.name") where attributes["net.host.name"] != nil
+          - delete_key(attributes, "net.host.port") where attributes["net.host.port"] != nil
+          - delete_key(attributes, "url.scheme") where attributes["url.scheme"] != nil
+      - context: datapoint
+        statements:
+          - set(attributes["pod"], resource.attributes["pod"]) where resource.attributes["pod"] != nil
+          - set(attributes["namespace"], resource.attributes["namespace"]) where resource.attributes["namespace"] != nil
+          - set(attributes["node"], resource.attributes["node"]) where resource.attributes["node"] != nil
+
+  # Karpenter-specific resource detection: only cloud-level attributes (region, account).
+  # No host/AZ attributes — those would incorrectly reflect the scraper's node, not Karpenter's.
+  resourcedetection/cw_k8s_ci_v0_karpenter:
+    detectors: [eks, ec2]
+    ec2:
+      resource_attributes:
+        host.id: { enabled: false }
+        host.type: { enabled: false }
+        host.name: { enabled: false }
+        host.image.id: { enabled: false }
+        cloud.provider: { enabled: true }
+        cloud.platform: { enabled: true }
+        cloud.region: { enabled: true }
+        cloud.availability_zone: { enabled: false }
+        cloud.account.id: { enabled: true }
+{{- end }}
+
 {{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.keda.enabled }}
+  # Drop generic client-go / control-plane metrics that KEDA's Go runtime exposes.
+  # These conflict with the apiserver pipeline which owns them with a different scope.
+  filter/cw_k8s_ci_v0_keda_drop_non_keda:
+    error_mode: ignore
+    metrics:
+      metric:
+        - IsMatch(name, "^(rest_client_.*|apiserver_.*|workqueue_.*|process_.*|go_.*)$")
+
   transform/cw_k8s_ci_v0_set_scope_keda:
     error_mode: ignore
     metric_statements:
@@ -527,11 +618,33 @@ service:
       exporters:
         - otlphttp/cw_k8s_ci_v0_cwotel
 {{- end }}
+{{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.karpenter.enabled }}
+    metrics/cw_k8s_ci_v0_karpenter:
+      receivers: [prometheus/cw_k8s_ci_v0_karpenter]
+      processors:
+        - filter/cw_k8s_ci_v0_scrape_metadata
+        - transform/cw_k8s_ci_v0_set_unit
+        - metricstarttime/cw_k8s_ci_v0
+        - transform/cw_k8s_ci_v0_set_scope_karpenter
+        - transform/cw_k8s_ci_v0_set_cluster_name
+        - groupbyattrs/cw_k8s_ci_v0_karpenter
+        - transform/cw_k8s_ci_v0_karpenter_promote
+        - k8sattributes/cw_k8s_ci_v0_pod
+        - transform/cw_k8s_ci_v0_set_workload
+        - resourcedetection/cw_k8s_ci_v0_karpenter
+        - transform/cw_k8s_ci_v0_clear_schema_url
+        - transform/cw_k8s_ci_v0_set_cloud_resource_id
+        - awsattributelimit/cw_k8s_ci_v0
+        - batch/cw_k8s_ci_v0_cwotel
+      exporters:
+        - otlphttp/cw_k8s_ci_v0_cwotel
+{{- end }}
 {{- if and .Values.otelContainerInsights.solutions.enabled .Values.otelContainerInsights.solutions.keda.enabled }}
     metrics/cw_k8s_ci_v0_keda:
       receivers: [prometheus/cw_k8s_ci_v0_keda]
       processors:
         - filter/cw_k8s_ci_v0_scrape_metadata
+        - filter/cw_k8s_ci_v0_keda_drop_non_keda
         - transform/cw_k8s_ci_v0_set_unit
         - metricstarttime/cw_k8s_ci_v0
         - transform/cw_k8s_ci_v0_set_scope_keda

charts/amazon-cloudwatch-observability/values.yaml

@@ -1182,6 +1182,12 @@ otelContainerInsights:
     ## Global toggle: set to false to disable ALL solutions at once.
     ## Individual solutions won't activate even if their own enabled flag is true.
     enabled: true
+    ## Karpenter — cluster autoscaler for Kubernetes.
+    ## Scrapes the Prometheus metrics endpoint exposed by the Karpenter controller.
+    karpenter:
+      enabled: true
+      ## Namespace where Karpenter is deployed.
+      namespace: "kube-system"
     ## KEDA — Kubernetes Event-Driven Autoscaling.
     ## Scrapes the Prometheus metrics endpoint exposed by the KEDA operator.
     ## Requires prometheus.operator.enabled=true in KEDA's Helm values.