[nr-k8s-otel-collector] drop cadvisor metrics from intermediate levels of cgroup hierarchy (#2130)

<!--
Thank you for contributing to New Relic's Helm charts. Before you submit
this PR we'd like to
make sure you are aware of our technical requirements:

*
https://github.com/newrelic-experimental/helm-charts/blob/master/CONTRIBUTING.md#technical-requirements

For a quick overview across what we will look at reviewing your PR,
please read
our review guidelines:

*
https://github.com/newrelic-experimental/helm-charts/blob/master/REVIEW_GUIDELINES.md

Following our best practices right from the start will accelerate the
review process and
help get your PR merged quicker.

When updates to your PR are requested, please add new commits and do not
squash the
history. This will make it easier to identify new changes. The PR will
be squashed
anyways when it is merged. Thanks.

For fast feedback, please @-mention maintainers that are listed in the
Chart.yaml file.

Please make sure you test your changes before you push them. Once
pushed, a Github Action
will run across your changes and do some initial checks and linting.
These checks run
very quickly. Please check the results. We would like these checks to
pass before we
even continue reviewing your changes.
-->
#### What this PR does / why we need it:
https://new-relic.atlassian.net/browse/NR-411922
drop cadvisor metrics from intermediate levels of cgroup hierarchy. here
is an example of the filtered metrics for
`container_cpu_usage_seconds_total` from my test cluster in GKE:
https://onenr.io/0Vwg6z6bBQJ
an example of the old output for the same query can be found in the
story.

#### Which issue this PR fixes
*(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)`
format, will close that issue when PR gets merged)*
  - fixes #411922

#### Special notes for your reviewer:

#### Checklist
[Place an '[x]' (no spaces) in all applicable fields. Please remove
unrelated fields.]
- [x] Chart Version bumped
- [x] Variables are documented in the README.md
- [x] Title of the PR starts with chart name (e.g. `[mychartname]`)

# Release Notes to Publish (nr-k8s-otel-collector)
If this PR contains changes in `nr-k8s-otel-collector`, please complete
the following section. All other charts should ignore this section.

<!--BEGIN-RELEASE-NOTES-->
## 🚀 What's Changed
* Added filters to drop cadvisor metrics from intermediate levels of
cgroup hierarchy
<!--END-RELEASE-NOTES-->

Author: jamescripter

.github/workflows/nr-k8s-otel-e2e.yml

@@ -32,6 +32,7 @@ jobs:
           kubernetes version: ${{ env.LATEST_K8S_VERSION }}
           # default driver doesn't support 'eval $$(minikube docker-env)'.
           driver: docker
+          container runtime: containerd
           github token: ${{ secrets.GITHUB_TOKEN }}
       - name: Run e2e-test
         uses: newrelic/newrelic-integration-e2e-action@v1

charts/nr-k8s-otel-collector/Chart.yaml

@@ -17,7 +17,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.10.8
+version: 0.10.9
 
 
 dependencies:

charts/nr-k8s-otel-collector/examples/k8s-with-atp/rendered/clusterrole.yaml

@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.10.8
+    helm.sh/chart: nr-k8s-otel-collector-0.10.9
 rules:
   - apiGroups:
     - ""

charts/nr-k8s-otel-collector/examples/k8s-with-atp/rendered/clusterrolebinding.yaml

@@ -9,7 +9,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.10.8
+    helm.sh/chart: nr-k8s-otel-collector-0.10.9
 subjects:
   - kind: ServiceAccount
     name: nr-k8s-otel-collector

charts/nr-k8s-otel-collector/examples/k8s-with-atp/rendered/daemonset-configmap.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.10.8
+    helm.sh/chart: nr-k8s-otel-collector-0.10.9
 data:
   daemonset-config.yaml: |
     receivers:
@@ -119,6 +119,31 @@ data:
                 - source_labels: [__meta_kubernetes_node_name]
                   regex: ${KUBE_NODE_NAME}
                   action: keep
+              metric_relabel_configs:
+                # Init temporary label is_network_metric for all metrics
+                - source_labels: [__name__]
+                  regex: '.*'
+                  target_label: "is_network_metric"
+                  replacement: "false"
+                - source_labels: [__name__]
+                  regex: '^container_network_.*'
+                  target_label: "is_network_metric"
+                  replacement: "true"
+
+                # Drop metrics without container/image/name for non-network metrics
+                - source_labels: [is_network_metric, container]
+                  regex: '^false;$'
+                  action: drop
+                - source_labels: [is_network_metric, image]
+                  regex: '^false;$'
+                  action: drop
+                - source_labels: [is_network_metric, name]
+                  regex: '^false;$'
+                  action: drop
+
+                # Remove temporary label
+                - regex: 'is_network_metric'
+                  action: labeldrop
               scheme: https
               tls_config:
                 ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt
@@ -618,7 +643,7 @@ data:
             value: <cluser_name>
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.10.8
+            value: 0.10.9
           - key: newrelic.entity.type
             action: upsert
             value: "k8s"

charts/nr-k8s-otel-collector/examples/k8s-with-atp/rendered/daemonset.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.10.8
+    helm.sh/chart: nr-k8s-otel-collector-0.10.9
 spec:
   selector:
     matchLabels:
@@ -24,7 +24,7 @@ spec:
         app.kubernetes.io/name: nr-k8s-otel-collector
         component: daemonset
       annotations:
-        checksum/config: fa6441edefae88d7fa03cc9fd4a8be40e40ea734040fc5cb6361f5a7130a002b
+        checksum/config: a9f3ac9805c7551475c311c03335abfbea0fd6ca3c1200d79461acbd7928d3e6
     spec:
       serviceAccountName: nr-k8s-otel-collector
       initContainers:

charts/nr-k8s-otel-collector/examples/k8s-with-atp/rendered/deployment-configmap.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.10.8
+    helm.sh/chart: nr-k8s-otel-collector-0.10.9
 data:
   deployment-config.yaml: |
     receivers:
@@ -509,7 +509,7 @@ data:
             value: <cluser_name>
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.10.8
+            value: 0.10.9
           - key: newrelic.entity.type
             action: upsert
             value: "k8s"
@@ -527,7 +527,7 @@ data:
             value: <cluser_name>
           - key: "newrelic.chart.version"
             action: upsert
-            value: 0.10.8
+            value: 0.10.9
 
       transform/events:
         log_statements:

charts/nr-k8s-otel-collector/examples/k8s-with-atp/rendered/deployment.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.10.8
+    helm.sh/chart: nr-k8s-otel-collector-0.10.9
 spec:
   replicas: 1
   minReadySeconds: 5
@@ -26,7 +26,7 @@ spec:
         app.kubernetes.io/name: nr-k8s-otel-collector
         component: deployment
       annotations:
-        checksum/config: d38b8053af426b90b8ee9f7a683d00963f69800b375c423f523ab0256812267c
+        checksum/config: a3a6e91c37d1b9b5339877f8feed53a44a944f1abf7a79f11d9af202e1842d32
     spec:
       serviceAccountName: nr-k8s-otel-collector
       containers:

charts/nr-k8s-otel-collector/examples/k8s-with-atp/rendered/secret.yaml

@@ -10,6 +10,6 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.10.8
+    helm.sh/chart: nr-k8s-otel-collector-0.10.9
 data:
   licenseKey: PE5SX2xpY2Vuc2VLZXk+

charts/nr-k8s-otel-collector/examples/k8s-with-atp/rendered/service.yaml

@@ -10,7 +10,7 @@ metadata:
     app.kubernetes.io/managed-by: Helm
     app.kubernetes.io/name: nr-k8s-otel-collector
     app.kubernetes.io/version: 1.2.0
-    helm.sh/chart: nr-k8s-otel-collector-0.10.8
+    helm.sh/chart: nr-k8s-otel-collector-0.10.9
 spec:
   type: ClusterIP
   ports: