Support OTel Container Insights on AKS (#348)

jefchien · web-flow · commit 58182565546e · 2026-08-05T14:45:15.000-04:00
diff --git a/.github/workflows/amazon-cloudwatch-observability-integration-test.yaml b/.github/workflows/amazon-cloudwatch-observability-integration-test.yaml
@@ -44,6 +44,7 @@ jobs:
         # Keep sorted alphabetically.
         scenario:
           - agent-config-merge-isolation
+          - aks-otel-container-insights
           - appsignals-disabled
           - appsignals-disabled-multi-agents
           - appsignals-enabled-multi-agents
diff --git a/charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-cluster-scraper-config.tpl b/charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-cluster-scraper-config.tpl
@@ -291,6 +291,26 @@ processors:
           - set(resource.attributes["k8s.component.name"], attributes["component"])
 
   resourcedetection/cw_k8s_ci_v0:
+    {{- if eq .Values.k8sMode "AKS" }}
+    detectors: [aks, azure]
+    aks:
+      resource_attributes:
+        cloud.platform: { enabled: true }
+        cloud.provider: { enabled: true }
+        k8s.cluster.name: { enabled: false }
+    azure:
+      resource_attributes:
+        azure.resourcegroup.name: { enabled: true }
+        azure.vm.name: { enabled: false }
+        azure.vm.scaleset.name: { enabled: false }
+        azure.vm.size: { enabled: false }
+        cloud.account.id: { enabled: true }
+        cloud.platform: { enabled: true }
+        cloud.provider: { enabled: true }
+        cloud.region: { enabled: true }
+        host.id: { enabled: false }
+        host.name: { enabled: false }
+    {{- else }}
     detectors: [eks, ec2]
     ec2:
       resource_attributes:
@@ -303,6 +323,7 @@ processors:
         cloud.region: { enabled: true }
         cloud.availability_zone: { enabled: true }
         cloud.account.id: { enabled: true }
+    {{- end }}
 
   nodemetadataenricher/cw_k8s_ci_v0: {}
 
@@ -340,8 +361,19 @@ processors:
     metric_statements:
       - context: resource
         statements:
+          {{- if eq .Values.k8sMode "AKS" }}
+          {{/* azure.resourcegroup.name is the node's group, not the cluster's: AKS nodes live in an
+               auto-generated MC_<clusterResourceGroup>_<cluster>_<region> group. */}}
+          - set(resource.attributes["_tmp.azure.resourcegroup.name"], resource.attributes["azure.resourcegroup.name"])
+            where resource.attributes["azure.resourcegroup.name"] != nil
+          - replace_pattern(resource.attributes["_tmp.azure.resourcegroup.name"], "^MC_(.+)_{{ .Values.clusterName }}_[^_]+$", "$$$1")
+          - set(resource.attributes["cloud.resource_id"], Concat(["/subscriptions/", resource.attributes["cloud.account.id"], "/resourceGroups/", resource.attributes["_tmp.azure.resourcegroup.name"], "/providers/Microsoft.ContainerService/managedClusters/", resource.attributes["k8s.cluster.name"]], ""))
+            where resource.attributes["cloud.account.id"] != nil and resource.attributes["_tmp.azure.resourcegroup.name"] != nil and resource.attributes["k8s.cluster.name"] != nil
+          - delete_key(resource.attributes, "_tmp.azure.resourcegroup.name")
+          {{- else }}
           - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:eks:", resource.attributes["cloud.region"], ":", resource.attributes["cloud.account.id"], ":cluster/", resource.attributes["k8s.cluster.name"]], ""))
             where resource.attributes["cloud.region"] != nil and resource.attributes["cloud.account.id"] != nil and resource.attributes["k8s.cluster.name"] != nil
+          {{- end }}
 
   batch/cw_k8s_ci_v0_cwotel:
     send_batch_size: 500
diff --git a/charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-config.tpl b/charts/amazon-cloudwatch-observability/templates/linux/_otel-container-insights-config.tpl
@@ -374,6 +374,26 @@ processors:
           - set(resource.attributes["k8s.node.name"], attributes["node_name"]) where attributes["node_name"] != nil
 
   resourcedetection/cw_k8s_ci_v0:
+    {{- if eq .Values.k8sMode "AKS" }}
+    detectors: [aks, azure]
+    aks:
+      resource_attributes:
+        cloud.platform: { enabled: true }
+        cloud.provider: { enabled: true }
+        k8s.cluster.name: { enabled: false }
+    azure:
+      resource_attributes:
+        azure.resourcegroup.name: { enabled: true }
+        azure.vm.name: { enabled: true }
+        azure.vm.scaleset.name: { enabled: true }
+        azure.vm.size: { enabled: true }
+        cloud.account.id: { enabled: true }
+        cloud.platform: { enabled: true }
+        cloud.provider: { enabled: true }
+        cloud.region: { enabled: true }
+        host.id: { enabled: true }
+        host.name: { enabled: true }
+    {{- else }}
     detectors: [eks, ec2]
     ec2:
       resource_attributes:
@@ -386,6 +406,7 @@ processors:
         cloud.region: { enabled: true }
         cloud.availability_zone: { enabled: true }
         cloud.account.id: { enabled: true }
+    {{- end }}
 
   transform/cw_k8s_ci_v0_clear_schema_url:
     error_mode: ignore
@@ -464,8 +485,19 @@ processors:
     metric_statements:
       - context: resource
         statements:
+          {{- if eq .Values.k8sMode "AKS" }}
+          {{/* azure.resourcegroup.name is the node's group, not the cluster's: AKS nodes live in an
+               auto-generated MC_<clusterResourceGroup>_<cluster>_<region> group. */}}
+          - set(resource.attributes["_tmp.azure.resourcegroup.name"], resource.attributes["azure.resourcegroup.name"])
+            where resource.attributes["azure.resourcegroup.name"] != nil
+          - replace_pattern(resource.attributes["_tmp.azure.resourcegroup.name"], "^MC_(.+)_{{ .Values.clusterName }}_[^_]+$", "$$$1")
+          - set(resource.attributes["cloud.resource_id"], Concat(["/subscriptions/", resource.attributes["cloud.account.id"], "/resourceGroups/", resource.attributes["_tmp.azure.resourcegroup.name"], "/providers/Microsoft.ContainerService/managedClusters/", resource.attributes["k8s.cluster.name"]], ""))
+            where resource.attributes["cloud.account.id"] != nil and resource.attributes["_tmp.azure.resourcegroup.name"] != nil and resource.attributes["k8s.cluster.name"] != nil
+          - delete_key(resource.attributes, "_tmp.azure.resourcegroup.name")
+          {{- else }}
           - set(resource.attributes["cloud.resource_id"], Concat(["arn:aws:eks:", resource.attributes["cloud.region"], ":", resource.attributes["cloud.account.id"], ":cluster/", resource.attributes["k8s.cluster.name"]], ""))
             where resource.attributes["cloud.region"] != nil and resource.attributes["cloud.account.id"] != nil and resource.attributes["k8s.cluster.name"] != nil
+          {{- end }}
 
   metricstarttime/cw_k8s_ci_v0:
 
@@ -486,6 +518,11 @@ processors:
           - set(resource.attributes["k8s.workload.type"], "CronJob") where resource.attributes["k8s.cronjob.name"] != nil and resource.attributes["k8s.workload.type"] == nil
           - set(resource.attributes["k8s.workload.name"], resource.attributes["k8s.replicaset.name"]) where resource.attributes["k8s.workload.name"] == nil and resource.attributes["k8s.replicaset.name"] != nil
           - set(resource.attributes["k8s.workload.type"], "ReplicaSet") where resource.attributes["k8s.replicaset.name"] != nil and resource.attributes["k8s.workload.type"] == nil
+          {{- if eq .Values.k8sMode "AKS" }}
+          - set(resource.attributes["service.name"], resource.attributes["k8s.workload.name"]) where resource.attributes["service.name"] == nil and resource.attributes["k8s.workload.name"] != nil
+          - set(resource.attributes["service.namespace"], resource.attributes["k8s.namespace.name"]) where resource.attributes["service.namespace"] == nil and resource.attributes["k8s.namespace.name"] != nil
+          - set(resource.attributes["deployment.environment.name"], Concat([resource.attributes["cloud.platform"], Concat([resource.attributes["k8s.cluster.name"], resource.attributes["k8s.namespace.name"]], "/")], ":")) where resource.attributes["deployment.environment.name"] == nil and resource.attributes["cloud.platform"] != nil and resource.attributes["k8s.cluster.name"] != nil and resource.attributes["k8s.namespace.name"] != nil
+          {{- end }}
 
   batch/cw_k8s_ci_v0_metrics_dest:
     send_batch_size: 500
@@ -667,8 +704,13 @@ processors:
           - set(attributes["k8s.workload.name"], attributes["k8s.replicaset.name"]) where attributes["k8s.workload.name"] == nil and attributes["k8s.replicaset.name"] != nil
           - set(attributes["k8s.workload.type"], "ReplicaSet") where attributes["k8s.replicaset.name"] != nil and attributes["k8s.workload.type"] == nil
           # Derive service.name from k8s.workload.name (OTEL logs semconv).
-          # Logs need service.name; metrics use k8s.workload.name directly.
+          # Logs need service.name; metrics use k8s.workload.name directly, except on
+          # AKS — see transform/cw_k8s_ci_v0_set_workload.
           - set(attributes["service.name"], attributes["k8s.workload.name"]) where attributes["service.name"] == nil and attributes["k8s.workload.name"] != nil
+          {{- if eq .Values.k8sMode "AKS" }}
+          - set(attributes["service.namespace"], attributes["k8s.namespace.name"]) where attributes["service.namespace"] == nil and attributes["k8s.namespace.name"] != nil
+          - set(attributes["deployment.environment.name"], Concat([attributes["cloud.platform"], Concat([attributes["k8s.cluster.name"], attributes["k8s.namespace.name"]], "/")], ":")) where attributes["deployment.environment.name"] == nil and attributes["cloud.platform"] != nil and attributes["k8s.cluster.name"] != nil and attributes["k8s.namespace.name"] != nil
+          {{- end }}
 
   transform/cw_k8s_ci_v0_logs_set_cluster_and_node:
     error_mode: ignore
@@ -683,8 +725,18 @@ processors:
     log_statements:
       - context: resource
         statements:
+          {{- if eq .Values.k8sMode "AKS" }}
+          {{/* Node group -> cluster group, as in transform/cw_k8s_ci_v0_set_cloud_resource_id. */}}
+          - set(attributes["_tmp.azure.resourcegroup.name"], attributes["azure.resourcegroup.name"])
+            where attributes["azure.resourcegroup.name"] != nil
+          - replace_pattern(attributes["_tmp.azure.resourcegroup.name"], "^MC_(.+)_{{ .Values.clusterName }}_[^_]+$", "$$$1")
+          - set(attributes["cloud.resource_id"], Concat(["/subscriptions/", attributes["cloud.account.id"], "/resourceGroups/", attributes["_tmp.azure.resourcegroup.name"], "/providers/Microsoft.ContainerService/managedClusters/", attributes["k8s.cluster.name"]], ""))
+            where attributes["cloud.account.id"] != nil and attributes["_tmp.azure.resourcegroup.name"] != nil and attributes["k8s.cluster.name"] != nil
+          - delete_key(attributes, "_tmp.azure.resourcegroup.name")
+          {{- else }}
           - set(attributes["cloud.resource_id"], Concat(["arn:aws:eks:", attributes["cloud.region"], ":", attributes["cloud.account.id"], ":cluster/", attributes["k8s.cluster.name"]], ""))
             where attributes["cloud.region"] != nil and attributes["cloud.account.id"] != nil and attributes["k8s.cluster.name"] != nil
+          {{- end }}
 
   transform/cw_k8s_ci_v0_logs_clear_schema_url:
     error_mode: ignore
diff --git a/integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/aks-otel-container-insights/main.tf b/integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/aks-otel-container-insights/main.tf
@@ -0,0 +1,26 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+module "base" {
+  source           = "../.."
+  helm_values_file = "${path.module}/values.yaml"
+  helm_dir         = var.helm_dir
+}
+
+resource "null_resource" "validator" {
+  depends_on = [module.base.helm_release]
+
+  provisioner "local-exec" {
+    command = "go test ${var.test_dir} -v -run=TestAKSOtelContainerInsights"
+  }
+}
+
+variable "test_dir" {
+  type    = string
+  default = "../../../../validations/minikube/scenarios"
+}
+
+variable "helm_dir" {
+  type    = string
+  default = "../../../../../../charts/amazon-cloudwatch-observability"
+}
diff --git a/integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/aks-otel-container-insights/values.yaml b/integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/aks-otel-container-insights/values.yaml
@@ -0,0 +1,14 @@
+k8sMode: AKS
+# roleArn is required when k8sMode is AKS. The agent pods will not become healthy on minikube
+# (no Azure IMDS / workload identity), but the AmazonCloudWatchAgent CRs are rendered by the chart
+# and applied by helm regardless — the operator does not gate the helm release on pod readiness —
+# so the test validates the AKS-specific config baked into the CRs.
+roleArn: arn:aws:iam::123456789012:role/CloudWatchAgentServerRole
+region: us-east-1
+clusterName: minikube
+# Exercise the OTEL CI path (metrics + logs), which is where the AKS resource-detection and
+# cloud.resource_id derivation live.
+otelContainerInsights:
+  enabled: true
+  logs:
+    enabled: true
diff --git a/integration-tests/amazon-cloudwatch-observability/validations/minikube/scenarios/aks_otel_container_insights_test.go b/integration-tests/amazon-cloudwatch-observability/validations/minikube/scenarios/aks_otel_container_insights_test.go
diff --git a/integration-tests/amazon-cloudwatch-observability/validations/minikube/scenarios/default_test.go b/integration-tests/amazon-cloudwatch-observability/validations/minikube/scenarios/default_test.go
