Add OTEL LIS CSI MetricsV2 integration tests (#693)

Add integration tests for Local Instance Store CSI driver metrics
validation in the OTEL container insights pipeline. Validates all 13
metrics (9 volume-scoped counters/gauge, 2 latency histograms, 2
collector-internal counters) with proper attribute assertions.

- Add test/otel/lis_csi/ test suite following EBS CSI v2 pattern
- Add terraform/eks/daemon/otel-lis-csi/ infrastructure
- Add otel/lis_csi entry to test case generator
- Split metrics into volume-scoped and collector-internal slices for
  correct volume_id/instance_id assertion scoping

Author: nathalapooja

generator/test_case_generator.go

@@ -452,6 +452,13 @@ var testTypeToTestConfig = map[string][]testConfig{
 			instanceType: "g4dn.xlarge",
 			ami:          "AL2023_x86_64_NVIDIA",
 		},
+		{
+			testDir:      "./test/otel/lis_csi",
+			terraformDir: "terraform/eks/daemon/otel-lis-csi",
+			targets:      map[string]map[string]struct{}{"arc": {"amd64": {}}},
+			instanceType: "i7i.xlarge",
+			ami:          "AL2023_x86_64_STANDARD",
+		},
 		{
 			testDir:      "./test/otel/multi_efa",
 			terraformDir: "terraform/eks/daemon/otel-multi-efa",

terraform/eks/daemon/otel-lis-csi/main.tf

@@ -0,0 +1,319 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT
+
+module "common" {
+  source             = "../../../common"
+  cwagent_image_repo = var.cwagent_image_repo
+  cwagent_image_tag  = var.cwagent_image_tag
+}
+
+module "basic_components" {
+  source = "../../../basic_components"
+  region = var.region
+}
+
+locals {
+  aws_eks = "aws eks --region ${var.region}"
+}
+
+resource "aws_eks_cluster" "this" {
+  name     = "cwagent-eks-integ-${module.common.testing_id}"
+  role_arn = module.basic_components.role_arn
+  version  = var.k8s_version
+  vpc_config {
+    subnet_ids         = module.basic_components.public_subnet_ids
+    security_group_ids = [module.basic_components.security_group]
+  }
+}
+
+# EKS Node Group
+resource "aws_eks_node_group" "this" {
+  cluster_name    = aws_eks_cluster.this.name
+  node_group_name = "cwagent-otel-liscsi-integ-node-${module.common.testing_id}"
+  node_role_arn   = aws_iam_role.node_role.arn
+  subnet_ids      = module.basic_components.public_subnet_ids
+
+  scaling_config {
+    desired_size = 1
+    max_size     = 1
+    min_size     = 1
+  }
+
+  ami_type       = var.ami_type
+  capacity_type  = "ON_DEMAND"
+  disk_size      = 20
+  instance_types = [var.instance_type]
+
+  depends_on = [
+    aws_iam_role_policy_attachment.node_AmazonEC2ContainerRegistryReadOnly,
+    aws_iam_role_policy_attachment.node_AmazonEKS_CNI_Policy,
+    aws_iam_role_policy_attachment.node_AmazonEKSWorkerNodePolicy,
+    aws_iam_role_policy_attachment.node_CloudWatchAgentServerPolicy,
+  ]
+}
+
+# EKS Node IAM Role
+resource "aws_iam_role" "node_role" {
+  name = "cwagent-otel-liscsi-eks-Worker-Role-${module.common.testing_id}"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Principal = { Service = "ec2.amazonaws.com" }
+      Action    = "sts:AssumeRole"
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "node_AmazonEKSWorkerNodePolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy"
+  role       = aws_iam_role.node_role.name
+}
+
+resource "aws_iam_role_policy_attachment" "node_AmazonEKS_CNI_Policy" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy"
+  role       = aws_iam_role.node_role.name
+}
+
+resource "aws_iam_role_policy_attachment" "node_AmazonEC2ContainerRegistryReadOnly" {
+  policy_arn = "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly"
+  role       = aws_iam_role.node_role.name
+}
+
+resource "aws_iam_role_policy_attachment" "node_CloudWatchAgentServerPolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+  role       = aws_iam_role.node_role.name
+}
+
+# Pod Identity IAM Role
+resource "aws_iam_role" "pod_identity_role" {
+  name = "cwagent-otel-liscsi-pod-identity-${module.common.testing_id}"
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Effect    = "Allow"
+      Principal = { Service = "pods.eks.amazonaws.com" }
+      Action    = ["sts:AssumeRole", "sts:TagSession"]
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "pod_identity_CloudWatchAgentServerPolicy" {
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+  role       = aws_iam_role.pod_identity_role.name
+}
+
+# --- EKS Addon: Pod Identity agent ---
+
+resource "aws_eks_addon" "pod_identity_agent" {
+  depends_on   = [aws_eks_node_group.this]
+  cluster_name = aws_eks_cluster.this.name
+  addon_name   = "eks-pod-identity-agent"
+}
+
+# --- Update kubeconfig ---
+
+resource "null_resource" "kubectl" {
+  depends_on = [aws_eks_cluster.this, aws_eks_node_group.this]
+  provisioner "local-exec" {
+    command = "${local.aws_eks} update-kubeconfig --name ${aws_eks_cluster.this.name}"
+  }
+}
+
+# --- LIS CSI addon ---
+
+resource "aws_eks_addon" "lis_csi_addon" {
+  depends_on           = [aws_eks_node_group.this]
+  cluster_name         = aws_eks_cluster.this.name
+  addon_name           = "aws-ec2-local-instance-store-csi-driver"
+  configuration_values = jsonencode({ metrics = { enabled = true } })
+}
+
+resource "null_resource" "wait_for_lis_csi" {
+  depends_on = [aws_eks_addon.lis_csi_addon, null_resource.kubectl]
+  provisioner "local-exec" {
+    command = <<-EOT
+      echo "Waiting for LIS CSI DaemonSet rollout..."
+      kubectl rollout status daemonset/ec2-instance-store-plugin -n kube-system --timeout=300s
+      echo "LIS CSI pods:"
+      kubectl get pods -n kube-system -l app.kubernetes.io/name=ec2-instance-store-plugin -o wide
+      echo "StorageClasses:"
+      kubectl get sc
+    EOT
+  }
+}
+
+# --- Helm chart install ---
+
+data "external" "clone_helm_chart" {
+  program = ["bash", "-c", <<-EOT
+    rm -rf ./helm-charts
+    git clone -b ${var.helm_chart_branch} https://github.com/aws-observability/helm-charts.git ./helm-charts
+    echo '{"status":"ready"}'
+  EOT
+  ]
+}
+
+resource "helm_release" "aws_observability" {
+  name             = "amazon-cloudwatch-observability"
+  chart            = "./helm-charts/charts/amazon-cloudwatch-observability"
+  namespace        = "amazon-cloudwatch"
+  create_namespace = true
+
+  set = [
+    { name = "clusterName", value = aws_eks_cluster.this.name },
+    { name = "region", value = var.region }
+  ]
+
+  depends_on = [
+    aws_eks_addon.pod_identity_agent,
+    null_resource.kubectl,
+    data.external.clone_helm_chart,
+  ]
+}
+
+# --- Pod Identity association (after Helm creates the service account) ---
+
+resource "aws_eks_pod_identity_association" "cloudwatch_agent" {
+  depends_on      = [helm_release.aws_observability]
+  cluster_name    = aws_eks_cluster.this.name
+  namespace       = "amazon-cloudwatch"
+  service_account = "cloudwatch-agent"
+  role_arn        = aws_iam_role.pod_identity_role.arn
+}
+
+# --- Patch agent image ---
+
+resource "null_resource" "update_image" {
+  depends_on = [helm_release.aws_observability, null_resource.kubectl]
+  triggers   = { timestamp = timestamp() }
+  provisioner "local-exec" {
+    command = <<-EOT
+      sleep 30
+      kubectl -n amazon-cloudwatch patch AmazonCloudWatchAgent cloudwatch-agent --type='json' \
+        -p='[{"op": "replace", "path": "/spec/image", "value": "${var.cwagent_image_repo}:${var.cwagent_image_tag}"}]'
+      sleep 10
+    EOT
+  }
+}
+
+# --- Restart pods to pick up Pod Identity + new image ---
+
+resource "null_resource" "restart_pods" {
+  depends_on = [aws_eks_pod_identity_association.cloudwatch_agent, null_resource.update_image]
+  triggers   = { timestamp = timestamp() }
+  provisioner "local-exec" {
+    command = <<-EOT
+      kubectl -n amazon-cloudwatch rollout restart daemonset/cloudwatch-agent
+      kubectl -n amazon-cloudwatch rollout status daemonset/cloudwatch-agent --timeout=120s
+    EOT
+  }
+}
+
+# --- Test workload: nginx ---
+
+resource "kubernetes_deployment_v1" "nginx_test" {
+  depends_on = [aws_eks_node_group.this]
+  metadata {
+    name      = "nginx-test"
+    namespace = "default"
+  }
+  spec {
+    replicas = 1
+    selector { match_labels = { app = "nginx-test" } }
+    template {
+      metadata { labels = { app = "nginx-test" } }
+      spec {
+        container {
+          name  = "nginx"
+          image = "public.ecr.aws/nginx/nginx:latest"
+          port { container_port = 80 }
+        }
+      }
+    }
+  }
+}
+
+# --- LIS CSI IO workload ---
+
+resource "kubernetes_deployment_v1" "lis_csi_io_workload" {
+  depends_on = [null_resource.wait_for_lis_csi]
+  metadata {
+    name      = "liscsi-integ-test-io-workload"
+    namespace = "default"
+    labels = {
+      app = "liscsi-integ-test"
+    }
+  }
+  spec {
+    replicas = 1
+    selector {
+      match_labels = {
+        app = "liscsi-integ-test"
+      }
+    }
+    template {
+      metadata {
+        labels = {
+          app = "liscsi-integ-test"
+        }
+      }
+      spec {
+        container {
+          name    = "liscsi-integ-test-writer"
+          image   = "busybox:1.35"
+          command = ["sh", "-c", "while true; do dd if=/dev/zero of=/data/out.txt bs=1M count=10; sleep 5; done"]
+          volume_mount {
+            name       = "lis-storage"
+            mount_path = "/data"
+          }
+        }
+        volume {
+          name = "lis-storage"
+          ephemeral {
+            volume_claim_template {
+              spec {
+                access_modes       = ["ReadWriteOnce"]
+                storage_class_name = "ec2-instance-store-sc"
+                resources {
+                  requests = {
+                    storage = "1Gi"
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+# --- Test runner ---
+
+resource "null_resource" "validator" {
+  depends_on = [
+    null_resource.restart_pods,
+    null_resource.wait_for_lis_csi,
+    kubernetes_deployment_v1.nginx_test,
+    kubernetes_deployment_v1.lis_csi_io_workload,
+  ]
+
+  triggers = { always_run = timestamp() }
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      echo "Running OTEL LIS CSI integration tests"
+      cd ../../../..
+
+      echo "Waiting 3 minutes for metrics to propagate..."
+      sleep 180
+
+      go test -tags integration -timeout 1h -v ${var.test_dir} \
+        -eksClusterName=${aws_eks_cluster.this.name} \
+        -computeType=EKS \
+        -eksDeploymentStrategy=DAEMON \
+        -region=${var.region}
+    EOT
+  }
+}

terraform/eks/daemon/otel-lis-csi/providers.tf

@@ -0,0 +1,28 @@
+# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+# SPDX-License-Identifier: MIT
+
+provider "aws" {
+  region = var.region
+}
+
+provider "kubernetes" {
+  host                   = aws_eks_cluster.this.endpoint
+  cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data)
+  exec {
+    api_version = "client.authentication.k8s.io/v1beta1"
+    command     = "aws"
+    args        = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name]
+  }
+}
+
+provider "helm" {
+  kubernetes = {
+    host                   = aws_eks_cluster.this.endpoint
+    cluster_ca_certificate = base64decode(aws_eks_cluster.this.certificate_authority.0.data)
+    exec = {
+      api_version = "client.authentication.k8s.io/v1beta1"
+      command     = "aws"
+      args        = ["eks", "get-token", "--cluster-name", aws_eks_cluster.this.name]
+    }
+  }
+}