Assert the derived log stream and clean up test log destinations on success

Log validation scanned every stream in the group and passed if any one of them
matched. That grew with each new VM writing to the shared group, and it could not
tell a routing bug from a delivery failure -- a stream named wrongly was simply
skipped. The agent derives the stream as {host.id}/{service.name}, both of which
the test already knows, so it now asserts that exact name and the check covers
routing as well as delivery.

Both suites now also remove the log destination they created, but only when
validation passed, so a failing run leaves its logs behind as evidence. The VM
deletes just its own stream because /aws/cwagent/otlp is shared across runs; the
AKS group name carries the per-run cluster, so the whole group goes. Neither
touches aws/spans, which is service-managed and shared with other suites.

The VM role gains logs:DeleteLogStream since that test runs on the VM under it.
The AKS test runs on the runner under the runner's own credentials.

Author: movence

terraform/azure/vm/iam.tf

@@ -67,6 +67,8 @@ data "aws_iam_policy_document" "cwagent_permissions" {
       "cloudwatch:ListMetrics",
       "cloudwatch:GetMetricData",
       "logs:GetLogEvents",
+      # Cleanup: the test deletes its own stream from the shared /aws/cwagent/otlp group when it finishes.
+      "logs:DeleteLogStream",
       # StartQuery/GetQueryResults validate OTLP trace delivery via the aws/spans log group. That group
       # is only populated where the X-Ray trace segment destination is set to CloudWatchLogs, which is a
       # per-region setting -- hence the region default in variables.tf.

test/azure/aks/aks_test.go

@@ -82,6 +82,13 @@ func validateLogs() status.TestResult {
 	// where the agent's identity transform fills service.namespace from k8s.namespace.name.
 	// AssertLogsNotEmpty guards against a vacuous pass on an empty window.
 	logGroup := fmt.Sprintf("/aws/cwagent/%s/otlp", env.AKSClusterName)
+	// Clean up only on success: the group name carries this run's cluster so the whole group is
+	// disposable, but on failure it is left in place as evidence for whoever debugs the run.
+	defer func() {
+		if testResult.Status == status.SUCCESSFUL {
+			awsservice.DeleteLogGroup(logGroup)
+		}
+	}()
 	logStream := fmt.Sprintf("amazon-cloudwatch/amazon-cloudwatch/%s", serviceName)
 	marker := fmt.Sprintf("aks_otlp_log_%s", env.AKSClusterName)
 	const maxRetries = 4

test/azure/vm/azurevm_test.go

@@ -118,36 +118,39 @@ func TestAzureVM(t *testing.T) {
 
 func measuredMetrics() []string { return []string{"azurevm_otlp_counter", "azurevm_otlp_gauge"} }
 
-// validateLogs confirms the OTLP log record landed in the default:otel log group for this host.
+// validateLogs confirms the OTLP log record landed in the default:otel log group on the stream the
+// agent's log routing is expected to derive for this host.
 func validateLogs() status.TestResult {
 	testResult := status.TestResult{Name: "AzureVM_Logs", Status: status.FAILED}
 
-	// Retry on the same schedule as the AKS log path: the stream and its events can both lag the
-	// load window, so a single attempt fails on ingestion delay rather than on delivery.
+	// The agent routes OTLP logs to {host.id}/{service.name}, so assert that exact stream: it makes the
+	// check prove log routing rather than just delivery, and keeps cost flat as the shared group
+	// accumulates a stream per VM. Retries match the AKS path, since the stream and events both lag.
+	logStream := fmt.Sprintf("%s/%s", env.InstanceId, serviceName)
+	// Clean up only on success: the group is shared by every VM run, so drop this run's stream but never
+	// the group. On failure the stream is left in place as evidence for whoever debugs the run.
+	defer func() {
+		if testResult.Status == status.SUCCESSFUL {
+			awsservice.DeleteLogStream(otlpLogGroup, logStream)
+		}
+	}()
 	marker := fmt.Sprintf("azurevm_otlp_log_%s", env.InstanceId)
 	const maxRetries = 4
 	const retryInterval = 30 * time.Second
 	for attempt := 1; attempt <= maxRetries; attempt++ {
 		since := time.Now().Add(-loadWindow - time.Minute)
 		until := time.Now()
-
-		streams := awsservice.GetLogStreams(otlpLogGroup)
-		if len(streams) == 0 {
-			testResult.Reason = fmt.Errorf("attempt %d: no log streams found in %s", attempt, otlpLogGroup)
-		}
-		for _, stream := range streams {
-			log.Printf("[AzureVM_Logs] attempt %d: checking %s/%s", attempt, otlpLogGroup, *stream.LogStreamName)
-			err := awsservice.ValidateLogs(
-				otlpLogGroup, *stream.LogStreamName, &since, &until,
-				awsservice.AssertLogsNotEmpty(),
-				awsservice.AssertPerLog(awsservice.AssertLogContainsSubstring(marker)),
-			)
-			if err == nil {
-				testResult.Status = status.SUCCESSFUL
-				return testResult
-			}
-			testResult.Reason = err
+		log.Printf("[AzureVM_Logs] attempt %d: checking %s/%s", attempt, otlpLogGroup, logStream)
+		err := awsservice.ValidateLogs(
+			otlpLogGroup, logStream, &since, &until,
+			awsservice.AssertLogsNotEmpty(),
+			awsservice.AssertPerLog(awsservice.AssertLogContainsSubstring(marker)),
+		)
+		if err == nil {
+			testResult.Status = status.SUCCESSFUL
+			return testResult
 		}
+		testResult.Reason = err
 		if attempt < maxRetries {
 			log.Printf("[AzureVM_Logs] %v — retrying in %v", testResult.Reason, retryInterval)
 			time.Sleep(retryInterval)