Fix OTLP Export Testing issues by removing test runs for itar and china regions (#670)

Author: JayPolanco

generator/test_case_generator.go

@@ -444,6 +444,8 @@ type partition struct {
 	// testConfigOverrides allows partition-specific test configurations
 	// key is testDir, value is the override config
 	testConfigOverrides map[string]testConfig
+	// excludedTestDirs allows excluding specific test directories from a partition
+	excludedTestDirs map[string]struct{}
 }
 
 var partitionTests = map[string]partition{
@@ -456,6 +458,11 @@ var partitionTests = map[string]partition{
 		configName: "_itar",
 		tests:      []string{testTypeKeyEc2Linux},
 		ami:        []string{"cloudwatch-agent-integration-test-aarch64-al2023*"},
+		excludedTestDirs: map[string]struct{}{
+			"./test/otlp_export/hostmetrics": {},
+			"./test/otlp_export/statsd":      {},
+			"./test/otlp_export/collectd":    {},
+		},
 		testConfigOverrides: map[string]testConfig{
 			"./test/metric_value_benchmark": {
 				// Exclude DiskIOInstanceStore and DiskIOEBS tests - custom AMI doesn't support NVMe instance store metrics
@@ -471,6 +478,11 @@ var partitionTests = map[string]partition{
 		configName: "_china",
 		tests:      []string{testTypeKeyEc2Linux},
 		ami:        []string{"cloudwatch-agent-integration-test-aarch64-al2023*"},
+		excludedTestDirs: map[string]struct{}{
+			"./test/otlp_export/hostmetrics": {},
+			"./test/otlp_export/statsd":      {},
+			"./test/otlp_export/collectd":    {},
+		},
 		testConfigOverrides: map[string]testConfig{
 			"./test/metric_value_benchmark": {
 				// Exclude DiskIOInstanceStore and DiskIOEBS tests - custom AMI doesn't support NVMe instance store metrics
@@ -498,7 +510,7 @@ func main() {
 			if len(partition.tests) != 0 && !slices.Contains(partition.tests, testType) {
 				continue
 			}
-			testMatrix := genMatrix(testType, testConfigs, partition.ami, partition.testConfigOverrides)
+			testMatrix := genMatrix(testType, testConfigs, partition.ami, partition.testConfigOverrides, partition.excludedTestDirs)
 			writeTestMatrixFile(testType+partition.configName, testMatrix)
 		}
 	}
@@ -528,7 +540,7 @@ func generateTestName(testType string, test_directory string) string {
 
 	return strings.Join(cleaned, "_")
 }
-func genMatrix(testType string, testConfigs []testConfig, ami []string, overrides map[string]testConfig) []matrixRow {
+func genMatrix(testType string, testConfigs []testConfig, ami []string, overrides map[string]testConfig, excludedTestDirs map[string]struct{}) []matrixRow {
 	openTestMatrix, err := os.Open(fmt.Sprintf("generator/resources/%v_test_matrix.json", testType))
 
 	if err != nil {
@@ -548,6 +560,13 @@ func genMatrix(testType string, testConfigs []testConfig, ami []string, override
 	testMatrixComplete := make([]matrixRow, 0, len(testMatrix))
 	for _, test := range testMatrix {
 		for _, testConfig := range testConfigs {
+			// Skip test dirs excluded for this partition
+			if excludedTestDirs != nil {
+				if _, excluded := excludedTestDirs[testConfig.testDir]; excluded {
+					continue
+				}
+			}
+
 			// Apply partition-specific overrides if available
 			if overrides != nil {
 				if override, ok := overrides[testConfig.testDir]; ok {

test/otlp_export/otlpvalidation/validate.go

@@ -14,22 +14,41 @@ import (
 )
 
 func ValidateOtlpMetrics(testName string, region string, metrics []string) status.TestGroupResult {
+	const maxRetries = 5
+	const retryInterval = 20 * time.Second
+
+	// Track which metrics have been validated
+	validated := make(map[string]bool, len(metrics))
+
+	for attempt := 0; attempt < maxRetries; attempt++ {
+		if attempt > 0 {
+			time.Sleep(retryInterval)
+		}
+		for _, m := range metrics {
+			if validated[m] {
+				continue
+			}
+			promql := fmt.Sprintf(`{__name__="%s"}`, m)
+			resp, err := awsservice.QueryOtlpMetrics(region, promql)
+			if err != nil || len(resp.Data.Result) == 0 {
+				continue
+			}
+			validated[m] = true
+		}
+		if len(validated) == len(metrics) {
+			break
+		}
+	}
+
 	results := make([]status.TestResult, 0, len(metrics)+1)
-	successCount := 0
 	for _, m := range metrics {
-		promql := fmt.Sprintf(`{__name__="%s"}`, m)
-		resp, err := awsservice.QueryOtlpMetricsWithRetry(region, promql, 10, 30*time.Second)
-		if err != nil {
-			results = append(results, status.TestResult{Name: m, Status: status.FAILED, Reason: err})
-			continue
-		}
-		if len(resp.Data.Result) == 0 {
-			results = append(results, status.TestResult{Name: m, Status: status.FAILED, Reason: fmt.Errorf("no results for %s", m)})
-			continue
+		if validated[m] {
+			results = append(results, status.TestResult{Name: m, Status: status.SUCCESSFUL})
+		} else {
+			results = append(results, status.TestResult{Name: m, Status: status.FAILED, Reason: fmt.Errorf("metric %s not found after %d retries", m, maxRetries)})
 		}
-		results = append(results, status.TestResult{Name: m, Status: status.SUCCESSFUL})
-		successCount++
 	}
+	successCount := len(validated)
 	if successCount != len(metrics) {
 		results = append(results, status.TestResult{
 			Name:   "MetricCountCheck",

util/awsservice/otlpmetricsquery.go

@@ -90,19 +90,3 @@ func QueryOtlpMetrics(region string, promql string) (PrometheusResponse, error)
 	return resp, nil
 }
 
-func QueryOtlpMetricsWithRetry(region string, promql string, retries int, retryInterval time.Duration) (PrometheusResponse, error) {
-	var lastErr error
-	for i := 0; i < retries; i++ {
-		resp, err := QueryOtlpMetrics(region, promql)
-		if err == nil && resp.Status == "success" && len(resp.Data.Result) > 0 {
-			return resp, nil
-		}
-		if err != nil {
-			lastErr = err
-		} else {
-			lastErr = fmt.Errorf("otlp query unsuccessful: status=%s, results=%d, error=%s", resp.Status, len(resp.Data.Result), resp.Error)
-		}
-		time.Sleep(retryInterval)
-	}
-	return PrometheusResponse{}, fmt.Errorf("otlp query failed after %d retries: %w", retries, lastErr)
-}