fix(test): implement retry logic for checking SDM gateway online status in integration test

Author: coolguy-5

tests/integration/integration_test.go

@@ -43,14 +43,37 @@ func TestTerraformIntegration(t *testing.T) {
 	})
 }
 
+// TODO: refactor assertions! Ugly!
 func testSDMGatewayIsOnline(t *testing.T, opts *terraform.Options) {
 	gatewayIP := terraform.Output(t, opts, "ec2_instance_public_dns")
-	cmd := exec.Command("sdm", "admin", "nodes", "list", "--filter", fmt.Sprintf("listen_addr:%s:5000", gatewayIP))
-	output, err := cmd.CombinedOutput()
-	assert.NoError(t, err, "Failed to execute SDM CLI command")
 
-	// Check if the output contains "online"
-	assert.Contains(t, string(output), "online", "Gateway should be online")
+	maxRetries := 12
+	retryInterval := 10 * time.Second
+
+	var lastErr error
+	var output []byte
+
+	for i := 0; i < maxRetries; i++ {
+		cmd := exec.Command("sdm", "admin", "nodes", "list", "--filter", fmt.Sprintf("listen_addr:%s:5000", gatewayIP))
+		output, lastErr = cmd.CombinedOutput()
+
+		if lastErr == nil && strings.Contains(string(output), "online") {
+			// Success - gateway is online
+			return
+		}
+
+		if i < maxRetries-1 {
+			t.Logf("Attempt %d/%d: Gateway not online yet, retrying in %v...", i+1, maxRetries, retryInterval)
+			time.Sleep(retryInterval)
+		}
+	}
+
+	// All retries failed
+	if lastErr != nil {
+		assert.NoError(t, lastErr, "Failed to execute SDM CLI command after %d retries", maxRetries)
+	} else {
+		assert.Contains(t, string(output), "online", "Gateway should be online after %d retries (2 minutes)", maxRetries)
+	}
 }
 
 func testGatewayConnection(t *testing.T, opts *terraform.Options) {