fix: skip gpu e2e tests on sku quota exceeded and add configurable test timeout (#7836)

Author: awesomenix

.pipelines/e2e-gpu.yaml

@@ -3,6 +3,8 @@ variables:
   TAGS_TO_RUN: "gpu=true"
   TAGS_TO_SKIP: "os=windows"
   SKIP_E2E_TESTS: false
+  SKIP_TESTS_WITH_SKU_CAPACITY_ISSUE: true
+  E2E_GO_TEST_TIMEOUT: "75m"
 trigger:
   branches:
     include:

.pipelines/scripts/e2e_run.sh

@@ -37,6 +37,7 @@ LOGGING_DIR="${LOGGING_DIR:-}"
 E2E_SUBSCRIPTION_ID="${E2E_SUBSCRIPTION_ID:-}"
 TAGS_TO_SKIP="${TAGS_TO_SKIP:-}"
 TAGS_TO_RUN="${TAGS_TO_RUN:-}"
+E2E_GO_TEST_TIMEOUT="${E2E_GO_TEST_TIMEOUT:-90m}"
 GALLERY_NAME="${GALLERY_NAME:-}"
 SIG_GALLERY_NAME="${SIG_GALLERY_NAME:-}"
 
@@ -49,6 +50,7 @@ echo "TAGS_TO_SKIP: ${TAGS_TO_SKIP}"
 echo "TAGS_TO_RUN: ${TAGS_TO_RUN}"
 echo "GALLERY_NAME: ${GALLERY_NAME}"
 echo "SIG_GALLERY_NAME: ${SIG_GALLERY_NAME}"
+echo "E2E_GO_TEST_TIMEOUT: ${E2E_GO_TEST_TIMEOUT}"
 
 # set variables that the go program expects if we are running a specific build
 if [ -n "${VHD_BUILD_ID}" ]; then
@@ -94,7 +96,7 @@ rm -f "$temp_file"
 # gotestsum configure to only show logs for failed tests, json file for detailed logs
 # Run the tests! Yey!
 test_exit_code=0
-./bin/gotestsum --format testdox --junitfile "${BUILD_SRC_DIR}/e2e/report.xml" --jsonfile "${BUILD_SRC_DIR}/e2e/test-log.json" -- -parallel 150 -timeout 90m || test_exit_code=$?
+./bin/gotestsum --format testdox --junitfile "${BUILD_SRC_DIR}/e2e/report.xml" --jsonfile "${BUILD_SRC_DIR}/e2e/test-log.json" -- -parallel 150 -timeout "${E2E_GO_TEST_TIMEOUT}" || test_exit_code=$?
 
 # Upload test results as Azure DevOps artifacts
 echo "##vso[artifact.upload containerfolder=test-results;artifactname=e2e-test-log]${BUILD_SRC_DIR}/e2e/test-log.json"

e2e/vmss.go

@@ -439,14 +439,23 @@ func getPrivateIPFromVMSSVM(ctx context.Context, resourceGroup, vmssName, instan
 }
 
 func skipTestIfSKUNotAvailableErr(t testing.TB, err error) {
-	// sometimes the SKU is not available and we can't do anything. Skip the test in this case.
+	if !config.Config.SkipTestsWithSKUCapacityIssue {
+		return
+	}
 	var respErr *azcore.ResponseError
-	if config.Config.SkipTestsWithSKUCapacityIssue &&
-		errors.As(err, &respErr) &&
-		respErr.StatusCode == 409 &&
-		respErr.ErrorCode == "SkuNotAvailable" {
+	if !errors.As(err, &respErr) || respErr.StatusCode != 409 {
+		return
+	}
+	// sometimes the SKU is not available and we can't do anything. Skip the test in this case.
+	if respErr.ErrorCode == "SkuNotAvailable" {
 		t.Skip("skipping scenario SKU not available", t.Name(), err)
 	}
+	// sometimes the SKU quota is exceeded and we can't do anything. Skip the test in this case.
+	if respErr.ErrorCode == "OperationNotAllowed" &&
+		strings.Contains(respErr.Error(), "exceeding approved") &&
+		strings.Contains(respErr.Error(), "quota") {
+		t.Skip("skipping scenario SKU quota exceeded", t.Name(), err)
+	}
 }
 
 func cleanupVMSS(ctx context.Context, s *Scenario, vm *ScenarioVM) {