fix: stabilize periodic e2e test execution under parallel load

elenagerman · elenagerman · commit ebb5317989e4 · 2026-05-28T00:16:02.000Z
Stabilize periodic e2e: 6Gi memory, cap 9 parallel suites,
retry component init

Signed-off-by: Elena German &lt;elgerman@redhat.com&gt;
Assisted-by: Claude
diff --git a/integration-tests/README.md b/integration-tests/README.md
@@ -173,6 +173,13 @@ These integration tests are automatically executed in CI/CD pipelines:
 - **Pull Request Triggers** - Tests run when changes are made to relevant pipeline files or the `integration-tests/` directory
 - **E2E Pipeline** - Uses `integration-tests/pipelines/e2e-tests-staging-pipeline.yaml`
 - **Konflux E2E Pipeline** - Uses `integration-tests/pipelines/konflux-e2e-tests-pipeline.yaml`
+- **Periodic E2E Pipeline** - Uses `integration-tests/pipelines/e2e-tests-periodic-pipeline.yaml`
+  - Runs all integration suites in one Tekton step
+  - **Memory**: 6Gi on the `run-test` step (raised from 2Gi to avoid OOM during parallel setup)
+  - **Concurrency**: at most **9** suites at a time by default (`MAX_PARALLEL` pipeline param, overridable per PipelineRun)
+  - Component init retries when `kubectl get` is temporarily unavailable under load (`lib/test-functions.sh`)
+
+Local `./run-test.sh` runs one suite at a time; only the periodic pipeline runs many suites in parallel.
 
 ## Troubleshooting
 
@@ -182,7 +189,8 @@ These integration tests are automatically executed in CI/CD pipelines:
 2. **Cluster Access** - Ensure KUBECONFIG is properly configured
 3. **Secret Errors** - Check vault password file exists and is correct
 4. **Resource Conflicts** - Use cleanup scripts to remove stale resources
-5. **PaC token unrecognizable error** - The following error:
+5. **OOM or `kubectl create` exit 137 in periodic e2e** - Usually too many suites starting at once or insufficient step memory. The periodic pipeline caps parallelism and sets 6Gi; if failures persist, check Tekton step logs for `Killed` during tenant resource setup.
+6. **PaC token unrecognizable error** - The following error:
    ```bash
    Initialization check attempt 6/60...
    ⚠️ Warning: Could not get component PR from annotations: {"pac":{"state":"error","error-id":74,"error-message":"74: Access token is unrecognizable by GitHub"},"message":"done"}
diff --git a/integration-tests/lib/test-functions.sh b/integration-tests/lib/test-functions.sh
@@ -334,6 +334,25 @@ create_kubernetes_resources() {
     echo "Kubernetes resources applied."
 }
 
+# Fetch build.appstudio.openshift.io/status from a Component (stdout). Returns 1 if kubectl fails.
+# Relies on global variable: tenant_namespace
+fetch_component_build_status_annotation() {
+    local comp_name="$1"
+    local component_json=""
+    local kubectl_status=0
+
+    set +e
+    component_json=$(kubectl get component/"${comp_name}" -n "${tenant_namespace}" -ojson 2>/dev/null)
+    kubectl_status=$?
+    set -e
+
+    if [ "${kubectl_status}" -ne 0 ] || [ -z "${component_json}" ]; then
+        return 1
+    fi
+
+    jq -r --arg k "build.appstudio.openshift.io/status" '.metadata.annotations[$k] // ""' <<< "${component_json}"
+}
+
 # Function to wait for component initialization and get PR details
 # Modifies global variables: component_pr, pr_number
 # Relies on global variables: component_name, tenant_namespace
@@ -348,9 +367,15 @@ wait_for_component_initialization() {
     while [ $attempt -le $max_attempts ]; do
       echo "Initialization check attempt ${attempt}/${max_attempts}..."
 
-      # Try to get component annotations
-      component_annotations=$(kubectl get component/"${component_name}" -n "${tenant_namespace}" -ojson 2>/dev/null | \
-        jq -r --arg k "build.appstudio.openshift.io/status" '.metadata.annotations[$k] // ""')
+      if ! component_annotations=$(fetch_component_build_status_annotation "${component_name}"); then
+        log_warning "Could not reach component ${component_name} (kubectl get failed); retrying..."
+        if [ $attempt -lt $max_attempts ]; then
+          echo "Waiting 10 seconds before retry..."
+          sleep 10
+        fi
+        attempt=$((attempt + 1))
+        continue
+      fi
 
       if [ -n "${component_annotations}" ]; then
         # component_pr is made global by not declaring it local
@@ -742,9 +767,15 @@ wait_for_single_component_initialization() {
     while [ $attempt -le $max_attempts ]; do
       echo "Initialization check attempt ${attempt}/${max_attempts} for ${comp_name}..."
 
-      # Try to get component annotations
-      component_annotations=$(kubectl get component/"${comp_name}" -n "${tenant_namespace}" -ojson 2>/dev/null | \
-        jq -r --arg k "build.appstudio.openshift.io/status" '.metadata.annotations[$k] // ""')
+      if ! component_annotations=$(fetch_component_build_status_annotation "${comp_name}"); then
+        echo "⚠️  Could not reach component ${comp_name} (kubectl get failed); retrying..."
+        if [ $attempt -lt $max_attempts ]; then
+          echo "Waiting 10 seconds before retry..."
+          sleep 10
+        fi
+        attempt=$((attempt + 1))
+        continue
+      fi
 
       if [ -n "${component_annotations}" ]; then
         # component_pr is made global by not declaring it local
diff --git a/integration-tests/pipelines/e2e-tests-periodic-pipeline.yaml b/integration-tests/pipelines/e2e-tests-periodic-pipeline.yaml
@@ -17,6 +17,10 @@ spec:
     - name: KUBECONFIG_SECRET_NAME
       default: 'kubeconfig-secret'
       type: string
+    - name: MAX_PARALLEL
+      default: '9'
+      type: string
+      description: Maximum integration test suites to run concurrently in the periodic run-test step
   tasks:
     - name: get-snapshot-data
       params:
@@ -49,6 +53,8 @@ spec:
           value: $(params.GITHUB_TOKEN_SECRET_NAME)
         - name: KUBECONFIG_SECRET_NAME
           value: $(params.KUBECONFIG_SECRET_NAME)
+        - name: MAX_PARALLEL
+          value: $(params.MAX_PARALLEL)
       taskSpec:
         params:
           - name: STEP_IMAGE
@@ -61,6 +67,8 @@ spec:
           - name: VAULT_PASSWORD_SECRET_NAME
           - name: GITHUB_TOKEN_SECRET_NAME
           - name: KUBECONFIG_SECRET_NAME
+          - name: MAX_PARALLEL
+            type: string
         results:
           - name: TEST_OUTPUT
             description: Test output
@@ -69,9 +77,9 @@ spec:
             image: $(params.STEP_IMAGE)
             computeResources:
               limits:
-                memory: 2Gi
+                memory: 6Gi
               requests:
-                memory: 2Gi
+                memory: 6Gi
             env:
               - name: VAULT_PASSWORD
                 valueFrom:
@@ -171,29 +179,53 @@ spec:
               export RELEASE_CATALOG_GIT_URL
               export RELEASE_CATALOG_GIT_REVISION
 
-              # Run all testcases in parallel and collect their statuses
+              # Cap parallel suites to limit kubectl/API and memory spikes (step memory: 6Gi).
+              MAX_PARALLEL="$(params.MAX_PARALLEL)"
               declare -A pids
-              # declare -A testcase_names
-              for testcase in "${ALL_TESTCASES[@]}"; do
-                echo "Running test case: $testcase"
-                "/home/e2e/tests/run-test.sh" "$testcase" &
-                pid=$!
-                pids["$pid"]=$testcase
-                # testcase_names["$testcase"]=$pid
-              done
+              testcase_index=0
 
-              # Wait for all tests to complete, track success and failure
-              for pid in "${!pids[@]}"; do
-                testcase="${pids[$pid]}"
-                if wait "$pid"; then
-                  success_test_cases+=("$testcase")
-                  success_test_count=$((success_test_count + 1))
-                  echo "Test case $testcase succeeded"
-                else
-                  failed_test_cases+=("$testcase")
-                  failed_test_count=$((failed_test_count + 1))
-                  echo "Test case $testcase failed"
+              reap_finished_test_jobs() {
+                local pid testcase wait_status
+                for pid in "${!pids[@]}"; do
+                  if kill -0 "${pid}" 2>/dev/null; then
+                    continue
+                  fi
+                  testcase="${pids[$pid]}"
+                  set +e
+                  wait "${pid}"
+                  wait_status=$?
+                  set -e
+                  if [ "${wait_status}" -eq 0 ]; then
+                    success_test_cases+=("${testcase}")
+                    success_test_count=$((success_test_count + 1))
+                    echo "Test case ${testcase} succeeded"
+                  else
+                    failed_test_cases+=("${testcase}")
+                    failed_test_count=$((failed_test_count + 1))
+                    echo "Test case ${testcase} failed"
+                  fi
+                  unset 'pids[$pid]'
+                done
+              }
+
+              while [ "${testcase_index}" -lt "${overall_test_count}" ] || [ "${#pids[@]}" -gt 0 ]; do
+                while [ "${testcase_index}" -lt "${overall_test_count}" ] && [ "${#pids[@]}" -lt "${MAX_PARALLEL}" ]; do
+                  testcase="${ALL_TESTCASES[$testcase_index]}"
+                  testcase_index=$((testcase_index + 1))
+                  echo "Running test case: ${testcase} (${testcase_index}/${overall_test_count}," \
+                    "max ${MAX_PARALLEL} concurrent)"
+                  "/home/e2e/tests/run-test.sh" "${testcase}" &
+                  pids[$!]="${testcase}"
+                done
+
+                if [ "${#pids[@]}" -eq 0 ]; then
+                  continue
                 fi
+
+                set +e
+                wait -n 2>/dev/null
+                set -e
+                reap_finished_test_jobs
               done
               if [[ "$failed_test_count" -gt  0 ]]; then
                  exit 1
