ci: reduce resource pressure on GHA runners

- Lower minikube memory from 6GB to 4GB (leaves 3GB for host processes)
- Reduce parallel workers: argo 2→1, airflow 2→1, sfn 4→2
- Add pytest-rerunfailures with --reruns 1 for transient infra flakes
- Add background pod cleanup loop during argo/airflow test runs

Author: Nissan Pow

.github/workflows/ux-tests.yml

@@ -102,13 +102,13 @@ jobs:
             workers: 4
           - backend: argo-kubernetes
             services: "minio,postgresql,metadata-service,argo-workflows"
-            workers: 2
+            workers: 1
           - backend: airflow-kubernetes
             services: "minio,postgresql,metadata-service,airflow"
-            workers: 2
+            workers: 1
           - backend: sfn-batch
             services: "minio,postgresql,metadata-service,localbatch,ddb-local,sfn-local"
-            workers: 4
+            workers: 2
 
     runs-on: ubuntu-latest
 
@@ -130,15 +130,15 @@ jobs:
       - name: Install Metaflow and test dependencies
         run: |
           pip install --upgrade pip
-          pip install -e ".[dev]" pytest-xdist pytest-timeout pytest-cov
+          pip install -e ".[dev]" pytest-xdist pytest-timeout pytest-cov pytest-rerunfailures
           pip install "git+https://github.com/npow/localbatch.git@main#egg=localbatch"
 
       - name: Set up minikube
         uses: medyagh/setup-minikube@aba8d5ff1666c72adf94ccd078b2ca12e7756382
         with:
           driver: docker
           cpus: 2
-          memory: 6144
+          memory: 4096
 
       - name: Restore minikube image cache
         id: image-cache
@@ -236,11 +236,17 @@ jobs:
         if: matrix.backend == 'airflow-kubernetes'
         run: devtools/ci/wait-airflow-api.sh
 
-      - name: Clean up completed pods (argo/airflow only)
+      - name: Clean up completed pods and start background cleanup
         if: matrix.backend == 'argo-kubernetes' || matrix.backend == 'airflow-kubernetes'
         run: |
           kubectl delete pods --field-selector=status.phase=Succeeded --all-namespaces 2>/dev/null || true
           kubectl delete pods --field-selector=status.phase=Failed --all-namespaces 2>/dev/null || true
+          # Periodically clean up completed pods during test runs to free cluster resources
+          while true; do
+            sleep 60
+            kubectl delete pods --field-selector=status.phase=Succeeded --all-namespaces 2>/dev/null || true
+            kubectl delete pods --field-selector=status.phase=Failed --all-namespaces 2>/dev/null || true
+          done &
 
       - name: Run UX tests — ${{ matrix.backend }}
         run: |
@@ -253,6 +259,8 @@ jobs:
             -v \
             --tb=short \
             --timeout=1800 \
+            --reruns 1 \
+            --reruns-delay 10 \
             --cov=metaflow \
             --cov-report=term-missing \
             --cov-report=xml:coverage.xml \