Implement per-cluster working directories for job isolation

Changed from shared WORKING_DIR to cluster-specific directories to
prevent conflicts when multiple CI jobs run in parallel.

Workflow changes:
- Renamed WORKING_DIR var to BASE_WORKING_DIR for clarity
- After cluster name generation, create cluster-specific directory:
  ${BASE_WORKING_DIR}/clusters/${ENCLAVE_CLUSTER_NAME}
- Set WORKING_DIR to the cluster-specific path
- Applied to all 4 workflows (infra-verify, e2e-deployment, nightly x2)

Cleanup script changes:
- Updated to handle new directory structure
- Remove entire cluster-specific working directory
- Still clean up shared resources (pool, landing-zone) from BASE_DIR
- Backwards compatible with old structure during transition
- Enhanced verification to check both old and new structures

Benefits:
- Full isolation between parallel CI jobs
- No conflicts on environment.json or other working files
- Cleaner cleanup (remove entire cluster directory)
- Easier to debug (all cluster files in one place)

Example structure:
  /opt/dev-scripts/clusters/eci-6fc9fa16/  (isolated per job)
  /opt/dev-scripts/pool/eci-6fc9fa16/      (shared storage)
  /opt/dev-scripts/landing-zone/           (shared)

Author: eliorerz

.github/workflows/e2e-deployment.yml

@@ -104,7 +104,7 @@ jobs:
 
     env:
       DEV_SCRIPTS_PATH: ${{ vars.DEV_SCRIPTS_PATH }}
-      WORKING_DIR: ${{ vars.WORKING_DIR }}
+      BASE_WORKING_DIR: ${{ vars.BASE_WORKING_DIR }}
       PULL_SECRET: ${{ secrets.PULL_SECRET }}
       # Use manual input if available, otherwise default to 'connected' for PR runs
       ENCLAVE_DEPLOYMENT_MODE: ${{ github.event.inputs.deployment_mode || 'connected' }}
@@ -124,6 +124,14 @@ jobs:
           prefix: eci
           run-id: ${{ github.run_id }}
 
+      - name: Setup cluster-specific working directory
+        run: |
+          CLUSTER_WORKING_DIR="${BASE_WORKING_DIR}/clusters/${ENCLAVE_CLUSTER_NAME}"
+          echo "Creating cluster-specific working directory: $CLUSTER_WORKING_DIR"
+          mkdir -p "$CLUSTER_WORKING_DIR"
+          echo "WORKING_DIR=$CLUSTER_WORKING_DIR" >> $GITHUB_ENV
+          echo "Cluster working directory: $CLUSTER_WORKING_DIR"
+
       - name: Pre-flight checks
         uses: ./.github/actions/preflight-checks
         with:

.github/workflows/infra-verify.yml

@@ -29,7 +29,7 @@ jobs:
 
     env:
       DEV_SCRIPTS_PATH: ${{ vars.DEV_SCRIPTS_PATH }}
-      WORKING_DIR: ${{ vars.WORKING_DIR }}
+      BASE_WORKING_DIR: ${{ vars.BASE_WORKING_DIR }}
       PULL_SECRET: ${{ secrets.PULL_SECRET }}
       # For PR runs: always cleanup to avoid leaving infrastructure
       # For manual runs: use input parameter (default: on_failure)
@@ -48,6 +48,14 @@ jobs:
           prefix: eci
           run-id: ${{ github.run_id }}
 
+      - name: Setup cluster-specific working directory
+        run: |
+          CLUSTER_WORKING_DIR="${BASE_WORKING_DIR}/clusters/${ENCLAVE_CLUSTER_NAME}"
+          echo "Creating cluster-specific working directory: $CLUSTER_WORKING_DIR"
+          mkdir -p "$CLUSTER_WORKING_DIR"
+          echo "WORKING_DIR=$CLUSTER_WORKING_DIR" >> $GITHUB_ENV
+          echo "Cluster working directory: $CLUSTER_WORKING_DIR"
+
       - name: Workflow information
         env:
           PR_TITLE: ${{ github.event.pull_request.title }}

.github/workflows/nightly-e2e-connected.yml

@@ -23,7 +23,7 @@ jobs:
 
     env:
       DEV_SCRIPTS_PATH: ${{ vars.DEV_SCRIPTS_PATH }}
-      WORKING_DIR: ${{ vars.WORKING_DIR }}
+      BASE_WORKING_DIR: ${{ vars.BASE_WORKING_DIR }}
       PULL_SECRET: ${{ secrets.PULL_SECRET }}
       ENCLAVE_DEPLOYMENT_MODE: connected
       CLEANUP_AFTER: 'true'
@@ -39,6 +39,14 @@ jobs:
           naming-strategy: date
           prefix: nc
 
+      - name: Setup cluster-specific working directory
+        run: |
+          CLUSTER_WORKING_DIR="${BASE_WORKING_DIR}/clusters/${ENCLAVE_CLUSTER_NAME}"
+          echo "Creating cluster-specific working directory: $CLUSTER_WORKING_DIR"
+          mkdir -p "$CLUSTER_WORKING_DIR"
+          echo "WORKING_DIR=$CLUSTER_WORKING_DIR" >> $GITHUB_ENV
+          echo "Cluster working directory: $CLUSTER_WORKING_DIR"
+
       - name: Workflow information
         run: |
           echo "## Nightly E2E Deployment - Connected Mode" >> $GITHUB_STEP_SUMMARY

.github/workflows/nightly-e2e-disconnected.yml

@@ -23,7 +23,7 @@ jobs:
 
     env:
       DEV_SCRIPTS_PATH: ${{ vars.DEV_SCRIPTS_PATH }}
-      WORKING_DIR: ${{ vars.WORKING_DIR }}
+      BASE_WORKING_DIR: ${{ vars.BASE_WORKING_DIR }}
       PULL_SECRET: ${{ secrets.PULL_SECRET }}
       ENCLAVE_DEPLOYMENT_MODE: disconnected
       CLEANUP_AFTER: 'true'
@@ -39,6 +39,14 @@ jobs:
           naming-strategy: date
           prefix: nd
 
+      - name: Setup cluster-specific working directory
+        run: |
+          CLUSTER_WORKING_DIR="${BASE_WORKING_DIR}/clusters/${ENCLAVE_CLUSTER_NAME}"
+          echo "Creating cluster-specific working directory: $CLUSTER_WORKING_DIR"
+          mkdir -p "$CLUSTER_WORKING_DIR"
+          echo "WORKING_DIR=$CLUSTER_WORKING_DIR" >> $GITHUB_ENV
+          echo "Cluster working directory: $CLUSTER_WORKING_DIR"
+
       - name: Workflow information
         run: |
           echo "## Nightly E2E Deployment - Disconnected Mode" >> $GITHUB_STEP_SUMMARY