Isolate pyenv on self hosted runners and cleanup when done

.github/workflows/build_wheel_manylinux2014.yml

@@ -44,6 +44,26 @@ jobs:
         with:
           python-version: '3.8'
 
+      # Since the self-hosted runner can be re-used. It is best to set up all package
+      # installations in a virtual environment that gets cleaned at the end of each workflow run
+      - name: Setup Python virtual environment
+        id: setup_venv
+        env:
+          VENV_NAME: venv_${{ steps.setup_python.outputs.python-version }}_${{ github.sha }}
+        run: |
+          # Clear any pre-existing venvs
+          rm -rf venv_*
+
+          # Create new venv for this workflow_run
+          python --version
+          python -m venv ${{ env.VENV_NAME }}
+
+          # Add the venv to PATH for subsequent steps
+          echo ${{ env.VENV_NAME }}/bin >> $GITHUB_PATH
+
+          # Adding venv name as an output for subsequent steps to reference if needed
+          echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
+
       - name: Install cibuildwheel
         run: python -m pip install cibuildwheel~=2.11.0
 
@@ -60,6 +80,7 @@ jobs:
 
           # Python build settings
           CIBW_BEFORE_BUILD: |
+            which python
             python -m pip install ninja cmake~=3.24.3 auditwheel
             python -m pip install custatevec-cu11
             yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/cuda-rhel7.repo -y
@@ -91,6 +112,17 @@ jobs:
           name: ${{ runner.os }}-wheels
           path: ./wheelhouse/*.whl
 
+      # Cleans up the working directory so subsequent jobs on same runner get a clean working directory
+      # This step runs even if any of the previous steps fail
+      - name: Clean-up
+        if: always()
+        run: |
+          rm -rf ${{ steps.setup_venv.outputs.venv_name }}
+          rm -rf *
+          rm -rf .git
+          rm -rf .gitignore
+          rm -rf .github
+
   upload-pypi:
     needs: linux-wheels-x86-64
     runs-on: ubuntu-latest

.github/workflows/tests_linux_x86.yml

@@ -32,6 +32,26 @@ jobs:
         with:
           python-version: '3.8'
 
+      # Since the self-hosted runner can be re-used. It is best to set up all package
+      # installations in a virtual environment that gets cleaned at the end of each workflow run
+      - name: Setup Python virtual environment
+        id: setup_venv
+        env:
+          VENV_NAME: venv_${{ steps.setup_python.outputs.python-version }}_${{ github.sha }}
+        run: |
+          # Clear any pre-existing venvs
+          rm -rf venv_*
+
+          # Create new venv for this workflow_run
+          python --version
+          python -m venv ${{ env.VENV_NAME }}
+
+          # Add the venv to PATH for subsequent steps
+          echo ${{ env.VENV_NAME }}/bin >> $GITHUB_PATH
+
+          # Adding venv name as an output for subsequent steps to reference if needed
+          echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
+
       - name: Remove Ubuntu unattended upgrades
         run: sudo apt-get remove -y -q unattended-upgrades
 
@@ -97,12 +117,12 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
+          rm -rf ${{ steps.setup_venv.outputs.venv_name }}
           rm -rf *
           rm -rf .git
           rm -rf .gitignore
           rm -rf .github
 
-
   pythontests:
     runs-on:
       - self-hosted
@@ -118,6 +138,26 @@ jobs:
         with:
           python-version: '3.8'
 
+      # Since the self-hosted runner can be re-used. It is best to set up all package
+      # installations in a virtual environment that gets cleaned at the end of each workflow run
+      - name: Setup Python virtual environment
+        id: setup_venv
+        env:
+          VENV_NAME: venv_${{ steps.setup_python.outputs.python-version }}_${{ github.sha }}
+        run: |
+          # Clear any pre-existing venvs
+          rm -rf venv_*
+
+          # Create new venv for this workflow_run
+          python --version
+          python -m venv ${{ env.VENV_NAME }}
+
+          # Add the venv to PATH for subsequent steps
+          echo ${{ env.VENV_NAME }}/bin >> $GITHUB_PATH
+
+          # Adding venv name as an output for subsequent steps to reference if needed
+          echo "venv_name=${{ env.VENV_NAME }}" >> $GITHUB_OUTPUT
+
       - name: Install required packages
         run: |
           python -m pip install ninja cmake~=3.24.3 cuquantum pytest pytest-mock flaky pytest-cov
@@ -150,6 +190,7 @@ jobs:
       - name: Cleanup
         if: always()
         run: |
+          rm -rf ${{ steps.setup_venv.outputs.venv_name }}
           rm -rf *
           rm -rf .git
           rm -rf .gitignore