Fix CI/CD workflow issues: justfile environment variables and manylinux setup

oberstet · oberstet · commit 5488c947fce7 · 2025-09-27T18:58:04.000+02:00
## Issues Fixed

### 1. justfile AUTOBAHN_USE_NVX Override
- justfile was hardcoding AUTOBAHN_USE_NVX=1, ignoring environment variables
- Changed to: AUTOBAHN_USE_NVX=${AUTOBAHN_USE_NVX:-1} (respects existing setting)
- Now workflows can control NVX via environment variables

### 2. manylinux Container Environment
- Fixed Python path issues in manylinux containers (/opt/python/*/bin)
- Added proper PATH setup for all Python versions
- Ensured python3 symlink availability in /usr/local/bin
- Fixed tool installation and PATH persistence (uv, rust)
- Added comprehensive environment verification

### 3. Workflow Artifact Naming
- Renamed "pure-python-wheel" to "linux-wheels-no-nvx" (more accurate)
- Updated all download/upload steps to use consistent naming
- Fixed artifact listing and verification steps

## Key Changes

**justfile**: Environment-aware NVX setting
**wheels-docker.yml**: Proper manylinux container setup with Python path fixes
**wheels.yml**: Consistent artifact naming and verification

The workflows should now properly:
- Build pure Python wheels on Linux (AUTOBAHN_USE_NVX=0)
- Build binary wheels with NVX on macOS/Windows (AUTOBAHN_USE_NVX=1)
- Build manylinux binary wheels with NVX in Docker containers
- Handle manylinux Python environment correctly
diff --git a/.github/workflows/wheels-docker.yml b/.github/workflows/wheels-docker.yml
@@ -80,17 +80,31 @@ jobs:
             dnf install -y snappy-devel || echo "snappy-devel not available, skipping"
           fi
 
-      - name: Verify Python installation
+      - name: Setup Python environment
         run: |
-          # manylinux images come with multiple Python versions pre-installed
+          # manylinux images come with multiple Python versions pre-installed in /opt/python/
           echo "==> Available Python versions:"
           ls -la /opt/python/*/bin/python* 2>/dev/null || echo "No /opt/python found"
+          
+          # Add all Python versions to PATH for uv/just to discover
+          for pyver in /opt/python/*/bin; do
+            if [ -d "$pyver" ]; then
+              echo "Adding $pyver to PATH"
+              export PATH="$pyver:$PATH"
+            fi
+          done
+          
+          # Also ensure we have a working python3 symlink
+          which python3 || ln -sf $(find /opt/python -name python3 | head -1) /usr/local/bin/python3
+          
           echo ""
-          echo "==> Default Python version:"
-          python3 --version 2>/dev/null || python --version 2>/dev/null || echo "No default Python found"
-          echo ""
+          echo "==> Current Python version:"
+          python3 --version
           echo "==> pip version:"
-          python3 -m pip --version 2>/dev/null || pip --version 2>/dev/null || echo "No pip found"
+          python3 -m pip --version
+          
+          # Save the updated PATH for subsequent steps
+          echo "PATH=$PATH" >> $GITHUB_ENV
 
       - name: Install Just
         run: |
@@ -100,12 +114,20 @@ jobs:
       - name: Install uv
         run: |
           curl -LsSf https://astral.sh/uv/install.sh | sh
+          # Add to both GITHUB_PATH and current environment
           echo "/root/.cargo/bin" >> $GITHUB_PATH
+          echo "PATH=/root/.cargo/bin:$PATH" >> $GITHUB_ENV
+          export PATH="/root/.cargo/bin:$PATH"
+          uv --version
 
       - name: Install Rust
         run: |
           curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+          # Add to both GITHUB_PATH and current environment  
           echo "/root/.cargo/bin" >> $GITHUB_PATH
+          echo "PATH=/root/.cargo/bin:$PATH" >> $GITHUB_ENV
+          export PATH="/root/.cargo/bin:$PATH"
+          rustc --version
 
       - name: Verify toolchain
         run: |
@@ -120,10 +142,21 @@ jobs:
           echo "glibc: $(ldd --version 2>/dev/null | head -1 || echo 'N/A')"
 
       - name: Build manylinux wheels with NVX
+        env:
+          AUTOBAHN_USE_NVX: 1
         run: |
+          # Ensure tools are in PATH
           export PATH="/root/.cargo/bin:$PATH"
+          
+          # Verify environment
+          echo "==> Environment verification:"
+          echo "AUTOBAHN_USE_NVX=$AUTOBAHN_USE_NVX"
+          echo "PATH=$PATH"
+          echo "Python: $(python3 --version)"
+          echo "uv: $(uv --version)"
+          echo "just: $(just --version)"
+          
           # Build binary wheels WITH NVX acceleration for manylinux
-          export AUTOBAHN_USE_NVX=1
           just build-all
 
       - name: Generate build metadata
diff --git a/.github/workflows/wheels.yml b/.github/workflows/wheels.yml
@@ -209,15 +209,18 @@ jobs:
           path: dist/*.whl
           retention-days: 30
 
-      - name: Verify pure Python wheel (Linux x86_64 only)
+      - name: Verify wheels built without NVX (Linux x86_64 only)
         if:
           matrix.platform == 'linux' && matrix.arch == 'x86_64'
         run: |
-          # Verify that pure Python wheel was built (should be automatic from build step above)
-          echo "Pure Python wheel verification:"
-          ls -la dist/*py3-none-any.whl || echo "No pure Python wheel found (this is expected since Linux builds pure Python wheels)"
-          echo "All Linux wheels (should be pure Python compatible):"
+          # Verify that wheels were built without NVX acceleration
+          echo "==> Wheels built without NVX (should be pure Python or at least NVX-free):"
           ls -la dist/*.whl || echo "No wheels found"
+          echo ""
+          echo "==> Source distribution:"
+          ls -la dist/*.tar.gz || echo "No source dist found"
+          echo ""
+          echo "==> Wheel count: $(ls dist/*.whl 2>/dev/null | wc -l)"
         shell: bash
 
       - name: Upload source distribution (Linux x86_64 only)
@@ -229,13 +232,13 @@ jobs:
           path: dist/*.tar.gz
           retention-days: 30
 
-      - name: Upload pure Python wheel (Linux x86_64 only)
+      - name: Upload Linux wheels without NVX (Linux x86_64 only)
         if:
           matrix.platform == 'linux' && matrix.arch == 'x86_64'
         uses: actions/upload-artifact@v4
         with:
-          name: pure-python-wheel
-          path: dist/*py3-none-any.whl
+          name: linux-wheels-no-nvx
+          path: dist/*.whl
           retention-days: 30
 
   publish-github-releases:
@@ -260,19 +263,19 @@ jobs:
           name: source-distribution
           path: dist/
 
-      - name: Download pure Python wheel
+      - name: Download Linux wheels without NVX
         uses: actions/download-artifact@v4
         with:
-          name: pure-python-wheel
+          name: linux-wheels-no-nvx
           path: dist/
 
       - name: List all artifacts
         run: |
           echo "All built artifacts:"
           ls -la dist/
           echo ""
-          echo "Platform-specific wheels: $(ls dist/*-cp*-*.whl dist/*-pp*-*.whl 2>/dev/null | wc -l)"
-          echo "Pure Python wheels: $(ls dist/*-py3-none-any.whl 2>/dev/null | wc -l)"
+          echo "Binary wheels (macOS/Windows): $(ls dist/*macos*.whl dist/*win*.whl 2>/dev/null | wc -l)"
+          echo "Linux wheels (no NVX): $(ls dist/*linux*.whl 2>/dev/null | wc -l)"
           echo "Source distributions: $(ls dist/*.tar.gz 2>/dev/null | wc -l)"
 
       - name: Create GitHub Release
@@ -337,10 +340,10 @@ jobs:
           name: source-distribution
           path: dist/
 
-      - name: Download pure Python wheel
+      - name: Download Linux wheels without NVX
         uses: actions/download-artifact@v4
         with:
-          name: pure-python-wheel
+          name: linux-wheels-no-nvx
           path: dist/
 
       - name: List artifacts for PyPI (selective publishing)
@@ -350,7 +353,7 @@ jobs:
           echo ""
           echo "macOS wheels: $(ls dist/*macos*.whl 2>/dev/null | wc -l)"
           echo "Windows wheels: $(ls dist/*win*.whl 2>/dev/null | wc -l)"
-          echo "Pure Python wheels: $(ls dist/*py3-none-any.whl 2>/dev/null | wc -l)"
+          echo "Linux wheels (fallback, no NVX): $(ls dist/*linux*.whl 2>/dev/null | wc -l)"
           echo "Source distributions: $(ls dist/*.tar.gz 2>/dev/null | wc -l)"
           echo ""
           echo "Total PyPI artifacts: $(ls dist/* | wc -l)"
diff --git a/justfile b/justfile
@@ -572,8 +572,8 @@ build venv="": (install-tools venv)
     VENV_PATH="{{ VENV_DIR }}/${VENV_NAME}"
     VENV_PYTHON=$(just --quiet _get-venv-python "${VENV_NAME}")
     echo "==> Building distribution packages..."
-    # Set environment variable for NVX acceleration
-    AUTOBAHN_USE_NVX=1 ${VENV_PYTHON} -m build
+    # Set environment variable for NVX acceleration (defaults to enabled, respects existing setting)
+    AUTOBAHN_USE_NVX=${AUTOBAHN_USE_NVX:-1} ${VENV_PYTHON} -m build
     ls -la dist/
 
 # Meta-recipe to run `build` on all environments
