v0.2.0 (honest beta): merge train complete, gates green, real jobs verified #95
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Fast CI | |
| on: | |
| # No `paths:` filter on pull_request, deliberately. | |
| # | |
| # The `status-check` job below publishes the `CI Status` context, and | |
| # master's branch protection lists that context as required. A required | |
| # check that is never reported is not treated as passing -- GitHub blocks | |
| # the merge on "Expected -- Waiting for status to be reported", forever -- | |
| # so while this trigger was path-filtered, a pull request touching only | |
| # docs/, README.md or the notebooks could never be merged by anyone | |
| # without an admin override (#169). A required check whose reporting | |
| # depends on which files changed is a trap; the filter is not worth it. | |
| # | |
| # This repository is public, so Actions minutes are free and the cost of | |
| # running the four jobs on a docs-only pull request is wall-clock time, | |
| # not money. | |
| pull_request: | |
| branches: [main, master, develop] | |
| push: | |
| # The filter stays here. `CI Status` is not a required check for pushes | |
| # to develop, so a run that never happens blocks nothing. | |
| branches: [develop] # Only run on develop pushes to avoid duplication with main Tests workflow | |
| paths: | |
| - 'clustrix/**' | |
| - 'tests/**' | |
| - 'setup.py' | |
| - 'pyproject.toml' | |
| - 'requirements*.txt' | |
| jobs: | |
| quick-checks: | |
| name: Quick Checks | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Cache dependencies | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/pip | |
| key: ${{ runner.os }}-pip-quick-${{ hashFiles('**/requirements.txt') }} | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| # The dev extra, not a hand-listed set: that list installed an | |
| # unpinned black -- the drift #110 pinned pyproject against -- and | |
| # omitted pytest-timeout while the test step below passes | |
| # --timeout=60, so this job failed with "unrecognized arguments" | |
| # regardless of the code. | |
| pip install -e ".[dev]" | |
| - name: Format check with Black | |
| # Same scope as tests.yml's black step. They covered different | |
| # directories, so a file could pass one gate and fail the other. | |
| run: black --check clustrix/ tests/ scripts/ | |
| - name: Lint with Flake8 | |
| run: flake8 clustrix/ --count --select=E9,F63,F7,F82 --show-source --statistics | |
| - name: Type check with MyPy | |
| run: mypy clustrix/ --ignore-missing-imports | |
| - name: Run quick unit tests | |
| run: | | |
| pytest tests/unit/ -v \ | |
| -m "not real_world and not slow" \ | |
| --timeout=60 \ | |
| -x \ | |
| --tb=short \ | |
| --maxfail=3 | |
| # Sized for the suite as it is now (~1,100 unit tests, several of | |
| # which execute notebooks and real subprocesses): the old 5-minute | |
| # step timeout was set when tests/unit held ~350 fast tests and cut | |
| # the run at 60% regardless of what passed before it. | |
| timeout-minutes: 15 | |
| local-integration: | |
| name: Local Integration Test | |
| runs-on: ubuntu-latest | |
| needs: quick-checks | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -e ".[dev]" | |
| - name: Run local executor test | |
| run: | | |
| python -c " | |
| from clustrix import cluster, configure | |
| # Configure for local execution | |
| configure(cluster_type='local') | |
| @cluster(cores=2, memory='2GB') | |
| def test_function(x, y): | |
| import numpy as np | |
| return np.array([x, y]).sum() | |
| result = test_function(10, 20) | |
| assert result == 30, f'Expected 30, got {result}' | |
| print('✅ Local execution test passed') | |
| " | |
| - name: Run serialization test | |
| run: | | |
| python -c " | |
| from clustrix import cluster | |
| import pickle | |
| import cloudpickle | |
| @cluster(cores=1) | |
| def test_serialization(): | |
| return {'status': 'success', 'value': 42} | |
| # Test function can be serialized | |
| serialized = cloudpickle.dumps(test_serialization) | |
| assert len(serialized) > 0 | |
| print('✅ Serialization test passed') | |
| # Test execution | |
| result = test_serialization() | |
| assert result['status'] == 'success' | |
| assert result['value'] == 42 | |
| print('✅ Execution test passed') | |
| " | |
| docker-test: | |
| name: Docker Container Test | |
| runs-on: ubuntu-latest | |
| needs: quick-checks | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Build test container | |
| run: | | |
| cat << 'EOF' > Dockerfile.test | |
| FROM python:3.10-slim | |
| WORKDIR /app | |
| COPY . . | |
| RUN pip install -e . | |
| RUN pip install pytest numpy pandas | |
| CMD ["python", "-c", "from clustrix import cluster; print('✅ Import successful')"] | |
| EOF | |
| docker build -f Dockerfile.test -t clustrix-test . | |
| - name: Run container test | |
| run: | | |
| docker run --rm clustrix-test | |
| # Test with actual computation | |
| docker run --rm clustrix-test python -c " | |
| from clustrix import cluster, configure | |
| import numpy as np | |
| configure(cluster_type='local') | |
| @cluster(cores=1, memory='1GB') | |
| def compute(): | |
| data = np.random.randn(1000, 1000) | |
| return {'mean': float(np.mean(data)), 'std': float(np.std(data))} | |
| result = compute() | |
| print(f'Result: {result}') | |
| assert abs(result['mean']) < 0.1 | |
| assert abs(result['std'] - 1.0) < 0.1 | |
| print('✅ Container test passed') | |
| " | |
| security-scan: | |
| name: Security Scan | |
| runs-on: ubuntu-latest | |
| needs: quick-checks | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run Trivy security scan | |
| uses: aquasecurity/trivy-action@master | |
| with: | |
| scan-type: 'fs' | |
| scan-ref: '.' | |
| format: 'sarif' | |
| output: 'trivy-results.sarif' | |
| - name: Upload Trivy results | |
| uses: github/codeql-action/upload-sarif@v2 | |
| with: | |
| sarif_file: 'trivy-results.sarif' | |
| - name: Check for secrets | |
| run: python scripts/check_for_secrets.py | |
| status-check: | |
| name: CI Status | |
| runs-on: ubuntu-latest | |
| needs: [quick-checks, local-integration, docker-test, security-scan] | |
| if: always() | |
| steps: | |
| - name: Check status | |
| run: | | |
| # security-scan is in `needs` above but was missing from this | |
| # condition, so a failing security scan still reported "All CI | |
| # checks passed". Every job this gate depends on must be checked, | |
| # or the gate is decorative. | |
| failed=0 | |
| for job in quick-checks local-integration docker-test security-scan; do | |
| case "$job" in | |
| quick-checks) result="${{ needs.quick-checks.result }}" ;; | |
| local-integration) result="${{ needs.local-integration.result }}" ;; | |
| docker-test) result="${{ needs.docker-test.result }}" ;; | |
| security-scan) result="${{ needs.security-scan.result }}" ;; | |
| esac | |
| if [ "$result" != "success" ]; then | |
| echo "❌ $job: $result" | |
| failed=1 | |
| else | |
| echo "✅ $job: success" | |
| fi | |
| done | |
| if [ "$failed" -ne 0 ]; then | |
| echo "CI checks failed" | |
| exit 1 | |
| fi | |
| echo "All CI checks passed" |