Wire styxcache into setup_runner for subprocess rbc invocations #162
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Full Testing Suite | |
| on: | |
| push: | |
| paths: | |
| - src/** | |
| - scripts/** | |
| - tests/** | |
| - pyproject.toml | |
| - uv.lock | |
| - .github/workflows/test_full.yaml | |
| workflow_dispatch: | |
| inputs: | |
| no_cleanup: | |
| description: Keep intermediate files after running tests | |
| default: false | |
| type: boolean | |
| styxcache_dir: | |
| description: Override persistent styxcache directory on the self-hosted runner | |
| default: "" | |
| type: string | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| test: | |
| runs-on: arcana | |
| env: | |
| STYXCACHE_DIR_OVERRIDE: ${{ inputs.styxcache_dir }} | |
| STYXCACHE_MAX_GB: "50" | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: astral-sh/setup-uv@v8.0.0 | |
| with: | |
| version: "0.11.3" | |
| enable-cache: true # not automatic on self-hosted runners | |
| - run: uv sync | |
| - name: Prepare styx cache | |
| shell: bash | |
| run: | | |
| # Resolve cache dir: honor explicit override, else use the runner user's | |
| # home cache (persistent across runs, user-owned — no sudo required). | |
| STYXCACHE_DIR="${STYXCACHE_DIR_OVERRIDE:-$HOME/.cache/rbc-styxcache}" | |
| echo "STYXCACHE_DIR=$STYXCACHE_DIR" >> "$GITHUB_ENV" | |
| mkdir -p "$STYXCACHE_DIR" | |
| # Purge stale staging dirs from crashed previous runs (safe: atomic-rename commits) | |
| rm -rf "$STYXCACHE_DIR/.incoming" 2>/dev/null || true | |
| size_kb=$(du -sk "$STYXCACHE_DIR" 2>/dev/null | awk '{print $1}') | |
| size_kb=${size_kb:-0} | |
| cap_kb=$(( STYXCACHE_MAX_GB * 1024 * 1024 )) | |
| if [ "$size_kb" -gt "$cap_kb" ]; then | |
| echo "Cache size ${size_kb}KB exceeds cap ${cap_kb}KB; evicting oldest shards" | |
| find "$STYXCACHE_DIR" -mindepth 2 -maxdepth 2 -type d -printf '%T@ %p\n' \ | |
| | sort -n \ | |
| | while read -r _ path; do | |
| rm -rf "$path" | |
| cur=$(du -sk "$STYXCACHE_DIR" 2>/dev/null | awk '{print $1}') | |
| [ "${cur:-0}" -le "$cap_kb" ] && break | |
| done | |
| fi | |
| { | |
| echo "## styxcache" | |
| echo "- dir: \`$STYXCACHE_DIR\`" | |
| echo "- size before: $(du -sh "$STYXCACHE_DIR" 2>/dev/null | cut -f1)" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Run all tests | |
| shell: bash | |
| run: | | |
| # Clean up stale job tmp dirs from previous crashed runs | |
| find "$RUNNER_TEMP" -maxdepth 1 -name 'rbc_test_*' -mmin +120 -exec rm -rf {} + 2>/dev/null || true | |
| export JOB_TMP="$RUNNER_TEMP/rbc_test_$(date +%s%N)" | |
| mkdir -p $JOB_TMP | |
| echo "JOB_TMP=$JOB_TMP" >> $GITHUB_ENV | |
| echo "Job temp directory: $JOB_TMP" | |
| export PYTEST_CACHE_DIR="$JOB_TMP/.pytest_cache" | |
| export PYTHONPYCACHEPREFIX="$JOB_TMP/__pycache__" | |
| export COVERAGE_FILE="$JOB_TMP/.coverage" | |
| export RBC_STYXCACHE_DIR="$STYXCACHE_DIR" | |
| uv run pytest \ | |
| -n 8 \ | |
| --runner=podman \ | |
| --basetemp=$JOB_TMP/pytest-tmp \ | |
| --cov-report= \ | |
| --cov=src \ | |
| --durations=0 \ | |
| --log-level=DEBUG \ | |
| --verbose tests | |
| - name: Generate pipeline report | |
| if: always() | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| uv run scripts/visualize_pipeline.py --output $JOB_TMP/pipeline_report.png | |
| if [ -f "$JOB_TMP/pipeline_report.png" ]; then | |
| echo "## Pipeline Report" >> $GITHUB_STEP_SUMMARY | |
| echo "Download the **pipeline-report** artifact for the full visualization." >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Upload pipeline report | |
| if: always() | |
| uses: actions/upload-artifact@v7 | |
| with: | |
| name: pipeline-report | |
| path: ${{ env.JOB_TMP }}/pipeline_report.png | |
| - name: Coverage summary | |
| if: always() | |
| env: | |
| COVERAGE_FAIL: 75 | |
| shell: bash | |
| run: | | |
| uv run coverage report \ | |
| --data-file=$JOB_TMP/.coverage \ | |
| --fail-under=$COVERAGE_FAIL \ | |
| --show-missing \ | |
| --format=markdown >> $GITHUB_STEP_SUMMARY | |
| - name: styxcache summary | |
| if: always() | |
| shell: bash | |
| run: | | |
| size=$(du -sh "$STYXCACHE_DIR" 2>/dev/null | cut -f1) | |
| entries=$(find "$STYXCACHE_DIR" -mindepth 2 -maxdepth 2 -type d 2>/dev/null | wc -l) | |
| { | |
| echo "- size after: ${size:-0}" | |
| echo "- entries: ${entries:-0}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Cleanup | |
| if: ${{ always() && github.event.inputs.no_cleanup != true }} | |
| shell: bash | |
| run: | | |
| echo "Cleaning up job temp directory: $JOB_TMP" | |
| echo "Temp directory size: $(du -sh $JOB_TMP 2>/dev/null | cut -f1)" | |
| rm -rf $JOB_TMP |