Nightly Deep Tests #29
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # IMPORTANT: `schedule` triggers only fire from the repository's DEFAULT | |
| # branch (main) — GitHub ignores schedule triggers defined on any other | |
| # branch. This workflow checks out `develop` explicitly in every job so the | |
| # nightly run tests the integration branch, not whatever HEAD of main | |
| # happens to be. In practice this means: (1) this file only starts firing | |
| # on its cron once it has been merged to main via the next release, and | |
| # (2) `workflow_dispatch` on main is the way to test it immediately after | |
| # that merge, before waiting for the next 05:00 UTC tick. | |
| name: Nightly Deep Tests | |
| on: | |
| schedule: | |
| - cron: '0 5 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| check-python-floor: | |
| description: 'Run an install-only floor check on this Python version, e.g. 3.10' | |
| required: false | |
| type: string | |
| permissions: | |
| contents: read | |
| concurrency: | |
| group: nightly | |
| cancel-in-progress: true | |
| jobs: | |
| deep-test: | |
| name: Deep Tests (${{ matrix.os }} / py${{ matrix.python }}) | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 90 | |
| # Experimental legs (3.12/3.13) prove wheel availability for the | |
| # scipy/numpy pins ahead of adopting them as a supported floor/ceiling — | |
| # promote to non-experimental once they've proven stable over time. | |
| continue-on-error: ${{ matrix.experimental || false }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| python: ['3.11'] | |
| include: | |
| - os: ubuntu-latest | |
| python: '3.12' | |
| experimental: true | |
| - os: ubuntu-latest | |
| python: '3.13' | |
| experimental: true | |
| env: | |
| # Scoped to this job ON PURPOSE — see the note on test.yml's `test` job. | |
| # Do NOT hoist to workflow level: mcp-e2e must stay WITHOUT it. | |
| # (Job-level env does not cross the WSL boundary; wsl2-suite exports its own.) | |
| PYTHONUTF8: "1" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: develop | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| # No pip cache — a from-scratch install is part of what nightly proves. | |
| - name: Install system dependencies (Linux) | |
| if: runner.os == 'Linux' | |
| run: | | |
| sudo apt-get update -qq | |
| sudo apt-get install -y --no-install-recommends ffmpeg | |
| - name: Install system dependencies (macOS) | |
| if: runner.os == 'macOS' | |
| run: | | |
| brew install ffmpeg | |
| # ffmpeg is installed on all three OSes so the ffmpeg-gated audio tests | |
| # run everywhere; AnthemScore/MuseScore remain WSL-recommended externals. | |
| - name: Install system dependencies (Windows) | |
| if: runner.os == 'Windows' | |
| run: choco install ffmpeg -y --no-progress | |
| # Every ffmpeg test gates on a module-level shutil.which("ffmpeg") skipif, | |
| # so a half-succeeded install (or a PATH the shim never reached) would let | |
| # ~10 audio tests silently skip while the leg still went green. Assert both | |
| # binaries resolve here instead. shell: bash runs with -eo pipefail on all | |
| # three OSes (Git Bash on Windows), so a missing binary fails the job. | |
| - name: Verify ffmpeg and ffprobe are on PATH | |
| shell: bash | |
| run: | | |
| ffmpeg -version | |
| ffprobe -version | |
| - name: Install test dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-test.txt | |
| pip install -r requirements.txt | |
| - name: Run tests | |
| id: tests | |
| # No coverage gate here — coverage is the PR job's (test.yml) gate. | |
| # Nightly's job is proving from-scratch installs and extra Python | |
| # versions across all three OSes, not re-litigating coverage. | |
| run: | | |
| python -m pytest tests/ --tb=short -n auto | |
| # Experimental legs run under continue-on-error, which reports the JOB as | |
| # successful — so notify-failure's `if: failure()` never fires for them and | |
| # a broken 3.12/3.13 leg is invisible unless a human opens the run. Publish | |
| # the real step outcome to the run's summary page so every leg is legible | |
| # at a glance. `if: always()` is required: a failed step otherwise skips | |
| # the rest of the job even under continue-on-error. | |
| - name: Record leg outcome in job summary | |
| if: always() | |
| shell: bash | |
| env: | |
| LEG: "${{ matrix.os }} / py${{ matrix.python }}" | |
| OUTCOME: "${{ steps.tests.outcome }}" | |
| TIER: "${{ matrix.experimental && 'experimental' || 'supported' }}" | |
| run: | | |
| if [ "$OUTCOME" = "success" ]; then icon="✅"; else icon="❌"; fi | |
| echo "$icon **$LEG** ($TIER) — tests: \`$OUTCOME\`" >> "$GITHUB_STEP_SUMMARY" | |
| # DELIBERATELY has no PYTHONUTF8 — same rationale as test.yml's mcp-boot job: | |
| # this drives a real stdio JSON-RPC handshake, so it must run in the cp1252 | |
| # locale a real user's MCP server starts in. Adding the var here would mask | |
| # encoding regressions in the boot/stdio path. | |
| mcp-e2e: | |
| name: MCP e2e nightly (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: develop | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: '3.11' | |
| cache: 'pip' | |
| # Simulates the documented user install (requirements.txt header): | |
| # python3 -m venv ~/.bitwize-music/venv && pip install -r requirements.txt | |
| # The venv path is computed with Path.home() so it agrees byte-for-byte | |
| # with run.py's venv discovery (on Windows, Path.home() uses USERPROFILE | |
| # and ignores HOME, so a shell $HOME would be ambiguous here). | |
| - name: Create user-style venv and install pinned requirements | |
| run: | | |
| python -c "import pathlib, venv; venv.create(pathlib.Path.home() / '.bitwize-music' / 'venv', with_pip=True)" | |
| VENV_PY="$(python -c "import pathlib, sys; d = pathlib.Path.home() / '.bitwize-music' / 'venv'; print((d / ('Scripts/python.exe' if sys.platform == 'win32' else 'bin/python3')).as_posix())")" | |
| "$VENV_PY" -m pip install --upgrade pip | |
| "$VENV_PY" -m pip install -r requirements.txt | |
| # Boot the server through the SAME launcher .mcp.json invokes, and drive a | |
| # real stdio JSON-RPC handshake. Catches process-level startup failures | |
| # (e.g. #476) that the in-process unit suite with its mocked FastMCP cannot | |
| # see. .mcp.json's command is the extensionless mcp-launch, which Claude Code | |
| # runs as the POSIX shebang script here and resolves to mcp-launch.cmd on | |
| # Windows (validated against real Claude Code on a Windows VM). | |
| - name: Boot check via mcp-launch (POSIX) | |
| if: runner.os != 'Windows' | |
| env: | |
| CLAUDE_PLUGIN_ROOT: ${{ github.workspace }} | |
| run: | | |
| python tests/e2e/mcp_boot_check.py --timeout 120 --call-tool health_check --scenario state-workflow -- \ | |
| "$GITHUB_WORKSPACE/servers/bitwize-music-server/mcp-launch" | |
| - name: Boot check via mcp-launch.cmd (Windows) | |
| if: runner.os == 'Windows' | |
| env: | |
| CLAUDE_PLUGIN_ROOT: ${{ github.workspace }} | |
| run: | | |
| python tests/e2e/mcp_boot_check.py --timeout 240 --call-tool health_check --scenario state-workflow -- \ | |
| "$GITHUB_WORKSPACE/servers/bitwize-music-server/mcp-launch.cmd" | |
| # Exercises the documented Windows support path ("Windows users: use WSL", | |
| # README) end-to-end: full pinned install + full suite inside a real WSL2 | |
| # distro on a hosted Windows runner. Distilled from the capability probes on | |
| # PR #485, where this exact recipe ran green (4256 passed, 28 skipped, | |
| # ~11.5 min suite; install ~48s). Native Windows legs can't cover this: the | |
| # audio externals are WSL-recommended by support-tier decision. | |
| # | |
| # IMPORTANT: do NOT set WSL_UTF8=1 at job level. Probe-established caveat: | |
| # setup-wsl v7.0.0 parses wsl.exe output as UTF-16LE, and a job-level | |
| # WSL_UTF8=1 switches that output to UTF-8, breaking the action's WSL | |
| # version detection — it then fails with | |
| # "This Windows environment only has WSLv1 available but WSLv2 was | |
| # requested, please verify your 'runs-on' and 'wsl-version' settings" | |
| # on an image where WSL2 demonstrably works (and in the wsl-version: 1 path | |
| # it silently skips version pinning instead of erroring). | |
| # Chromium launch smoke test for the document-hunter dependency. | |
| # | |
| # Lives in nightly, not PR CI, on cost: chromium is ~150MB per leg and almost | |
| # nothing exercises it. It runs on all three OSes because browser download and | |
| # launch are precisely the parts that differ per platform. | |
| # | |
| # `pip install playwright` does NOT install a browser — `playwright install | |
| # chromium` does, out-of-band — so the cheap import check in the normal suite | |
| # (tests/unit/shared/test_pinned_dependencies.py) proves nothing about whether | |
| # a browser is usable. This closes that half. | |
| playwright-smoke: | |
| name: Playwright Chromium smoke (${{ matrix.os }}) | |
| runs-on: ${{ matrix.os }} | |
| timeout-minutes: 25 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: [ubuntu-latest, windows-latest, macos-latest] | |
| env: | |
| BITWIZE_INTEGRATION: "1" | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: develop | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements-test.txt | |
| pip install -r requirements.txt | |
| # --with-deps pulls the system libraries chromium needs on Linux; it is a | |
| # no-op on macOS/Windows. | |
| - name: Install Chromium | |
| run: python -m playwright install --with-deps chromium | |
| - name: Run the Chromium smoke test | |
| run: | | |
| python -m pytest tests/integration/test_playwright_browser.py -m integration -v --tb=short | |
| wsl2-suite: | |
| name: WSL2 Suite (windows-latest) | |
| runs-on: windows-latest | |
| timeout-minutes: 45 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: develop | |
| - name: Set up WSL2 (Ubuntu-24.04) | |
| uses: Vampire/setup-wsl@d1da7f2c0322a5ee4f24975344f67fc0f5baf364 # v7.0.0 | |
| with: | |
| # Ubuntu-24.04 ships Python 3.12, which the pinned requirements | |
| # support; 22.04's Python 3.10 is below scipy==1.17.1's >=3.11 floor. | |
| distribution: Ubuntu-24.04 | |
| wsl-version: '2' | |
| use-cache: 'true' | |
| # Runs inside the WSL distro via the wsl-bash wrapper the action generates | |
| # (bash --noprofile --norc -euo pipefail — fail-fast by default). The | |
| # Windows workspace is visible through automatic cwd translation | |
| # (/mnt/d/...); job-level env does NOT cross the WSL boundary. No ffmpeg | |
| # inside WSL — ffmpeg-gated tests skip via shutil.which fixtures, matching | |
| # the probe benchmark (the native legs install ffmpeg since #484, but the | |
| # Windows-side choco install is invisible inside the distro). | |
| - name: Install pinned requirements and run full suite inside WSL2 | |
| shell: wsl-bash {0} | |
| run: | | |
| uname -a | |
| apt-get update -qq | |
| DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends python3 python3-venv python3-pip | |
| python3 -m venv /tmp/wsl-venv | |
| /tmp/wsl-venv/bin/pip install --upgrade pip -q | |
| /tmp/wsl-venv/bin/pip install -q -r requirements-test.txt -r requirements.txt | |
| # Exported inside the distro only (Linux defaults to UTF-8 anyway — | |
| # parity with the native legs' PYTHONUTF8 job env). | |
| export PYTHONUTF8=1 | |
| /tmp/wsl-venv/bin/python -m pytest tests/ --tb=short -n auto | |
| floor-check: | |
| # Falls back to a label when no version is given: the job is skipped in that | |
| # case (see `if` below), and GitHub does not expand an empty template on a | |
| # skipped job — so without this the run page shows the raw ${{ ... }} string. | |
| name: Python Floor Check (${{ inputs.check-python-floor || 'not requested' }}) | |
| runs-on: ubuntu-latest | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.check-python-floor != '' }} | |
| timeout-minutes: 30 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| with: | |
| ref: develop | |
| - name: Set up Python | |
| uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 | |
| with: | |
| python-version: ${{ inputs.check-python-floor }} | |
| # Install-only: settles whether the documented Python floor can still | |
| # install the pins (e.g. dispatch with 3.10). No tests are run — this | |
| # job exists purely to prove `pip install` succeeds on the floor version. | |
| - name: Install pinned requirements | |
| run: | | |
| pip install -r requirements.txt -r requirements-test.txt | |
| notify-failure: | |
| name: Notify on Failure | |
| needs: [deep-test, mcp-e2e, wsl2-suite] | |
| if: failure() | |
| runs-on: ubuntu-latest | |
| permissions: | |
| issues: write | |
| steps: | |
| - name: Open or update nightly-failure issue | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| run: | | |
| # `// empty` keeps $existing blank when no issue matches — a bare | |
| # `.[0].number` would yield the literal string "null" and route the | |
| # first-ever failure to `gh issue comment null`. | |
| existing=$(gh issue list -R "$GITHUB_REPOSITORY" --label nightly-failure --state open --json number -q '.[0].number // empty') | |
| url="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" | |
| if [ -n "$existing" ]; then | |
| gh issue comment "$existing" -R "$GITHUB_REPOSITORY" -b "Nightly failed again: $url" | |
| else | |
| gh issue create -R "$GITHUB_REPOSITORY" -t "Nightly deep test failure" -b "Run: $url" -l nightly-failure | |
| fi |