Nightly Cross-Runtime Benchmark #68
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Cross-Runtime Benchmark | |
| on: | |
| schedule: | |
| - cron: "30 14 * * *" # 22:30 UTC+8, after diff-fuzz starts | |
| workflow_dispatch: | |
| inputs: | |
| modes: | |
| description: "Storage modes override (comma-separated, empty = use fixture default)" | |
| default: "" | |
| filter: | |
| description: "Only run fixtures matching this substring" | |
| default: "" | |
| skip: | |
| description: "Runtimes to skip (comma-separated)" | |
| default: "" | |
| generate: | |
| description: "Also generate synthetic fixtures" | |
| type: boolean | |
| default: false | |
| permissions: | |
| actions: read | |
| jobs: | |
| cross-runtime-benchmark: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: pine-go/go.mod | |
| cache-dependency-path: pine-go/go.sum | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: temurin | |
| java-version: "25" | |
| cache: maven | |
| - name: Install system deps | |
| # ci-apt-install.sh retries with a per-attempt timeout so one slow | |
| # mirror rotation doesn't burn the whole budget (#125, #164). cmake / | |
| # gcc / make (build-essential) are preinstalled on the runner image — | |
| # assert instead of apt-installing shadowed duplicates. | |
| run: | | |
| bash scripts/ci-apt-install.sh libluajit-5.1-dev libcurl4-openssl-dev | |
| cmake --version | head -1 | |
| g++ --version | head -1 | |
| - name: Install hey | |
| run: go install github.com/rakyll/hey@latest | |
| - name: Stop unneeded services for isolation | |
| run: | | |
| sudo systemctl stop mysql.service 2>/dev/null || true | |
| sudo systemctl stop postgresql.service 2>/dev/null || true | |
| sudo systemctl stop mono-xsp4.service 2>/dev/null || true | |
| sudo systemctl stop docker.service 2>/dev/null || true | |
| sudo systemctl stop snapd.service 2>/dev/null || true | |
| - name: Download previous successful report | |
| id: prev | |
| run: | | |
| PREV_RUN_ID=$(gh run list \ | |
| --workflow="nightly-benchmark.yml" \ | |
| --status=success \ | |
| -L1 --json databaseId -q '.[0].databaseId' 2>/dev/null || echo "") | |
| if [ -n "$PREV_RUN_ID" ] && [ "$PREV_RUN_ID" != "null" ]; then | |
| mkdir -p /tmp/bench_prev | |
| ARTIFACT_NAME=$(gh api "repos/${{ github.repository }}/actions/runs/${PREV_RUN_ID}/artifacts" \ | |
| --jq '.artifacts[] | select(.name | startswith("cross-runtime-benchmark-")) | .name' \ | |
| | head -1) | |
| if [ -n "$ARTIFACT_NAME" ]; then | |
| gh run download "$PREV_RUN_ID" -n "$ARTIFACT_NAME" -D /tmp/bench_prev/ 2>/dev/null || true | |
| fi | |
| if [ -f /tmp/bench_prev/report.txt ]; then | |
| echo "found=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "found=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| echo "found=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Run cross-runtime benchmark | |
| run: | | |
| ARGS="" | |
| MODES="${{ inputs.modes || '' }}" | |
| FILTER="${{ inputs.filter || '' }}" | |
| SKIP="${{ inputs.skip || '' }}" | |
| GENERATE="${{ inputs.generate || 'false' }}" | |
| if [ -n "$MODES" ]; then | |
| ARGS="$ARGS --modes $MODES" | |
| fi | |
| if [ -n "$FILTER" ]; then | |
| ARGS="$ARGS --filter $FILTER" | |
| fi | |
| if [ -n "$SKIP" ]; then | |
| ARGS="$ARGS --skip $SKIP" | |
| fi | |
| if [ "$GENERATE" = "true" ]; then | |
| ARGS="$ARGS --generate" | |
| fi | |
| bash scripts/bench-cross-runtime.sh $ARGS \ | |
| 2>&1 | tee bench-output.log | |
| - name: Generate comparison report | |
| if: steps.prev.outputs.found == 'true' | |
| run: | | |
| python3 scripts/bench-compare.py \ | |
| --prev /tmp/bench_prev/report.txt \ | |
| --curr bench-results/report.txt \ | |
| --output bench-results/comparison.txt | |
| - name: Analyze benchmark results | |
| if: always() | |
| run: | | |
| if [ -f bench-results/report.txt ]; then | |
| python3 scripts/bench-analyze.py bench-results/report.txt \ | |
| -o bench-results/analysis.txt | |
| python3 scripts/bench-analyze.py bench-results/report.txt \ | |
| --json -o bench-results/analysis.json | |
| fi | |
| - name: Write benchmark summary | |
| if: always() | |
| shell: bash | |
| run: | | |
| { | |
| echo "### Cross-Runtime Benchmark" | |
| echo "" | |
| if [ -f bench-results/analysis.txt ]; then | |
| echo "#### Analysis" | |
| echo '```text' | |
| cat bench-results/analysis.txt | |
| echo '```' | |
| echo "" | |
| fi | |
| if [ -f bench-results/comparison.txt ]; then | |
| echo "#### Delta vs previous run" | |
| echo '```text' | |
| cat bench-results/comparison.txt | |
| echo '```' | |
| echo "" | |
| fi | |
| echo "#### Full results" | |
| echo '```text' | |
| cat bench-results/report.txt 2>/dev/null || echo "No report generated" | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload benchmark report | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: cross-runtime-benchmark-${{ github.run_number }} | |
| # report.txt is the stable alias the next run's `gh run download` | |
| # looks for by name; keep it first and keep the name unchanged. | |
| path: | | |
| bench-results/report.txt | |
| bench-results/comparison.txt | |
| bench-results/analysis.txt | |
| bench-results/analysis.json | |
| retention-days: 30 | |
| - name: Notify via Bark | |
| if: always() | |
| env: | |
| BARK_KEY: ${{ secrets.BARK_DEVICE_KEY }} | |
| run: | | |
| if [ -z "$BARK_KEY" ]; then | |
| echo "BARK_DEVICE_KEY not set, skipping notification" | |
| exit 0 | |
| fi | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| if [ -f bench-results/analysis.txt ]; then | |
| BODY=$(grep -A5 "## Verdict" bench-results/analysis.txt | tail -n+2) | |
| elif [ -f bench-results/comparison.txt ]; then | |
| BODY=$(head -20 bench-results/comparison.txt) | |
| elif [ -f bench-results/report.txt ]; then | |
| BODY=$(grep -E "^\s+(go|java|cpp)" bench-results/report.txt | head -12) | |
| else | |
| BODY="No report generated. Check logs." | |
| fi | |
| STATUS="${{ job.status }}" | |
| TITLE="Benchmark ${STATUS}" | |
| ENCODED_BODY=$(echo "$BODY" | python3 -c "import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read()))") | |
| curl -sf "http://api.day.app/${BARK_KEY}/${TITLE}/${ENCODED_BODY}?url=${RUN_URL}&group=pineapple-ci" \ | |
| || echo "Bark notification failed (non-fatal)" |