Nightly Cross-Runtime Benchmark #58
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Nightly Cross-Runtime Benchmark | |
| on: | |
| schedule: | |
| - cron: "30 14 * * *" # 22:30 UTC+8, after diff-fuzz starts | |
| workflow_dispatch: | |
| inputs: | |
| modes: | |
| description: "Storage modes override (comma-separated, empty = use fixture default)" | |
| default: "" | |
| filter: | |
| description: "Only run fixtures matching this substring" | |
| default: "" | |
| skip: | |
| description: "Runtimes to skip (comma-separated)" | |
| default: "" | |
| generate: | |
| description: "Also generate synthetic fixtures" | |
| type: boolean | |
| default: false | |
| permissions: | |
| actions: read | |
| jobs: | |
| cross-runtime-benchmark: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 90 | |
| env: | |
| GH_TOKEN: ${{ github.token }} | |
| steps: | |
| - uses: actions/checkout@v6 | |
| - uses: actions/setup-go@v6 | |
| with: | |
| go-version-file: pine-go/go.mod | |
| cache-dependency-path: pine-go/go.sum | |
| - uses: actions/setup-java@v5 | |
| with: | |
| distribution: temurin | |
| java-version: "25" | |
| cache: maven | |
| - name: Install system deps | |
| # ci-apt-install.sh retries with a per-attempt timeout so one slow | |
| # mirror rotation doesn't burn the whole budget (#125, #164). cmake / | |
| # gcc / make (build-essential) are preinstalled on the runner image — | |
| # assert instead of apt-installing shadowed duplicates. | |
| run: | | |
| bash scripts/ci-apt-install.sh libluajit-5.1-dev libcurl4-openssl-dev | |
| cmake --version | head -1 | |
| g++ --version | head -1 | |
| - name: Install hey | |
| run: go install github.com/rakyll/hey@latest | |
| - name: Stop unneeded services for isolation | |
| run: | | |
| sudo systemctl stop mysql.service 2>/dev/null || true | |
| sudo systemctl stop postgresql.service 2>/dev/null || true | |
| sudo systemctl stop mono-xsp4.service 2>/dev/null || true | |
| sudo systemctl stop docker.service 2>/dev/null || true | |
| sudo systemctl stop snapd.service 2>/dev/null || true | |
| - name: Download previous successful report | |
| id: prev | |
| run: | | |
| PREV_RUN_ID=$(gh run list \ | |
| --workflow="nightly-benchmark.yml" \ | |
| --status=success \ | |
| -L1 --json databaseId -q '.[0].databaseId' 2>/dev/null || echo "") | |
| if [ -n "$PREV_RUN_ID" ] && [ "$PREV_RUN_ID" != "null" ]; then | |
| mkdir -p /tmp/bench_prev | |
| ARTIFACT_NAME=$(gh api "repos/${{ github.repository }}/actions/runs/${PREV_RUN_ID}/artifacts" \ | |
| --jq '.artifacts[] | select(.name | startswith("cross-runtime-benchmark-")) | .name' \ | |
| | head -1) | |
| if [ -n "$ARTIFACT_NAME" ]; then | |
| gh run download "$PREV_RUN_ID" -n "$ARTIFACT_NAME" -D /tmp/bench_prev/ 2>/dev/null || true | |
| fi | |
| if [ -f /tmp/bench_prev/report.txt ]; then | |
| echo "found=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "found=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| else | |
| echo "found=false" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Run cross-runtime benchmark | |
| run: | | |
| ARGS="" | |
| MODES="${{ inputs.modes || '' }}" | |
| FILTER="${{ inputs.filter || '' }}" | |
| SKIP="${{ inputs.skip || '' }}" | |
| GENERATE="${{ inputs.generate || 'false' }}" | |
| if [ -n "$MODES" ]; then | |
| ARGS="$ARGS --modes $MODES" | |
| fi | |
| if [ -n "$FILTER" ]; then | |
| ARGS="$ARGS --filter $FILTER" | |
| fi | |
| if [ -n "$SKIP" ]; then | |
| ARGS="$ARGS --skip $SKIP" | |
| fi | |
| if [ "$GENERATE" = "true" ]; then | |
| ARGS="$ARGS --generate" | |
| fi | |
| bash scripts/bench-cross-runtime.sh $ARGS \ | |
| 2>&1 | tee bench-output.log | |
| - name: Generate comparison report | |
| if: steps.prev.outputs.found == 'true' | |
| run: | | |
| python3 scripts/bench-compare.py \ | |
| --prev /tmp/bench_prev/report.txt \ | |
| --curr /tmp/bench_cross_runtime/report.txt \ | |
| --output /tmp/bench_cross_runtime/comparison.txt | |
| - name: Analyze benchmark results | |
| if: always() | |
| run: | | |
| if [ -f /tmp/bench_cross_runtime/report.txt ]; then | |
| python3 scripts/bench-analyze.py /tmp/bench_cross_runtime/report.txt \ | |
| -o /tmp/bench_cross_runtime/analysis.txt | |
| python3 scripts/bench-analyze.py /tmp/bench_cross_runtime/report.txt \ | |
| --json -o /tmp/bench_cross_runtime/analysis.json | |
| fi | |
| - name: Write benchmark summary | |
| if: always() | |
| shell: bash | |
| run: | | |
| { | |
| echo "### Cross-Runtime Benchmark" | |
| echo "" | |
| if [ -f /tmp/bench_cross_runtime/analysis.txt ]; then | |
| echo "#### Analysis" | |
| echo '```text' | |
| cat /tmp/bench_cross_runtime/analysis.txt | |
| echo '```' | |
| echo "" | |
| fi | |
| if [ -f /tmp/bench_cross_runtime/comparison.txt ]; then | |
| echo "#### Delta vs previous run" | |
| echo '```text' | |
| cat /tmp/bench_cross_runtime/comparison.txt | |
| echo '```' | |
| echo "" | |
| fi | |
| echo "#### Full results" | |
| echo '```text' | |
| cat /tmp/bench_cross_runtime/report.txt 2>/dev/null || echo "No report generated" | |
| echo '```' | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload benchmark report | |
| uses: actions/upload-artifact@v7 | |
| if: always() | |
| with: | |
| name: cross-runtime-benchmark-${{ github.run_number }} | |
| path: | | |
| /tmp/bench_cross_runtime/report.txt | |
| /tmp/bench_cross_runtime/comparison.txt | |
| /tmp/bench_cross_runtime/analysis.txt | |
| /tmp/bench_cross_runtime/analysis.json | |
| /tmp/bench_cross_runtime/*.csv | |
| retention-days: 30 | |
| - name: Notify via Bark | |
| if: always() | |
| env: | |
| BARK_KEY: ${{ secrets.BARK_DEVICE_KEY }} | |
| run: | | |
| if [ -z "$BARK_KEY" ]; then | |
| echo "BARK_DEVICE_KEY not set, skipping notification" | |
| exit 0 | |
| fi | |
| RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" | |
| if [ -f /tmp/bench_cross_runtime/analysis.txt ]; then | |
| BODY=$(grep -A5 "## Verdict" /tmp/bench_cross_runtime/analysis.txt | tail -n+2) | |
| elif [ -f /tmp/bench_cross_runtime/comparison.txt ]; then | |
| BODY=$(head -20 /tmp/bench_cross_runtime/comparison.txt) | |
| elif [ -f /tmp/bench_cross_runtime/report.txt ]; then | |
| BODY=$(grep -E "^\s+(go|java|cpp)" /tmp/bench_cross_runtime/report.txt | head -12) | |
| else | |
| BODY="No report generated. Check logs." | |
| fi | |
| STATUS="${{ job.status }}" | |
| TITLE="Benchmark ${STATUS}" | |
| ENCODED_BODY=$(echo "$BODY" | python3 -c "import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read()))") | |
| curl -sf "http://api.day.app/${BARK_KEY}/${TITLE}/${ENCODED_BODY}?url=${RUN_URL}&group=pineapple-ci" \ | |
| || echo "Bark notification failed (non-fatal)" |