nightly-benchmark.yml

name: Nightly Cross-Runtime Benchmark

on:
  schedule:
    - cron: "30 14 * * *"  # 22:30 UTC+8, after diff-fuzz starts
  workflow_dispatch:
    inputs:
      modes:
        description: "Storage modes override (comma-separated, empty = use fixture default)"
        default: ""
      filter:
        description: "Only run fixtures matching this substring"
        default: ""
      skip:
        description: "Runtimes to skip (comma-separated)"
        default: ""
      generate:
        description: "Also generate synthetic fixtures"
        type: boolean
        default: false

permissions:
  actions: read

jobs:
  cross-runtime-benchmark:
    runs-on: ubuntu-latest
    timeout-minutes: 90
    env:
      GH_TOKEN: ${{ github.token }}
    steps:
      - uses: actions/checkout@v6

      - uses: actions/setup-go@v6
        with:
          go-version-file: pine-go/go.mod
          cache-dependency-path: pine-go/go.sum

      - uses: actions/setup-java@v5
        with:
          distribution: temurin
          java-version: "25"
          cache: maven

      - name: Install system deps
        # ci-apt-install.sh retries with a per-attempt timeout so one slow
        # mirror rotation doesn't burn the whole budget (#125, #164). cmake /
        # gcc / make (build-essential) are preinstalled on the runner image —
        # assert instead of apt-installing shadowed duplicates.
        run: |
          bash scripts/ci-apt-install.sh libluajit-5.1-dev libcurl4-openssl-dev
          cmake --version | head -1
          g++ --version | head -1

      - name: Install hey
        run: go install github.com/rakyll/hey@latest

      - name: Stop unneeded services for isolation
        run: |
          sudo systemctl stop mysql.service 2>/dev/null || true
          sudo systemctl stop postgresql.service 2>/dev/null || true
          sudo systemctl stop mono-xsp4.service 2>/dev/null || true
          sudo systemctl stop docker.service 2>/dev/null || true
          sudo systemctl stop snapd.service 2>/dev/null || true

      - name: Download previous successful report
        id: prev
        run: |
          PREV_RUN_ID=$(gh run list \
            --workflow="nightly-benchmark.yml" \
            --status=success \
            -L1 --json databaseId -q '.[0].databaseId' 2>/dev/null || echo "")
          if [ -n "$PREV_RUN_ID" ] && [ "$PREV_RUN_ID" != "null" ]; then
            mkdir -p /tmp/bench_prev
            ARTIFACT_NAME=$(gh api "repos/${{ github.repository }}/actions/runs/${PREV_RUN_ID}/artifacts" \
              --jq '.artifacts[] | select(.name | startswith("cross-runtime-benchmark-")) | .name' \
              | head -1)
            if [ -n "$ARTIFACT_NAME" ]; then
              gh run download "$PREV_RUN_ID" -n "$ARTIFACT_NAME" -D /tmp/bench_prev/ 2>/dev/null || true
            fi
            if [ -f /tmp/bench_prev/report.txt ]; then
              echo "found=true" >> "$GITHUB_OUTPUT"
            else
              echo "found=false" >> "$GITHUB_OUTPUT"
            fi
          else
            echo "found=false" >> "$GITHUB_OUTPUT"
          fi

      - name: Run cross-runtime benchmark
        run: |
          ARGS=""
          MODES="${{ inputs.modes || '' }}"
          FILTER="${{ inputs.filter || '' }}"
          SKIP="${{ inputs.skip || '' }}"
          GENERATE="${{ inputs.generate || 'false' }}"
          if [ -n "$MODES" ]; then
            ARGS="$ARGS --modes $MODES"
          fi
          if [ -n "$FILTER" ]; then
            ARGS="$ARGS --filter $FILTER"
          fi
          if [ -n "$SKIP" ]; then
            ARGS="$ARGS --skip $SKIP"
          fi
          if [ "$GENERATE" = "true" ]; then
            ARGS="$ARGS --generate"
          fi
          bash scripts/bench-cross-runtime.sh $ARGS \
            2>&1 | tee bench-output.log

      - name: Generate comparison report
        if: steps.prev.outputs.found == 'true'
        run: |
          python3 scripts/bench-compare.py \
            --prev /tmp/bench_prev/report.txt \
            --curr bench-results/report.txt \
            --output bench-results/comparison.txt

      - name: Analyze benchmark results
        if: always()
        run: |
          if [ -f bench-results/report.txt ]; then
            python3 scripts/bench-analyze.py bench-results/report.txt \
              -o bench-results/analysis.txt
            python3 scripts/bench-analyze.py bench-results/report.txt \
              --json -o bench-results/analysis.json
          fi

      - name: Write benchmark summary
        if: always()
        shell: bash
        run: |
          {
            echo "### Cross-Runtime Benchmark"
            echo ""
            if [ -f bench-results/analysis.txt ]; then
              echo "#### Analysis"
              echo '```text'
              cat bench-results/analysis.txt
              echo '```'
              echo ""
            fi
            if [ -f bench-results/comparison.txt ]; then
              echo "#### Delta vs previous run"
              echo '```text'
              cat bench-results/comparison.txt
              echo '```'
              echo ""
            fi
            echo "#### Full results"
            echo '```text'
            cat bench-results/report.txt 2>/dev/null || echo "No report generated"
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"

      - name: Upload benchmark report
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: cross-runtime-benchmark-${{ github.run_number }}
          # report.txt is the stable alias the next run's `gh run download`
          # looks for by name; keep it first and keep the name unchanged.
          path: |
            bench-results/report.txt
            bench-results/comparison.txt
            bench-results/analysis.txt
            bench-results/analysis.json
          retention-days: 30

      - name: Notify via Bark
        if: always()
        env:
          BARK_KEY: ${{ secrets.BARK_DEVICE_KEY }}
        run: |
          if [ -z "$BARK_KEY" ]; then
            echo "BARK_DEVICE_KEY not set, skipping notification"
            exit 0
          fi
          RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
          if [ -f bench-results/analysis.txt ]; then
            BODY=$(grep -A5 "## Verdict" bench-results/analysis.txt | tail -n+2)
          elif [ -f bench-results/comparison.txt ]; then
            BODY=$(head -20 bench-results/comparison.txt)
          elif [ -f bench-results/report.txt ]; then
            BODY=$(grep -E "^\s+(go|java|cpp)" bench-results/report.txt | head -12)
          else
            BODY="No report generated. Check logs."
          fi
          STATUS="${{ job.status }}"
          TITLE="Benchmark ${STATUS}"
          ENCODED_BODY=$(echo "$BODY" | python3 -c "import sys,urllib.parse; print(urllib.parse.quote(sys.stdin.read()))")
          curl -sf "http://api.day.app/${BARK_KEY}/${TITLE}/${ENCODED_BODY}?url=${RUN_URL}&group=pineapple-ci" \
            || echo "Bark notification failed (non-fatal)"