rust-benchmark.yml

name: Rust Benchmark

on:
  push:
    branches: [main, master]
    paths: ['rust/**', 'spacetime-module/**', '.github/workflows/rust-benchmark.yml']
  pull_request:
    branches: [main, master]
    paths: ['rust/**', 'spacetime-module/**', '.github/workflows/rust-benchmark.yml']

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1

defaults:
  run:
    working-directory: rust

jobs:
  test:
    name: Test (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest]
    steps:
      - uses: actions/checkout@v4

      - name: Setup Rust (nightly)
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly
          components: rustfmt, clippy
          targets: wasm32-unknown-unknown

      - name: Cache cargo registry
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: rust -> target
          cache-on-failure: "true"

      - name: Check formatting
        run: cargo fmt --all -- --check

      - name: Run Clippy
        run: cargo clippy --all-targets

      - name: Install SpacetimeDB CLI
        shell: bash
        run: |
          curl -sSf https://install.spacetimedb.com | sh -s -- -y
          echo "$HOME/.local/bin" >> $GITHUB_PATH
        working-directory: .

      - name: Build SpacetimeDB module (WASM)
        run: cargo build --release --target wasm32-unknown-unknown
        working-directory: rust/spacetime-module

      - name: Start SpacetimeDB server
        shell: bash
        run: |
          spacetime start &
          echo "Waiting for SpacetimeDB server to be ready..."
          for i in $(seq 1 30); do
            if curl -sf http://localhost:3000/ > /dev/null 2>&1; then
              echo "SpacetimeDB server is ready"
              break
            fi
            sleep 1
          done
        working-directory: .

      - name: Publish SpacetimeDB module
        shell: bash
        run: |
          spacetime publish \
            --server http://localhost:3000 \
            --bin-path target/wasm32-unknown-unknown/release/spacetime_module.wasm \
            --yes \
            benchmark-links
        working-directory: rust/spacetime-module

      - name: Run tests
        env:
          SPACETIMEDB_URI: http://localhost:3000
          SPACETIMEDB_DB: benchmark-links
        # Run tests sequentially to avoid parallel interference with shared SpacetimeDB state.
        run: cargo test -- --test-threads=1

  # Quick benchmark validation for pull requests.
  # Runs benchmarks with reduced scale to verify they work and produce results
  # in well under 10 minutes. Results are not committed but uploaded as artifacts.
  #
  # Parameters chosen to keep total benchmark time under 5 minutes:
  #   BENCHMARK_LINK_COUNT=10, BACKGROUND_LINK_COUNT=30: reduces SpacetimeDB
  #   round trips per iteration from ~8000 to ~80, making each iteration ~0.1s.
  #   --sample-size 10: collect 10 samples per benchmark (instead of 100 default).
  #   --warm-up-time 1: 1s warm-up instead of 3s default.
  #   --measurement-time 2: 2s measurement instead of 5s default.
  #   Expected runtime: ~3-5 minutes total for all 35 benchmarks.
  benchmark-pr:
    name: Benchmark (PR validation)
    runs-on: ubuntu-latest
    needs: [test]
    if: github.event_name == 'pull_request'
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v4

      - name: Setup Rust (nightly)
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly
          targets: wasm32-unknown-unknown

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: pip install matplotlib numpy

      - name: Cache cargo registry
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: rust -> target
          cache-on-failure: "true"

      - name: Install SpacetimeDB CLI
        run: |
          curl -sSf https://install.spacetimedb.com | sh -s -- -y
          echo "$HOME/.local/bin" >> $GITHUB_PATH
        working-directory: .

      - name: Build SpacetimeDB module (WASM)
        run: cargo build --release --target wasm32-unknown-unknown
        working-directory: rust/spacetime-module

      - name: Start SpacetimeDB server
        run: |
          spacetime start &
          for i in $(seq 1 30); do
            if curl -sf http://localhost:3000/ > /dev/null 2>&1; then
              echo "SpacetimeDB server is ready"
              break
            fi
            sleep 1
          done
        working-directory: .

      - name: Publish SpacetimeDB module
        run: |
          spacetime publish \
            --server http://localhost:3000 \
            --bin-path target/wasm32-unknown-unknown/release/spacetime_module.wasm \
            --yes \
            benchmark-links
        working-directory: rust/spacetime-module

      - name: Build benchmark
        run: cargo build --release

      - name: Run benchmark (quick mode for PR validation)
        env:
          # Reduced scale: 10 links instead of 1000, 30 background instead of 3000.
          # This reduces SpacetimeDB round trips per iteration from ~8000 to ~80,
          # keeping each iteration under 0.1s and total benchmark time under 5 minutes.
          BENCHMARK_LINK_COUNT: 10
          BACKGROUND_LINK_COUNT: 30
          SPACETIMEDB_URI: http://localhost:3000
          SPACETIMEDB_DB: benchmark-links
        run: |
          cargo bench --bench bench -- \
            --output-format bencher \
            --sample-size 10 \
            --warm-up-time 1 \
            --measurement-time 2 \
            --nresamples 1000 \
            | tee out.txt

      - name: Generate charts
        run: python3 out.py

      - name: Upload PR benchmark artifacts
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results-pr
          path: |
            rust/out.txt
            rust/bench_rust.png
            rust/bench_rust_log_scale.png

  # Full benchmark run for commits to main/master.
  # Uses full scale (1000 links, 3000 background) with reduced sample count
  # to produce statistically meaningful results while fitting within 1 hour.
  #
  # Parameters:
  #   BENCHMARK_LINK_COUNT=1000, BACKGROUND_LINK_COUNT=3000: realistic scale.
  #   --sample-size 20: 20 samples per benchmark (down from 100 default).
  #   --nresamples 10000: 10k bootstrap resamples (down from 100k default).
  #   Expected runtime: ~30-45 minutes total for all 35 benchmarks.
  benchmark:
    name: Benchmark (full)
    runs-on: ubuntu-latest
    needs: [test]
    if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')
    timeout-minutes: 180
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Setup Rust (nightly)
        uses: dtolnay/rust-toolchain@master
        with:
          toolchain: nightly
          targets: wasm32-unknown-unknown

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Install Python dependencies
        run: pip install matplotlib numpy

      - name: Cache cargo registry
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: rust -> target
          cache-on-failure: "true"

      - name: Install SpacetimeDB CLI
        run: |
          curl -sSf https://install.spacetimedb.com | sh -s -- -y
          echo "$HOME/.local/bin" >> $GITHUB_PATH
        working-directory: .

      - name: Build SpacetimeDB module (WASM)
        run: cargo build --release --target wasm32-unknown-unknown
        working-directory: rust/spacetime-module

      - name: Start SpacetimeDB server
        run: |
          spacetime start &
          for i in $(seq 1 30); do
            if curl -sf http://localhost:3000/ > /dev/null 2>&1; then
              echo "SpacetimeDB server is ready"
              break
            fi
            sleep 1
          done
        working-directory: .

      - name: Publish SpacetimeDB module
        run: |
          spacetime publish \
            --server http://localhost:3000 \
            --bin-path target/wasm32-unknown-unknown/release/spacetime_module.wasm \
            --yes \
            benchmark-links
        working-directory: rust/spacetime-module

      - name: Build benchmark
        run: cargo build --release

      - name: Run benchmark (full mode for main branch)
        env:
          # Full scale: 1000 links, 3000 background for realistic results.
          # --sample-size 20 reduces total runtime from ~2h (default 100) to ~25-40 min
          # while still providing statistically valid measurements.
          BENCHMARK_LINK_COUNT: 1000
          BACKGROUND_LINK_COUNT: 3000
          SPACETIMEDB_URI: http://localhost:3000
          SPACETIMEDB_DB: benchmark-links
        run: |
          cargo bench --bench bench -- \
            --output-format bencher \
            --sample-size 20 \
            --nresamples 10000 \
            | tee out.txt

      - name: Generate charts
        run: python3 out.py

      - name: Configure git
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

      - name: Commit benchmark results
        run: |
          git add -f out.txt bench_rust.png bench_rust_log_scale.png 2>/dev/null || true
          git diff --staged --quiet || git commit -m "chore: update benchmark results [skip ci]"
          git push

      - name: Upload benchmark artifacts
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-results
          path: |
            rust/out.txt
            rust/bench_rust.png
            rust/bench_rust_log_scale.png