perf.yml

name: Performance Tests

on:
  pull_request:
  push:
    branches:
      - main
  workflow_dispatch:
    inputs:
      compare_upstream:
        type: boolean
        default: true
        description: Compare against upstream quinn
      publish_metrics:
        type: boolean
        default: false
        description: Publish metrics to perf.iroh.computer

concurrency:
  group: perf-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  RUST_BACKTRACE: 1
  RUSTFLAGS: -Dwarnings
  SCCACHE_GHA_ENABLED: "true"
  RUSTC_WRAPPER: "sccache"

jobs:
  build:
    name: Build ${{ matrix.impl }} perf
    runs-on: [self-hosted, linux, X64]
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - impl: noq
            artifact: noq-perf
          - impl: upstream-quinn
            artifact: quinn-perf-upstream
    steps:
      - uses: actions/checkout@v6
        if: matrix.impl == 'noq'

      - uses: dtolnay/rust-toolchain@stable

      - uses: mozilla-actions/sccache-action@v0.0.9

      - name: Get latest quinn release tag
        if: matrix.impl == 'upstream-quinn'
        id: quinn-release
        run: |
          LATEST_TAG=$(curl -s https://api.github.com/repos/quinn-rs/quinn/releases/latest | jq -r '.tag_name')
          echo "tag=$LATEST_TAG" >> $GITHUB_OUTPUT
          echo "Using upstream quinn release: $LATEST_TAG"

      - name: Clone upstream quinn
        if: matrix.impl == 'upstream-quinn'
        run: |
          rm -rf /tmp/upstream-quinn
          git clone --depth 1 --branch ${{ steps.quinn-release.outputs.tag }} \
            https://github.com/quinn-rs/quinn.git /tmp/upstream-quinn

      - name: Build perf binary
        run: cargo build --release -p perf --features json-output
        working-directory: ${{ matrix.impl == 'upstream-quinn' && '/tmp/upstream-quinn' || '.' }}

      - name: Upload binary
        uses: actions/upload-artifact@v5
        with:
          name: ${{ matrix.artifact }}
          path: ${{ matrix.impl == 'upstream-quinn' && '/tmp/upstream-quinn/target/release/quinn-perf' || 'target/release/noq-perf' }}
          retention-days: 1

  raw-perf:
    name: Raw perf ${{ matrix.impl }}
    needs: build
    runs-on: [self-hosted, linux, X64]
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        impl: [noq, upstream-quinn]
        include:
          - impl: noq
            artifact: noq-perf
          - impl: upstream-quinn
            artifact: quinn-perf-upstream
    steps:
      - uses: actions/checkout@v6

      - uses: actions/download-artifact@v5
        with:
          name: ${{ matrix.artifact }}
          path: ./bin

      - name: Run benchmarks
        run: |
          PERF_BIN=$(ls ./bin/noq-perf ./bin/quinn-perf 2>/dev/null | head -1)
          chmod +x "$PERF_BIN"
          ./.github/scripts/run_perf.sh "$PERF_BIN" results/${{ matrix.impl }}

      - uses: actions/upload-artifact@v5
        with:
          name: raw-results-${{ matrix.impl }}
          path: results/
          retention-days: 3

  netsim-perf:
    name: Netsim perf ${{ matrix.impl }}
    needs: build
    runs-on: [self-hosted, linux, X64]
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        impl: [noq, upstream-quinn]
        include:
          - impl: noq
            artifact: noq-perf
          - impl: upstream-quinn
            artifact: quinn-perf-upstream
    steps:
      - uses: actions/checkout@v6

      - uses: actions/download-artifact@v5
        with:
          name: ${{ matrix.artifact }}
          path: ./bin

      - name: Fetch netsim
        run: |
          rm -rf /tmp/chuck
          git clone --depth 1 https://github.com/n0-computer/chuck.git /tmp/chuck

      - name: Install netsim deps
        run: |
          cd /tmp/chuck/netsim
          sudo apt-get update
          ./setup.sh || true
          ./cleanup.sh || true

      - name: Setup netsim bins
        run: |
          mkdir -p /tmp/chuck/netsim/bins
          if [ -f ./bin/noq-perf ]; then
            chmod +x ./bin/noq-perf
            cp ./bin/noq-perf /tmp/chuck/netsim/bins/noq-perf
          else
            chmod +x ./bin/quinn-perf
            cp ./bin/quinn-perf /tmp/chuck/netsim/bins/noq-perf
          fi

      - name: Run netsim perf test
        id: run_netsim
        continue-on-error: true
        run: |
          ./.github/scripts/run_netsim_perf.sh ${{ matrix.impl }} /tmp/chuck/netsim

      - name: Collect results
        if: always()
        run: |
          mkdir -p results/${{ matrix.impl }}
          cp /tmp/chuck/netsim/report/*.json results/${{ matrix.impl }}/ 2>/dev/null || true
          cp /tmp/chuck/netsim/logs/*.log results/${{ matrix.impl }}/ 2>/dev/null || true

      - uses: actions/upload-artifact@v5
        if: always()
        with:
          name: netsim-results-${{ matrix.impl }}
          path: results/
          retention-days: 3

      - name: Cleanup netsim
        if: always()
        run: |
          cd /tmp/chuck/netsim
          ./cleanup.sh || true

  compare:
    name: Compare results
    needs: [raw-perf, netsim-perf]
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v6

      - uses: actions/download-artifact@v5
        with:
          pattern: raw-results-*
          merge-multiple: true
          path: raw-results/

      - uses: actions/download-artifact@v5
        with:
          pattern: netsim-results-*
          merge-multiple: true
          path: netsim-results/

      - name: Setup Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'

      - name: Generate comparison report
        id: compare
        run: |
          python .github/scripts/compare_results.py raw-results/ netsim-results/ > comparison.md
          echo "COMPARISON<<EOF" >> $GITHUB_OUTPUT
          cat comparison.md >> $GITHUB_OUTPUT
          echo "EOF" >> $GITHUB_OUTPUT

      - name: Convert to Metro format
        run: |
          python .github/scripts/perf_to_metro.py raw-results/ netsim-results/ \
            --commit ${{ github.sha }} > metro.json

      - name: Show results
        run: |
          echo "=== Comparison Report ==="
          cat comparison.md
          echo ""
          echo "=== Metro Metrics ==="
          cat metro.json

      - name: Upload comparison report
        uses: actions/upload-artifact@v5
        with:
          name: perf-comparison-report
          path: |
            comparison.md
            metro.json
          retention-days: 7

      - name: Publish metrics to Metro
        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
        run: |
          if [[ -n "${{ secrets.METRO_TOKEN }}" && -n "${{ secrets.METRO_ENDPOINT }}" ]]; then
            curl -X POST \
              -H "Content-Type: application/json" \
              -H "Authorization: Bearer ${{ secrets.METRO_TOKEN }}" \
              --data @metro.json \
              "${{ secrets.METRO_ENDPOINT }}"
          else
            echo "Metro secrets not configured, skipping publish"
          fi

      - name: Find existing comment
        if: github.event_name == 'pull_request'
        uses: peter-evans/find-comment@v3
        id: find-comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: '<!-- perf-report -->'

      - name: Create new PR comment
        if: github.event_name == 'pull_request' && steps.find-comment.outputs.comment-id == ''
        uses: peter-evans/create-or-update-comment@v5
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body: |
            <!-- perf-report -->
            ## Performance Comparison Report

            <details>
            <summary><code>${{ github.event.pull_request.head.sha }}</code> - <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">artifacts</a></summary>

            ${{ steps.compare.outputs.COMPARISON }}

            </details>

      - name: Update existing PR comment
        if: github.event_name == 'pull_request' && steps.find-comment.outputs.comment-id != ''
        uses: peter-evans/create-or-update-comment@v5
        with:
          comment-id: ${{ steps.find-comment.outputs.comment-id }}
          edit-mode: append
          body: |

            ---
            <details>
            <summary><code>${{ github.event.pull_request.head.sha }}</code> - <a href="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}">artifacts</a></summary>

            ${{ steps.compare.outputs.COMPARISON }}

            </details>