dwarf-perf-regression.yml

name: DWARF Perf Regression

on:
  pull_request:
    branches: [ main ]
    paths:
      - 'ghostscope-dwarf/**'
      - 'bins/dwarf-tool/**'
      - 'scripts/dwarf-perf/**'
      - 'Cargo.toml'
      - 'Cargo.lock'
      - '.github/workflows/dwarf-perf-regression.yml'

permissions:
  contents: read

env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
  LLVM_SYS_181_PREFIX: /usr/lib/llvm-18

jobs:
  regression:
    name: DWARF Perf Regression
    runs-on: ubuntu-latest
    timeout-minutes: 75
    env:
      BASE_SHA: ${{ github.event.pull_request.base.sha }}
      HEAD_SHA: ${{ github.event.pull_request.head.sha }}
      CARGO_TARGET_DIR: ${{ github.workspace }}/.target_tmp/dwarf-perf-regression
      DWARF_PERF_RESULTS_DIR: ${{ github.workspace }}/.tmp/dwarf-perf-results
      DWARF_PERF_CHANGED_FILES: ${{ github.workspace }}/.tmp/dwarf-perf-changed-files.txt
    steps:
      - name: Checkout head
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          path: head
          fetch-depth: 0
          submodules: recursive

      - name: Detect benchmark mode
        id: detect
        working-directory: head
        run: |
          set -euo pipefail

          base_supported=true
          for required_path in \
            scripts/dwarf-perf/build_corpus.sh \
            scripts/dwarf-perf/run_baseline.sh \
            scripts/dwarf-perf/corpus/README.md
          do
            if ! git cat-file -e "${BASE_SHA}:${required_path}" 2>/dev/null; then
              base_supported=false
            fi
          done

          changed_files=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")
          mkdir -p "$(dirname "$DWARF_PERF_CHANGED_FILES")"
          printf '%s\n' "$changed_files" > "$DWARF_PERF_CHANGED_FILES"

          benchmark_changed=false
          if printf '%s\n' "$changed_files" | grep -Eq \
            '^(scripts/dwarf-perf/corpus/|scripts/dwarf-perf/build_corpus\.sh$|scripts/dwarf-perf/build_corpus_in_container\.sh$|scripts/dwarf-perf/run_baseline\.sh$|scripts/dwarf-perf/generate_parse_stress\.py$|scripts/dwarf-perf/generate_rust_parse_stress\.py$|scripts/dwarf-perf/generate_cpp_template_stress\.py$|scripts/dwarf-perf/generate_rust_generic_stress\.py$|scripts/dwarf-perf/generate_cpp_deep_namespace\.py$|scripts/dwarf-perf/builder_image_ref\.txt$|scripts/dwarf-perf/compare_baseline\.py$|\.github/workflows/dwarf-perf-regression\.yml$)'; then
            benchmark_changed=true
          fi

          echo "base_supported=$base_supported" >> "$GITHUB_OUTPUT"
          echo "benchmark_changed=$benchmark_changed" >> "$GITHUB_OUTPUT"

      - name: Checkout base
        if: steps.detect.outputs.base_supported == 'true'
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.base.sha }}
          path: base
          fetch-depth: 1
          submodules: recursive

      - name: Install LLVM 18 and jq
        run: |
          wget https://apt.llvm.org/llvm.sh
          chmod +x llvm.sh
          sudo ./llvm.sh 18
          sudo apt-get install -y llvm-18-dev libpolly-18-dev jq

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@1.88.0

      - name: Cache cargo registry
        uses: actions/cache@v4
        with:
          path: ~/.cargo/registry
          key: ${{ runner.os }}-dwarf-perf-cargo-registry-${{ hashFiles('head/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-dwarf-perf-cargo-registry-

      - name: Cache cargo index
        uses: actions/cache@v4
        with:
          path: ~/.cargo/git
          key: ${{ runner.os }}-dwarf-perf-cargo-index-${{ hashFiles('head/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-dwarf-perf-cargo-index-

      - name: Cache cargo build
        uses: actions/cache@v4
        with:
          path: ${{ github.workspace }}/.target_tmp/dwarf-perf-regression
          key: ${{ runner.os }}-dwarf-perf-target-${{ hashFiles('head/Cargo.lock') }}
          restore-keys: |
            ${{ runner.os }}-dwarf-perf-target-

      - name: Prepare result directories
        run: mkdir -p "$DWARF_PERF_RESULTS_DIR"

      - name: Build base corpus
        if: steps.detect.outputs.base_supported == 'true'
        id: base_corpus
        working-directory: base
        run: |
          set -euo pipefail
          ./scripts/dwarf-perf/build_corpus.sh
          manifest="$GITHUB_WORKSPACE/base/scripts/dwarf-perf/corpus/out/manifest.json"
          parse_targets=(
            parse-stress
            rust-parse-stress
            cpp-template-stress
            rust-generic-stress
            cpp-deep-namespace
          )
          missing_targets=false
          for target in "${parse_targets[@]}"; do
            safe_target=${target//-/_}
            if jq -e --arg target "$target" '.artifacts[] | select(.name==$target)' "$manifest" >/dev/null; then
              echo "has_${safe_target}=true" >> "$GITHUB_OUTPUT"
            else
              echo "has_${safe_target}=false" >> "$GITHUB_OUTPUT"
              missing_targets=true
            fi
          done
          echo "missing_targets=$missing_targets" >> "$GITHUB_OUTPUT"

      - name: Run base baselines
        if: steps.detect.outputs.base_supported == 'true'
        working-directory: base
        run: |
          set -euo pipefail
          manifest="$GITHUB_WORKSPACE/base/scripts/dwarf-perf/corpus/out/manifest.json"
          parse_targets=(
            parse-stress
            rust-parse-stress
            cpp-template-stress
            rust-generic-stress
            cpp-deep-namespace
          )

          for target in "${parse_targets[@]}"; do
            if ! jq -e --arg target "$target" '.artifacts[] | select(.name==$target)' "$manifest" >/dev/null; then
              continue
            fi

            ./scripts/dwarf-perf/run_baseline.sh \
              --skip-build \
              --corpus-dir "$GITHUB_WORKSPACE/base/scripts/dwarf-perf/corpus/out" \
              --results-dir "$DWARF_PERF_RESULTS_DIR" \
              --result-name "base-${target}" \
              --runs 10 \
              --cargo-target-dir "$CARGO_TARGET_DIR" \
              --parse-target "$target"
          done

      - name: Run head baselines with base corpus
        if: steps.detect.outputs.base_supported == 'true'
        working-directory: head
        run: |
          set -euo pipefail
          manifest="$GITHUB_WORKSPACE/base/scripts/dwarf-perf/corpus/out/manifest.json"
          parse_targets=(
            parse-stress
            rust-parse-stress
            cpp-template-stress
            rust-generic-stress
            cpp-deep-namespace
          )

          for target in "${parse_targets[@]}"; do
            if ! jq -e --arg target "$target" '.artifacts[] | select(.name==$target)' "$manifest" >/dev/null; then
              continue
            fi

            ./scripts/dwarf-perf/run_baseline.sh \
              --skip-build \
              --corpus-dir "$GITHUB_WORKSPACE/base/scripts/dwarf-perf/corpus/out" \
              --results-dir "$DWARF_PERF_RESULTS_DIR" \
              --result-name "head-${target}" \
              --runs 10 \
              --cargo-target-dir "$CARGO_TARGET_DIR" \
              --parse-target "$target"
          done

      - name: Build head corpus for bootstrap targets
        if: steps.detect.outputs.base_supported != 'true' || steps.base_corpus.outputs.missing_targets == 'true'
        working-directory: head
        run: |
          set -euo pipefail
          ./scripts/dwarf-perf/build_corpus.sh

      - name: Run head-only baselines
        if: steps.detect.outputs.base_supported != 'true' || steps.base_corpus.outputs.missing_targets == 'true'
        working-directory: head
        env:
          BASE_SUPPORTED: ${{ steps.detect.outputs.base_supported }}
        run: |
          set -euo pipefail
          head_manifest="$GITHUB_WORKSPACE/head/scripts/dwarf-perf/corpus/out/manifest.json"
          base_manifest="$GITHUB_WORKSPACE/base/scripts/dwarf-perf/corpus/out/manifest.json"
          parse_targets=(
            parse-stress
            rust-parse-stress
            cpp-template-stress
            rust-generic-stress
            cpp-deep-namespace
          )

          for target in "${parse_targets[@]}"; do
            if ! jq -e --arg target "$target" '.artifacts[] | select(.name==$target)' "$head_manifest" >/dev/null; then
              continue
            fi

            if [[ "$BASE_SUPPORTED" == "true" ]] && jq -e --arg target "$target" '.artifacts[] | select(.name==$target)' "$base_manifest" >/dev/null; then
              continue
            fi

            ./scripts/dwarf-perf/run_baseline.sh \
              --skip-build \
              --corpus-dir "$GITHUB_WORKSPACE/head/scripts/dwarf-perf/corpus/out" \
              --results-dir "$DWARF_PERF_RESULTS_DIR" \
              --result-name "head-${target}" \
              --runs 10 \
              --cargo-target-dir "$CARGO_TARGET_DIR" \
              --parse-target "$target"
          done

      - name: Compare baselines
        if: steps.detect.outputs.base_supported == 'true'
        id: compare
        working-directory: head
        env:
          REPORT_ONLY: ${{ steps.detect.outputs.benchmark_changed }}
        run: |
          set +e

          compare_target() {
            local target="$1"
            local safe_target="$2"
            local base_result="$DWARF_PERF_RESULTS_DIR/base-${target}.json"
            local head_result="$DWARF_PERF_RESULTS_DIR/head-${target}.json"
            local summary_result="$DWARF_PERF_RESULTS_DIR/summary-${safe_target}.md"
            local compare_result="$DWARF_PERF_RESULTS_DIR/compare-${safe_target}.json"
            local target_exit=0
            local -a args

            args=(
              --base "$base_result"
              --head "$head_result"
              --base-label "base:${BASE_SHA::12}"
              --head-label "head:${HEAD_SHA::12}"
              --summary-file "$summary_result"
              --result-file "$compare_result"
            )

            if [[ "$REPORT_ONLY" == "true" ]]; then
              args+=(
                --report-only
                --report-only-reason "Benchmark inputs changed in this PR, so thresholds were not enforced."
              )
            fi

            echo "::group::DWARF perf compare: $target"
            python3 scripts/dwarf-perf/compare_baseline.py "${args[@]}"
            target_exit=$?
            echo "::endgroup::"

            {
              echo "### Parse Target: \`$target\`"
              echo
              sed '1,3d' "$summary_result"
              echo
            } >> "$DWARF_PERF_RESULTS_DIR/summary.md"

            return "$target_exit"
          }

          {
            printf '%s\n' '## DWARF Perf Regression'
            printf '\n'
          } > "$DWARF_PERF_RESULTS_DIR/summary.md"
          overall_exit=0

          parse_targets=(
            parse-stress
            rust-parse-stress
            cpp-template-stress
            rust-generic-stress
            cpp-deep-namespace
          )

          for target in "${parse_targets[@]}"; do
            safe_target=${target//-/_}
            if [[ -f "$DWARF_PERF_RESULTS_DIR/base-${target}.json" ]]; then
              compare_target "$target" "$safe_target"
              target_exit=$?
              if [[ "$target_exit" -ne 0 ]]; then
                overall_exit="$target_exit"
              fi
              continue
            fi

            {
              printf '### Parse Target: `%s`\n' "$target"
              printf '\n'
              printf '%s\n' '- Mode: `bootstrap`'
              printf '%s\n' "- Base: \`${BASE_SHA::12}\`"
              printf '%s\n' "- Head: \`${HEAD_SHA::12}\`"
              printf '%s\n' "- Note: The base revision does not provide the \`$target\` corpus yet, so threshold comparison was skipped."
              printf '\n'
              printf '%s\n' "A head-only baseline for \`$target\` was generated and uploaded as an artifact for inspection."
              printf '\n'
            } >> "$DWARF_PERF_RESULTS_DIR/summary.md"
          done

          echo "exit_code=$overall_exit" >> "$GITHUB_OUTPUT"
          exit 0

      - name: Write bootstrap summary
        if: steps.detect.outputs.base_supported != 'true'
        run: |
          {
            printf '%s\n' '## DWARF Perf Regression'
            printf '\n'
            printf '%s\n' '- Mode: `bootstrap`'
            printf '%s\n' "- Base: \`${BASE_SHA::12}\`"
            printf '%s\n' "- Head: \`${HEAD_SHA::12}\`"
            printf '%s\n' '- Note: The base revision does not contain the DWARF perf baseline tooling yet, so regression gating was skipped.'
            printf '\n'
            printf '%s\n' 'Head-only baselines for `parse-stress`, `rust-parse-stress`, `cpp-template-stress`, `rust-generic-stress`, and `cpp-deep-namespace` were generated and uploaded as artifacts for inspection.'
          } > "$DWARF_PERF_RESULTS_DIR/summary.md"

      - name: Publish summary
        if: always()
        run: |
          if [[ -f "$DWARF_PERF_RESULTS_DIR/summary.md" ]]; then
            cat "$DWARF_PERF_RESULTS_DIR/summary.md" >> "$GITHUB_STEP_SUMMARY"
          else
            {
              printf '%s\n' '## DWARF Perf Regression'
              printf '\n'
              printf '%s\n' 'Summary file was not produced. Inspect earlier workflow logs for the first failing step.'
            } >> "$GITHUB_STEP_SUMMARY"
          fi

      - name: Upload perf artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: dwarf-perf-regression-${{ github.event.pull_request.number }}
          path: |
            ${{ github.workspace }}/.tmp/dwarf-perf-results/*.json
            ${{ github.workspace }}/.tmp/dwarf-perf-results/*.md
            ${{ github.workspace }}/.tmp/dwarf-perf-changed-files.txt

      - name: Enforce regression gate
        if: ${{ steps.compare.outputs.exit_code && steps.compare.outputs.exit_code != '0' }}
        run: |
          echo "DWARF perf regression gate failed"
          exit 1