Add benchmarking script

- Fold CI scripts into local benchmark script
- Remove CI workflow
- Use importlib for pytest for identical file names across different test modules

Author: kaitj

.github/scripts/compare_benchmarks.py

@@ -2,88 +2,38 @@ name: Benchmark
 
 on:
   pull_request:
-    branches: [ "main" ]
+    branches: ["main"]
 
-permissions:
-  pull-requests: write
 jobs:
-  prep:
-    runs-on: ubuntu-latest
-    outputs:
-      tag: ${{ steps.last_tag.outputs.tag }}
-      short_sha: ${{ steps.short.outputs.sha }}
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          fetch-tags: true
-          fetch-depth: 0
-      - id: last_tag
-        run: echo "tag=$(git describe --tags --abbrev=0)" >> $GITHUB_OUTPUT
-      - id: short
-        run: echo "sha=$(echo ${{ github.sha }} | cut -c1-7)" >> $GITHUB_OUTPUT
-
   benchmark:
-    needs: prep
     runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        target:
-          - name: ${{ needs.prep.outputs.short_sha }}
-            ref: ${{ github.sha }}
-          - name: main
-            ref: main
-          # Tag comparison disabled until next release (missing benchmark dependencies)
-          # - name: ${{ needs.prep.outputs.tag }}
-          #   ref: ${{ needs.prep.outputs.tag }}
     steps:
       - uses: actions/checkout@v6
         with:
-          ref: ${{ matrix.target.ref }}
           submodules: true
       - uses: astral-sh/setup-uv@v8.1.0
-      - run: uv sync --extra "cloud"
+      - run: uv sync --frozen --all-extras
       - name: Run benchmarks
+        id: run-benchmarks
         run: |
-          FILENAME="benchmark-${{ matrix.target.name }}-PR-${{ github.event.pull_request.number }}.json"
-          uv run .github/scripts/run_benchmarks.py --output "$FILENAME"
-          echo "REPORT_PATH=$FILENAME" >> $GITHUB_ENV
+          uv run python scripts/benchmark.py --branch ${{ github.sha }}
       - uses: actions/upload-artifact@v7
         with:
-          name: benchmark-${{ matrix.target.name }}-PR-${{
-            github.event.pull_request.number }}
-          path: ${{ env.REPORT_PATH }}
-          retention-days: 1
-          overwrite: true
-
-  report:
-    needs: [ prep, benchmark ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v6
-      - uses: astral-sh/setup-uv@v8.1.0
-      - uses: actions/download-artifact@v8
-        with:
-          pattern: benchmark-*
-          merge-multiple: true
-          path: benchmark-results
-      - name: Generate report
-        run: |
-          uv run .github/scripts/compare_benchmarks.py \
-            --output "benchmarks.md" \
-            --pattern "benchmark-results/benchmark-*-PR-${{ github.event.pull_request.number }}.json"
-      - name: Find Comment
-        uses: peter-evans/find-comment@v3
-        id: fc
-        with:
-          issue-number: ${{ github.event.pull_request.number }}
-          comment-author: "github-actions[bot]"
-          body-includes: "Benchmark Results"
+          name: benchmark-${{ matrix.target.name }}
+          path: ${{ steps.run-benchmarks.outputs.report_file }}
 
-      - name: Create / update comment
-        uses: peter-evans/create-or-update-comment@v5
-        with:
-          comment-id: ${{ steps.fc.outputs.comment-id }}
-          issue-number: ${{ github.event.pull_request.number }}
-          body-path: "benchmarks.md"
-          edit-mode: replace
+  # report:
+  #   needs: [benchmark]
+  #   runs-on: ubuntu-latest
+  #   steps:
+  #     - uses: actions/checkout@v6
+  #     - uses: astral-sh/setup-uv@v8.1.0
+  #     - run: uv sync --group "benchmark"
+  #     - uses: actions/download-artifact@v8
+  #       with:
+  #         pattern: benchmark-*
+  #     - name: Generate report
+  #       run: |
+  #         uv run .github/scripts/compare_benchmarks.py \
+  #           --output benchmarks.md \
+  #           --pattern benchmark-*.json

.github/scripts/run_benchmarks.py

@@ -26,6 +26,7 @@ htmlcov
 
 # Local data and scratch
 .scratch
+benchmarks/
 
 # Local virtual environment
 .venv

.github/workflows/benchmark.yaml

@@ -37,6 +37,7 @@ pybids = ["pandas>=2.0.0"]
 [dependency-groups]
 dev = [
     "pdoc>=16.0.0",
+    "polars>=1.40.1",
     "pre-commit>=4.6.0",
     "pytest>=9.0.3",
     "pytest-benchmark>=5.2.3",
@@ -69,6 +70,7 @@ lint.extend-select = ["I"]
 [tool.pytest.ini_options]
 log_cli = true
 log_cli_level = "INFO"
+addopts = "--import-mode=importlib"
 markers = [
     "benchmark: Tests used for benchmarking",
     "cloud: Tests requiring cloud group dependencies",