ci: chart impit's own npm/PyPI release history on PR comments

.github/workflows/version-benchmark-comment.yaml

@@ -0,0 +1,90 @@
+name: Comment the impit version chart
+
+on:
+  workflow_run:
+    workflows: [Compare impit versions]
+    types: [completed]
+
+permissions:
+  contents: read
+  actions: read # to download the artifact from the triggering workflow_run
+
+jobs:
+  comment:
+    name: Comment on the pull request
+    # A workflow_run job always runs the copy of this file from the default branch,
+    # never from the triggering run's head ref - unlike a second job gated by
+    # `if: github.event_name == 'pull_request'` directly in version-benchmark.yaml,
+    # which pull_request runs the workflow file from the pull request's own branch.
+    # Since that workflow's `paths` filter includes its own file, a pull request
+    # editing it would otherwise control the steps that see this job's push token.
+    # See https://securitylab.github.com/resources/github-actions-preventing-pwn-requests/
+    if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: master
+          token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+
+      - name: Download the chart
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: version-chart
+          path: downloaded
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Find the pull request
+        id: pr
+        env:
+          GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+          HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
+        run: |
+          number=$(gh api "repos/${GITHUB_REPOSITORY}/commits/${HEAD_SHA}/pulls" \
+            --jq '[.[] | select(.state == "open")][0].number')
+          echo "number=$number" >> "$GITHUB_OUTPUT"
+
+      - name: Publish the chart and comment on the pull request
+        if: steps.pr.outputs.number
+        env:
+          GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+          PR_NUMBER: ${{ steps.pr.outputs.number }}
+          RUN_ID: ${{ github.event.workflow_run.id }}
+        run: |
+          test -f downloaded/version-chart.png || { echo 'the artifact did not unpack where expected'; exit 1; }
+
+          versions=$(jq '.results | length' downloaded/results-node-versions.json)
+
+          # The chart is published to its own branch instead of the pull request,
+          # so a raw.githubusercontent.com URL can be embedded in a comment -
+          # GitHub strips data: URIs from comment markdown, and the pull request
+          # itself should not carry a generated PNG as one of its files.
+          git config user.name 'apify-service-account'
+          git config user.email 'apify-service-account@users.noreply.github.com'
+          git fetch origin benchmarks/version-chart-assets || true
+          if git show-ref --verify --quiet refs/remotes/origin/benchmarks/version-chart-assets; then
+            git checkout -B benchmarks/version-chart-assets origin/benchmarks/version-chart-assets
+          else
+            git checkout --orphan benchmarks/version-chart-assets
+            git rm -rf . > /dev/null
+          fi
+
+          filename="pr-${PR_NUMBER}.png"
+          mv downloaded/version-chart.png "$filename"
+          git add "$filename"
+          git commit -m "chore: publish version benchmark chart for #${PR_NUMBER}"
+          git push origin benchmarks/version-chart-assets
+
+          url="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/benchmarks/version-chart-assets/${filename}?run=${RUN_ID}"
+          {
+            echo '### impit version comparison'
+            echo
+            echo "Median throughput of the last $versions npm and PyPI releases of impit."
+            echo
+            echo "![impit version comparison]($url)"
+          } > comment.md
+
+          gh pr comment "$PR_NUMBER" --body-file comment.md --edit-last --create-if-none \
+            || gh pr comment "$PR_NUMBER" --body-file comment.md

.github/workflows/version-benchmark.yaml

@@ -0,0 +1,85 @@
+name: Compare impit versions
+
+on:
+  schedule:
+    # First of the month, offset from the client comparison so they don't compete for runners.
+    - cron: '0 5 1 * *'
+  workflow_dispatch:
+    inputs:
+      versions:
+        description: Versions per ecosystem to compare
+        default: '5'
+      requests:
+        description: Requests per run
+        default: '2000'
+      runs:
+        description: Runs per version, the median is reported
+        default: '11'
+  # Exercise the harness whenever it changes. version-benchmark-comment.yaml posts the
+  # resulting chart as a pull request comment - see that file for why it's a separate
+  # workflow rather than a second job here.
+  pull_request:
+    paths:
+      - benchmarks/**
+      - .github/workflows/version-benchmark.yaml
+
+permissions:
+  contents: read
+
+concurrency:
+  # Runs on different PRs shouldn't block each other, but a new push to the same PR
+  # makes its own previous run's comment stale, so that one is cancelled outright.
+  group: version-benchmark-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+env:
+  VERSIONS: ${{ inputs.versions || '5' }}
+  REQUESTS: ${{ inputs.requests || '2000' }}
+  RUNS: ${{ inputs.runs || '11' }}
+
+jobs:
+  benchmark:
+    name: Measure
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      # No credentials in this job: it installs and runs several unpinned releases
+      # of impit, so it must have nothing worth stealing.
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.sha || 'master' }}
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+        with:
+          node-version: 24
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+
+      - name: Set up a Python venv with matplotlib
+        run: |
+          uv venv --seed --python 3.12 benchmarks/python/.venv
+          uv pip install --python benchmarks/python/.venv/bin/python matplotlib
+
+      - name: Benchmark the npm releases
+        run: node benchmarks/node/bench-versions.mjs --versions "$VERSIONS" --requests "$REQUESTS" --runs "$RUNS"
+
+      - name: Benchmark the PyPI releases
+        run: benchmarks/python/.venv/bin/python benchmarks/python/bench_versions.py --versions "$VERSIONS" --requests "$REQUESTS" --runs "$RUNS"
+
+      - name: Render the chart
+        run: benchmarks/python/.venv/bin/python benchmarks/chart-versions.py
+
+      - name: Upload the chart and raw measurements
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: always()
+        with:
+          name: version-chart
+          path: |
+            benchmarks/version-chart.png
+            benchmarks/results-node-versions.json
+            benchmarks/results-python-versions.json
+            benchmarks/results-python-async-versions.json

.gitignore

@@ -17,5 +17,6 @@ _build/
 # Benchmarks
 /benchmarks/.cert
 /benchmarks/results-*.json
+/benchmarks/version-chart.png
 /benchmarks/node/node_modules
 /benchmarks/node/package-lock.json

benchmarks/README.md

@@ -53,3 +53,25 @@ resets and take a `GOAWAY` mid-run. Right for a public origin, wrong for a bench
 
 Add an entry to `CLIENTS`: how to build it, how to issue one request, and how to count its profiles.
 `update-readme.mjs` takes care of ordering and the caption.
+
+## Comparing impit's own releases
+
+[`node/bench-versions.mjs`](node/bench-versions.mjs) and
+[`python/bench_versions.py`](python/bench_versions.py) run the same measurement across the last N
+published releases of impit itself (from npm and PyPI respectively), to track throughput across
+versions rather than against other clients. The Python script measures both the sync `Client` and
+the async `AsyncClient`, since `AsyncClient` bridges each call through an asyncio event loop the way
+Node's Promise-returning `fetch()` does - separating the two shows how much of the npm/PyPI gap is
+that bridge rather than the underlying Rust client:
+
+```bash
+node node/bench-versions.mjs   # writes results-node-versions.json
+python/.venv/bin/python python/bench_versions.py   # writes results-python-versions.json (sync) and
+                                                    # results-python-async-versions.json (async)
+python/.venv/bin/python chart-versions.py          # writes version-chart.png
+```
+
+`--versions` picks how many releases to compare (default 5); `--requests`, `--runs` and `--warmup`
+work as above. `chart-versions.py` needs `matplotlib` (`uv pip install matplotlib`); its output is a
+CI artifact posted as a comment on the pull request that triggered it, not a file committed to the
+repository.

benchmarks/chart-versions.py

@@ -0,0 +1,118 @@
+"""Renders the version-history throughput chart from bench-versions.mjs / bench_versions.py.
+
+Reads results-node-versions.json, results-python-versions.json (the sync `Client`) and
+results-python-async-versions.json (the async `AsyncClient`), and plots median req/s
+against release date, one line per client. Node's `fetch()` and Python's `AsyncClient`
+both bridge each call through an event loop; the sync `Client` doesn't - splitting
+Python's two clients out shows how much of the npm/PyPI gap that bridge accounts for.
+The PNG is a CI artifact, not a committed file - see ../.github/workflows/version-benchmark.yaml.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from datetime import datetime
+from pathlib import Path
+
+import matplotlib
+
+matplotlib.use('Agg')
+
+import matplotlib.dates as mdates  # noqa: E402
+import matplotlib.pyplot as plt  # noqa: E402
+import matplotlib.ticker as mticker  # noqa: E402
+
+HERE = Path(__file__).resolve().parent
+
+INK = '#0b0b0b'
+SECONDARY_INK = '#52514e'
+MUTED = '#898781'
+GRIDLINE = '#e1e0d9'
+SURFACE = '#fcfcfb'
+SERIES = {
+    ('node', None): {'label': 'npm (Node.js)', 'color': '#2a78d6'},
+    ('python', 'sync'): {'label': 'PyPI (Python, sync)', 'color': '#eb6834'},
+    ('python', 'async'): {'label': 'PyPI (Python, async)', 'color': '#1baf7a'},
+}
+
+
+def load(path: Path) -> dict | None:
+    if not path.exists():
+        return None
+    report = json.loads(path.read_text())
+    if not report['results']:
+        return None
+    return report
+
+
+def plot(reports: list[dict], out: Path) -> None:
+    fig, ax = plt.subplots(figsize=(8, 4.5), dpi=200, facecolor=SURFACE)
+    ax.set_facecolor(SURFACE)
+
+    max_rate = max(point['rpsMedian'] for report in reports for point in report['results'])
+
+    for report in reports:
+        series = SERIES[(report['ecosystem'], report.get('variant'))]
+        points = sorted(report['results'], key=lambda r: r['publishedAt'])
+        dates = [datetime.fromisoformat(p['publishedAt'].replace('Z', '+00:00')) for p in points]
+        rates = [p['rpsMedian'] for p in points]
+
+        ax.plot(dates, rates, color=series['color'], linewidth=2, solid_capstyle='round',
+                marker='o', markersize=8, markerfacecolor=series['color'],
+                markeredgecolor=SURFACE, markeredgewidth=2, label=series['label'])
+
+        for point, date, rate in zip(points, dates, rates):
+            ax.annotate(point['version'], (date, rate), textcoords='offset points',
+                        xytext=(0, 10), ha='center', fontsize=8, color=MUTED)
+
+        last_date, last_rate = dates[-1], rates[-1]
+        ax.annotate(f'{last_rate:,.0f} req/s', (last_date, last_rate), textcoords='offset points',
+                    xytext=(10, -4), ha='left', fontsize=9, color=SECONDARY_INK, fontweight='bold')
+
+    ax.set_title('impit throughput by release', fontsize=13, color=INK, loc='left', pad=14)
+    ax.set_ylabel('req/s (median)', fontsize=10, color=SECONDARY_INK)
+    ax.yaxis.set_major_formatter(mticker.FuncFormatter(lambda value, _: f'{value:,.0f}'))
+    # Headroom above the highest point so its label never collides with the legend.
+    ax.set_ylim(0, max_rate * 1.3)
+
+    ax.xaxis.set_major_locator(mdates.AutoDateLocator(minticks=3, maxticks=6))
+    ax.xaxis.set_major_formatter(mdates.ConciseDateFormatter(ax.xaxis.get_major_locator()))
+    fig.autofmt_xdate(rotation=0, ha='center')
+
+    ax.grid(axis='y', color=GRIDLINE, linewidth=1)
+    ax.set_axisbelow(True)
+    for spine in ('top', 'right', 'left'):
+        ax.spines[spine].set_visible(False)
+    ax.spines['bottom'].set_color('#c3c2b7')
+    ax.tick_params(axis='both', colors=MUTED, labelsize=9, length=0)
+
+    legend = ax.legend(loc='upper left', frameon=False, fontsize=9, labelcolor=SECONDARY_INK)
+    legend.set_zorder(10)
+
+    fig.tight_layout()
+    fig.savefig(out, facecolor=SURFACE)
+    plt.close(fig)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--node', type=Path, default=HERE / 'results-node-versions.json')
+    parser.add_argument('--python', type=Path, default=HERE / 'results-python-versions.json')
+    parser.add_argument('--python-async', type=Path,
+                         default=HERE / 'results-python-async-versions.json')
+    parser.add_argument('--out', type=Path, default=HERE / 'version-chart.png')
+    args = parser.parse_args()
+
+    reports = [report for report in (load(args.node), load(args.python), load(args.python_async))
+               if report is not None]
+    if not reports:
+        raise SystemExit('neither results file has any results; nothing to chart')
+
+    plot(reports, args.out)
+    print(f'wrote {args.out}')
+    return 0
+
+
+if __name__ == '__main__':
+    raise SystemExit(main())

benchmarks/harness.mjs

@@ -1,7 +1,10 @@
 import { spawn } from 'node:child_process';
 import { mkdtemp, readdir, rm, stat } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { dirname, join } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+const here = dirname(fileURLToPath(import.meta.url));
 
 export function parseArgs(argv, defaults) {
   const out = { ...defaults };
@@ -64,15 +67,21 @@ function run(command, args, options = {}) {
   });
 }
 
+/** `npm install`s `pkg@version` into a fresh temp dir and returns its path. */
+export async function installPackage(pkg, version) {
+  const dir = await mkdtemp(join(tmpdir(), 'impit-bench-install-'));
+  await run('npm', [
+    'install', `${pkg}@${version}`,
+    '--prefix', dir,
+    '--no-save', '--no-audit', '--no-fund', '--loglevel', 'error',
+  ]);
+  return dir;
+}
+
 /** Bytes a fresh `npm install <pkg>` drops on disk, transitive dependencies included. */
 export async function installSize(pkg, version) {
-  const dir = await mkdtemp(join(tmpdir(), 'impit-bench-size-'));
+  const dir = await installPackage(pkg, version);
   try {
-    await run('npm', [
-      'install', `${pkg}@${version}`,
-      '--prefix', dir,
-      '--no-save', '--no-audit', '--no-fund', '--loglevel', 'error',
-    ]);
     return await treeSize(join(dir, 'node_modules'));
   } finally {
     await rm(dir, { recursive: true, force: true });
@@ -82,3 +91,20 @@ export async function installSize(pkg, version) {
 export function formatMB(bytes) {
   return `${(bytes / 1e6).toFixed(1)} MB`;
 }
+
+/** Spawns the shared HTTP/2 origin (../server.mjs) in its own process and resolves once it prints its URL. */
+export function spawnOrigin(bodyBytes) {
+  const child = spawn(process.execPath, [join(here, 'server.mjs')], {
+    env: { ...process.env, PORT: '0', BODY_BYTES: String(bodyBytes) },
+    stdio: ['ignore', 'pipe', 'inherit'],
+  });
+  return new Promise((resolve, reject) => {
+    let buffered = '';
+    child.stdout.on('data', (chunk) => {
+      buffered += chunk;
+      const newline = buffered.indexOf('\n');
+      if (newline !== -1) resolve({ child, url: buffered.slice(0, newline) });
+    });
+    child.on('exit', (code) => reject(new Error(`server exited with ${code} before listening`)));
+  });
+}