feat(benchmarks): generate the README comparison table in CI (#535)

Closes #469 — replaces the hand-measured table from #523 with a
benchmark harness that measures the latest published clients against a
local server in CI, monthly or on demand, and opens a PR when the
numbers move.

Author: barjin

.github/workflows/comparison-benchmark.yaml

@@ -0,0 +1,128 @@
+name: Refresh comparison benchmark
+
+on:
+  schedule:
+    # First of the month; the table tracks other projects' releases, not ours.
+    - cron: '0 4 1 * *'
+  workflow_dispatch:
+    inputs:
+      requests:
+        description: Requests per run
+        default: '2000'
+      runs:
+        description: Runs per client, the median is reported
+        default: '11'
+  # Exercise the harness whenever it changes, without proposing anything.
+  pull_request:
+    paths:
+      - benchmarks/**
+      - .github/workflows/comparison-benchmark.yaml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: comparison-benchmark
+  cancel-in-progress: false
+
+env:
+  BRANCH: benchmarks/comparison-refresh
+  REQUESTS: ${{ inputs.requests || '2000' }}
+  RUNS: ${{ inputs.runs || '11' }}
+
+jobs:
+  benchmark:
+    name: Measure
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    steps:
+      # No credentials in this job: it installs and executes unpinned releases of
+      # every client, so it must have nothing worth stealing. Reading a public
+      # repository needs no token.
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: ${{ github.event_name == 'pull_request' && github.sha || 'master' }}
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
+        with:
+          node-version: 24
+
+      - name: Setup uv
+        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
+
+      # No lockfiles anywhere here on purpose: the point is to compare whatever
+      # each project publishes today.
+      - name: Install the Node.js clients
+        run: npm install --prefix benchmarks/node --no-audit --no-fund --no-package-lock
+
+      - name: Install the Python clients
+        run: |
+          uv venv --seed --python 3.12 benchmarks/python/.venv
+          uv pip install --python benchmarks/python/.venv/bin/python -r benchmarks/python/requirements.txt
+
+      - name: Benchmark the Node.js clients
+        run: node benchmarks/node/bench.mjs --requests "$REQUESTS" --runs "$RUNS"
+
+      - name: Benchmark the Python clients
+        run: benchmarks/python/.venv/bin/python benchmarks/python/bench.py --requests "$REQUESTS" --runs "$RUNS"
+
+      - name: Rewrite the README table
+        run: node benchmarks/update-readme.mjs
+
+      - name: Upload the rewritten README and raw measurements
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
+        if: always()
+        with:
+          name: comparison-results
+          path: |
+            README.md
+            benchmarks/results-*.json
+
+  # Separate runner, so the token never shares an environment with third-party
+  # code. This job installs nothing.
+  propose:
+    name: Open a pull request
+    needs: benchmark
+    if: github.event_name != 'pull_request'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+        with:
+          ref: master
+          token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+
+      - name: Download the rewritten README
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8
+        with:
+          name: comparison-results
+
+      - name: Open a pull request
+        env:
+          GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }}
+        run: |
+          # Without this, an artifact that unpacked somewhere unexpected would
+          # look exactly like "the numbers did not move".
+          test -f benchmarks/results-node.json || { echo 'the artifact did not unpack where expected'; exit 1; }
+
+          if git diff --quiet -- README.md; then
+            echo "The measurements did not move the table; nothing to propose."
+            exit 0
+          fi
+
+          git config user.name 'apify-service-account'
+          git config user.email 'apify-service-account@users.noreply.github.com'
+          git checkout -B "$BRANCH"
+          git commit -m 'docs: refresh the client comparison benchmark' -- README.md
+          git push --force origin "$BRANCH"
+
+          if [ -z "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then
+            gh pr create --base master --head "$BRANCH" \
+              --title 'docs: refresh the client comparison benchmark' \
+              --body "Measured by \`.github/workflows/comparison-benchmark.yaml\`, $RUNS runs of $REQUESTS requests per client. Raw numbers are attached to [the run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})."
+          else
+            echo "The existing pull request now points at the new measurements."
+          fi

.gitignore

@@ -13,3 +13,9 @@ venv
 *.so
 *.py[cod]
 _build/
+
+# Benchmarks
+/benchmarks/.cert
+/benchmarks/results-*.json
+/benchmarks/node/node_modules
+/benchmarks/node/package-lock.json

README.md

@@ -31,34 +31,33 @@ async fn main() {
 }
 ```
 
+<!-- comparison:start -->
 ### Comparison
 
-Sequential requests from a single client against a local Node.js HTTP/2 server (1 KiB JSON body, keep-alive), best of 11 runs of 2000 requests, pinned to one core. Profile counts are the distinct versioned impersonation targets exposed by the public API. Numbers are indicative — rerun them on your own hardware before drawing conclusions.
+Median of 11 runs of 2000 sequential requests to a local HTTP/2 server, 1 KiB JSON responses over one warm connection. `Profiles` counts the impersonation targets each API exposes.
 
 **Python**
 
 | Package | req/s | Wheel | Profiles | Backend |
 | --- | --- | --- | --- | --- |
-| [`rnet`](https://github.com/0x676e67/rnet) | 3808 | 3.7 MB | 75 | Rust |
-| [`primp`](https://github.com/deedy5/primp) | 3547 | 5.3 MB | n/a[^1] | Rust |
-| **`impit`** | 2885 | 4.3 MB | 20 | Rust |
-| [`tls-client`](https://github.com/FlorianREGAZ/Python-Tls-Client) | 1831 | 41.3 MB | 51 | Go |
-| [`curl_cffi`](https://github.com/lexiforest/curl_cffi) | 1548 | 13.5 MB | 38 | C (libcurl) |
-| `httpx` (no impersonation) | 759 | 0.1 MB | — | Python |
+| [`primp`](https://github.com/deedy5/primp) | 6312 | 5.9 MB | n/a | Rust |
+| [`rnet`](https://github.com/0x676e67/rnet) | 5093 | 3.7 MB | 75 | Rust |
+| **`impit`** | 3808 | 4.2 MB | 20 | Rust |
+| [`curl_cffi`](https://github.com/lexiforest/curl_cffi) | 3428 | 13.5 MB | 38 | C (libcurl) |
+| [`tls-client`](https://github.com/FlorianREGAZ/Python-Tls-Client) | 3223 | 41.3 MB | 51 | Go |
+| `httpx` (no impersonation) | 2311 | 0.1 MB | — | Python |
 
 **Node.js**
 
 | Package | req/s | Install | Profiles | Backend |
 | --- | --- | --- | --- | --- |
-| **`impit`** | 1353 | 8.7 MB | 20 | Rust |
-| [`got-scraping`](https://github.com/apify/got-scraping) | 1149[^2] | 5.2 MB | 3[^3] | Node.js TLS |
-| [`node-tls-client`](https://github.com/Sahil1337/node-tls-client) | 901 | 31.1 MB | 63 | Go |
-| [`cycletls`](https://github.com/Danny-Dasilva/CycleTLS) | 287 | 133.3 MB | raw JA3 | Go subprocess |
-| `undici` (no impersonation) | 2030 | 2.0 MB | — | Node.js |
-
-[^1]: `primp` accepts arbitrary version strings and snaps to the nearest shipped profile, so the set is not enumerable through the public API.
-[^2]: Over HTTP/1.1 — with HTTP/2 enabled the server closes the session with `GOAWAY` after roughly a thousand requests.
-[^3]: Cipher suite and signature algorithm order only; no control over extension order, GREASE, or HTTP/2 `SETTINGS`.
+| [`got-scraping`](https://github.com/apify/got-scraping) | 2332 | 4.7 MB | 3 | Node.js TLS |
+| **`impit`** | 2260 | 8.7 MB | 20 | Rust |
+| [`cycletls`](https://github.com/Danny-Dasilva/CycleTLS) | 610 | 133.0 MB | raw JA3 | Go subprocess |
+| `undici` (no impersonation) | 4661 | 1.9 MB | — | Node.js |
+
+Measured by [`benchmarks/`](benchmarks) on linux-x64, 2026-09-03. Rerun it on your own hardware.
+<!-- comparison:end -->
 
 ### Other projects
 

benchmarks/README.md

@@ -0,0 +1,55 @@
+# Comparison benchmark
+
+Generates the comparison tables in the root [`README.md`](../README.md). Nothing is pinned and no
+lockfile is committed, so every run measures the latest published release of each client, `impit`
+included.
+
+## Running it
+
+```bash
+npm install --prefix node --no-audit --no-fund
+uv venv --seed --python 3.12 python/.venv
+uv pip install --python python/.venv/bin/python -r python/requirements.txt
+
+node node/bench.mjs                       # writes results-node.json
+python/.venv/bin/python python/bench.py   # writes results-python.json
+node update-readme.mjs                    # rewrites the table in ../README.md
+```
+
+`--requests`, `--runs` and `--warmup` shrink a run while iterating, `--body-bytes` changes the
+response size, and `--only impit,undici` limits it to a few clients. Pass the same values to both
+scripts — `update-readme.mjs` rejects reports taken with different parameters. `uv venv --seed`
+matters, because `bench.py` shells out to `pip download` to size each wheel.
+
+Run the two scripts one after the other, never in parallel: they compete for the same cores and both
+sets of numbers come out low.
+
+## What is measured
+
+[`server.mjs`](server.mjs) is the origin — a Node.js `http2` server on a self-signed certificate
+serving a 1 KiB JSON body. Each client then issues sequential requests over one connection, N
+times, and the median run is reported. Sequential single-client traffic isolates per-request client
+overhead, which is what differs between these libraries.
+
+The result files also carry the best and worst run, the negotiated protocol, and how many
+connections the client opened while being measured. That last one is worth checking when a number
+looks off: `cycletls` opens a fresh connection per request, so its figure includes a handshake every
+time.
+
+`Profiles` counts the impersonation targets each public API accepts, minus aliases that resolve to
+another target. There is no uniform way to ask for that, so each client has its own accessor in the
+`CLIENTS` table.
+
+Sizes are what each ecosystem distributes: the platform wheel for Python, and for Node.js whatever a
+fresh `npm install` leaves on disk with its transitive dependencies.
+
+## Notes on the origin
+
+Node's HTTP/2 Rapid-Reset mitigation is off in `server.mjs`. Clients that `RST_STREAM` each response
+once they have read it — got's `http2-wrapper` does — otherwise burn the default budget of 1000
+resets and take a `GOAWAY` mid-run. Right for a public origin, wrong for a benchmark.
+
+## Adding a client
+
+Add an entry to `CLIENTS`: how to build it, how to issue one request, and how to count its profiles.
+`update-readme.mjs` takes care of ordering and the caption.

benchmarks/harness.mjs

@@ -0,0 +1,84 @@
+import { spawn } from 'node:child_process';
+import { mkdtemp, readdir, rm, stat } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+export function parseArgs(argv, defaults) {
+  const out = { ...defaults };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (!arg.startsWith('--')) throw new Error(`unexpected argument ${arg}`);
+    const key = arg.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase());
+    if (!(key in out)) throw new Error(`unknown option ${arg}`);
+    const value = argv[i + 1];
+    if (value === undefined || value.startsWith('--')) throw new Error(`${arg} needs a value`);
+    out[key] = typeof out[key] === 'number' ? Number(value) : value;
+    i += 1;
+  }
+  return out;
+}
+
+/**
+ * `runs` batches of `requests` sequential requests over one warm connection. The
+ * median is what the table quotes; best and worst come along because some clients
+ * swing 3x between runs, which a best-of-N figure would flatter.
+ */
+export async function measure(request, { requests, runs, warmup }) {
+  for (let i = 0; i < warmup; i += 1) await request();
+
+  const rates = [];
+  for (let run = 0; run < runs; run += 1) {
+    const start = process.hrtime.bigint();
+    for (let i = 0; i < requests; i += 1) await request();
+    const elapsedNs = Number(process.hrtime.bigint() - start);
+    rates.push((requests * 1e9) / elapsedNs);
+  }
+  rates.sort((a, b) => a - b);
+  return {
+    rps: rates.at(-1),
+    rpsMedian: rates[Math.floor(rates.length / 2)],
+    rpsWorst: rates[0],
+  };
+}
+
+async function treeSize(dir) {
+  let total = 0;
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) total += await treeSize(path);
+    else if (entry.isFile()) total += (await stat(path)).size;
+  }
+  return total;
+}
+
+function run(command, args, options = {}) {
+  return new Promise((resolve, reject) => {
+    const child = spawn(command, args, { stdio: ['ignore', 'ignore', 'pipe'], ...options });
+    let stderr = '';
+    child.stderr.on('data', (chunk) => { stderr += chunk; });
+    child.on('error', reject);
+    child.on('close', (code) => {
+      if (code === 0) resolve();
+      else reject(new Error(`${command} exited with ${code}: ${stderr.trim().slice(0, 500)}`));
+    });
+  });
+}
+
+/** Bytes a fresh `npm install <pkg>` drops on disk, transitive dependencies included. */
+export async function installSize(pkg, version) {
+  const dir = await mkdtemp(join(tmpdir(), 'impit-bench-size-'));
+  try {
+    await run('npm', [
+      'install', `${pkg}@${version}`,
+      '--prefix', dir,
+      '--no-save', '--no-audit', '--no-fund', '--loglevel', 'error',
+    ]);
+    return await treeSize(join(dir, 'node_modules'));
+  } finally {
+    await rm(dir, { recursive: true, force: true });
+  }
+}
+
+export function formatMB(bytes) {
+  return `${(bytes / 1e6).toFixed(1)} MB`;
+}