diff --git a/.github/workflows/comparison-benchmark.yaml b/.github/workflows/comparison-benchmark.yaml new file mode 100644 index 00000000..9a955fdf --- /dev/null +++ b/.github/workflows/comparison-benchmark.yaml @@ -0,0 +1,128 @@ +name: Refresh comparison benchmark + +on: + schedule: + # First of the month; the table tracks other projects' releases, not ours. + - cron: '0 4 1 * *' + workflow_dispatch: + inputs: + requests: + description: Requests per run + default: '2000' + runs: + description: Runs per client, the median is reported + default: '11' + # Exercise the harness whenever it changes, without proposing anything. + pull_request: + paths: + - benchmarks/** + - .github/workflows/comparison-benchmark.yaml + +permissions: + contents: read + +concurrency: + group: comparison-benchmark + cancel-in-progress: false + +env: + BRANCH: benchmarks/comparison-refresh + REQUESTS: ${{ inputs.requests || '2000' }} + RUNS: ${{ inputs.runs || '11' }} + +jobs: + benchmark: + name: Measure + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + # No credentials in this job: it installs and executes unpinned releases of + # every client, so it must have nothing worth stealing. Reading a public + # repository needs no token. + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: ${{ github.event_name == 'pull_request' && github.sha || 'master' }} + persist-credentials: false + + - name: Setup Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 + with: + node-version: 24 + + - name: Setup uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + + # No lockfiles anywhere here on purpose: the point is to compare whatever + # each project publishes today. + - name: Install the Node.js clients + run: npm install --prefix benchmarks/node --no-audit --no-fund --no-package-lock + + - name: Install the Python clients + run: | + uv venv --seed --python 3.12 benchmarks/python/.venv + uv pip install --python benchmarks/python/.venv/bin/python -r benchmarks/python/requirements.txt + + - name: Benchmark the Node.js clients + run: node benchmarks/node/bench.mjs --requests "$REQUESTS" --runs "$RUNS" + + - name: Benchmark the Python clients + run: benchmarks/python/.venv/bin/python benchmarks/python/bench.py --requests "$REQUESTS" --runs "$RUNS" + + - name: Rewrite the README table + run: node benchmarks/update-readme.mjs + + - name: Upload the rewritten README and raw measurements + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + if: always() + with: + name: comparison-results + path: | + README.md + benchmarks/results-*.json + + # Separate runner, so the token never shares an environment with third-party + # code. This job installs nothing. + propose: + name: Open a pull request + needs: benchmark + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + ref: master + token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + + - name: Download the rewritten README + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + name: comparison-results + + - name: Open a pull request + env: + GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} + run: | + # Without this, an artifact that unpacked somewhere unexpected would + # look exactly like "the numbers did not move". + test -f benchmarks/results-node.json || { echo 'the artifact did not unpack where expected'; exit 1; } + + if git diff --quiet -- README.md; then + echo "The measurements did not move the table; nothing to propose." + exit 0 + fi + + git config user.name 'apify-service-account' + git config user.email 'apify-service-account@users.noreply.github.com' + git checkout -B "$BRANCH" + git commit -m 'docs: refresh the client comparison benchmark' -- README.md + git push --force origin "$BRANCH" + + if [ -z "$(gh pr list --head "$BRANCH" --state open --json number --jq '.[].number')" ]; then + gh pr create --base master --head "$BRANCH" \ + --title 'docs: refresh the client comparison benchmark' \ + --body "Measured by \`.github/workflows/comparison-benchmark.yaml\`, $RUNS runs of $REQUESTS requests per client. Raw numbers are attached to [the run](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID})." + else + echo "The existing pull request now points at the new measurements." + fi diff --git a/.gitignore b/.gitignore index 43076265..003ba457 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,9 @@ venv *.so *.py[cod] _build/ + +# Benchmarks +/benchmarks/.cert +/benchmarks/results-*.json +/benchmarks/node/node_modules +/benchmarks/node/package-lock.json diff --git a/README.md b/README.md index c8e43ae2..28329f41 100644 --- a/README.md +++ b/README.md @@ -31,34 +31,33 @@ async fn main() { } ``` + ### Comparison -Sequential requests from a single client against a local Node.js HTTP/2 server (1 KiB JSON body, keep-alive), best of 11 runs of 2000 requests, pinned to one core. Profile counts are the distinct versioned impersonation targets exposed by the public API. Numbers are indicative — rerun them on your own hardware before drawing conclusions. +Median of 11 runs of 2000 sequential requests to a local HTTP/2 server, 1 KiB JSON responses over one warm connection. `Profiles` counts the impersonation targets each API exposes. **Python** | Package | req/s | Wheel | Profiles | Backend | | --- | --- | --- | --- | --- | -| [`rnet`](https://github.com/0x676e67/rnet) | 3808 | 3.7 MB | 75 | Rust | -| [`primp`](https://github.com/deedy5/primp) | 3547 | 5.3 MB | n/a[^1] | Rust | -| **`impit`** | 2885 | 4.3 MB | 20 | Rust | -| [`tls-client`](https://github.com/FlorianREGAZ/Python-Tls-Client) | 1831 | 41.3 MB | 51 | Go | -| [`curl_cffi`](https://github.com/lexiforest/curl_cffi) | 1548 | 13.5 MB | 38 | C (libcurl) | -| `httpx` (no impersonation) | 759 | 0.1 MB | — | Python | +| [`primp`](https://github.com/deedy5/primp) | 6312 | 5.9 MB | n/a | Rust | +| [`rnet`](https://github.com/0x676e67/rnet) | 5093 | 3.7 MB | 75 | Rust | +| **`impit`** | 3808 | 4.2 MB | 20 | Rust | +| [`curl_cffi`](https://github.com/lexiforest/curl_cffi) | 3428 | 13.5 MB | 38 | C (libcurl) | +| [`tls-client`](https://github.com/FlorianREGAZ/Python-Tls-Client) | 3223 | 41.3 MB | 51 | Go | +| `httpx` (no impersonation) | 2311 | 0.1 MB | — | Python | **Node.js** | Package | req/s | Install | Profiles | Backend | | --- | --- | --- | --- | --- | -| **`impit`** | 1353 | 8.7 MB | 20 | Rust | -| [`got-scraping`](https://github.com/apify/got-scraping) | 1149[^2] | 5.2 MB | 3[^3] | Node.js TLS | -| [`node-tls-client`](https://github.com/Sahil1337/node-tls-client) | 901 | 31.1 MB | 63 | Go | -| [`cycletls`](https://github.com/Danny-Dasilva/CycleTLS) | 287 | 133.3 MB | raw JA3 | Go subprocess | -| `undici` (no impersonation) | 2030 | 2.0 MB | — | Node.js | - -[^1]: `primp` accepts arbitrary version strings and snaps to the nearest shipped profile, so the set is not enumerable through the public API. -[^2]: Over HTTP/1.1 — with HTTP/2 enabled the server closes the session with `GOAWAY` after roughly a thousand requests. -[^3]: Cipher suite and signature algorithm order only; no control over extension order, GREASE, or HTTP/2 `SETTINGS`. +| [`got-scraping`](https://github.com/apify/got-scraping) | 2332 | 4.7 MB | 3 | Node.js TLS | +| **`impit`** | 2260 | 8.7 MB | 20 | Rust | +| [`cycletls`](https://github.com/Danny-Dasilva/CycleTLS) | 610 | 133.0 MB | raw JA3 | Go subprocess | +| `undici` (no impersonation) | 4661 | 1.9 MB | — | Node.js | + +Measured by [`benchmarks/`](benchmarks) on linux-x64, 2026-09-03. Rerun it on your own hardware. + ### Other projects diff --git a/benchmarks/README.md b/benchmarks/README.md new file mode 100644 index 00000000..04dd9d6d --- /dev/null +++ b/benchmarks/README.md @@ -0,0 +1,55 @@ +# Comparison benchmark + +Generates the comparison tables in the root [`README.md`](../README.md). Nothing is pinned and no +lockfile is committed, so every run measures the latest published release of each client, `impit` +included. + +## Running it + +```bash +npm install --prefix node --no-audit --no-fund +uv venv --seed --python 3.12 python/.venv +uv pip install --python python/.venv/bin/python -r python/requirements.txt + +node node/bench.mjs # writes results-node.json +python/.venv/bin/python python/bench.py # writes results-python.json +node update-readme.mjs # rewrites the table in ../README.md +``` + +`--requests`, `--runs` and `--warmup` shrink a run while iterating, `--body-bytes` changes the +response size, and `--only impit,undici` limits it to a few clients. Pass the same values to both +scripts — `update-readme.mjs` rejects reports taken with different parameters. `uv venv --seed` +matters, because `bench.py` shells out to `pip download` to size each wheel. + +Run the two scripts one after the other, never in parallel: they compete for the same cores and both +sets of numbers come out low. + +## What is measured + +[`server.mjs`](server.mjs) is the origin — a Node.js `http2` server on a self-signed certificate +serving a 1 KiB JSON body. Each client then issues sequential requests over one connection, N +times, and the median run is reported. Sequential single-client traffic isolates per-request client +overhead, which is what differs between these libraries. + +The result files also carry the best and worst run, the negotiated protocol, and how many +connections the client opened while being measured. That last one is worth checking when a number +looks off: `cycletls` opens a fresh connection per request, so its figure includes a handshake every +time. + +`Profiles` counts the impersonation targets each public API accepts, minus aliases that resolve to +another target. There is no uniform way to ask for that, so each client has its own accessor in the +`CLIENTS` table. + +Sizes are what each ecosystem distributes: the platform wheel for Python, and for Node.js whatever a +fresh `npm install` leaves on disk with its transitive dependencies. + +## Notes on the origin + +Node's HTTP/2 Rapid-Reset mitigation is off in `server.mjs`. Clients that `RST_STREAM` each response +once they have read it — got's `http2-wrapper` does — otherwise burn the default budget of 1000 +resets and take a `GOAWAY` mid-run. Right for a public origin, wrong for a benchmark. + +## Adding a client + +Add an entry to `CLIENTS`: how to build it, how to issue one request, and how to count its profiles. +`update-readme.mjs` takes care of ordering and the caption. diff --git a/benchmarks/harness.mjs b/benchmarks/harness.mjs new file mode 100644 index 00000000..8df089d0 --- /dev/null +++ b/benchmarks/harness.mjs @@ -0,0 +1,84 @@ +import { spawn } from 'node:child_process'; +import { mkdtemp, readdir, rm, stat } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +export function parseArgs(argv, defaults) { + const out = { ...defaults }; + for (let i = 0; i < argv.length; i += 1) { + const arg = argv[i]; + if (!arg.startsWith('--')) throw new Error(`unexpected argument ${arg}`); + const key = arg.slice(2).replace(/-([a-z])/g, (_, c) => c.toUpperCase()); + if (!(key in out)) throw new Error(`unknown option ${arg}`); + const value = argv[i + 1]; + if (value === undefined || value.startsWith('--')) throw new Error(`${arg} needs a value`); + out[key] = typeof out[key] === 'number' ? Number(value) : value; + i += 1; + } + return out; +} + +/** + * `runs` batches of `requests` sequential requests over one warm connection. The + * median is what the table quotes; best and worst come along because some clients + * swing 3x between runs, which a best-of-N figure would flatter. + */ +export async function measure(request, { requests, runs, warmup }) { + for (let i = 0; i < warmup; i += 1) await request(); + + const rates = []; + for (let run = 0; run < runs; run += 1) { + const start = process.hrtime.bigint(); + for (let i = 0; i < requests; i += 1) await request(); + const elapsedNs = Number(process.hrtime.bigint() - start); + rates.push((requests * 1e9) / elapsedNs); + } + rates.sort((a, b) => a - b); + return { + rps: rates.at(-1), + rpsMedian: rates[Math.floor(rates.length / 2)], + rpsWorst: rates[0], + }; +} + +async function treeSize(dir) { + let total = 0; + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) total += await treeSize(path); + else if (entry.isFile()) total += (await stat(path)).size; + } + return total; +} + +function run(command, args, options = {}) { + return new Promise((resolve, reject) => { + const child = spawn(command, args, { stdio: ['ignore', 'ignore', 'pipe'], ...options }); + let stderr = ''; + child.stderr.on('data', (chunk) => { stderr += chunk; }); + child.on('error', reject); + child.on('close', (code) => { + if (code === 0) resolve(); + else reject(new Error(`${command} exited with ${code}: ${stderr.trim().slice(0, 500)}`)); + }); + }); +} + +/** Bytes a fresh `npm install ` drops on disk, transitive dependencies included. */ +export async function installSize(pkg, version) { + const dir = await mkdtemp(join(tmpdir(), 'impit-bench-size-')); + try { + await run('npm', [ + 'install', `${pkg}@${version}`, + '--prefix', dir, + '--no-save', '--no-audit', '--no-fund', '--loglevel', 'error', + ]); + return await treeSize(join(dir, 'node_modules')); + } finally { + await rm(dir, { recursive: true, force: true }); + } +} + +export function formatMB(bytes) { + return `${(bytes / 1e6).toFixed(1)} MB`; +} diff --git a/benchmarks/node/bench.mjs b/benchmarks/node/bench.mjs new file mode 100644 index 00000000..28ce1a99 --- /dev/null +++ b/benchmarks/node/bench.mjs @@ -0,0 +1,234 @@ +import { spawn } from 'node:child_process'; +import { readFile, writeFile } from 'node:fs/promises'; +import { arch, platform } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { installSize, measure, parseArgs } from '../harness.mjs'; + +const here = dirname(fileURLToPath(import.meta.url)); + +const CHROME_JA3 = '771,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,' + + '0-23-65281-10-11-35-16-5-13-18-51-45-43-27-17513,29-23-24,0'; +const CHROME_UA = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) ' + + 'Chrome/131.0.0.0 Safari/537.36'; + +// Read off disk rather than through `require`: some of these packages have an +// `exports` map that hides ./package.json. +async function packageVersion(pkg) { + const manifest = await readFile(join(here, 'node_modules', pkg, 'package.json'), 'utf8'); + return JSON.parse(manifest).version; +} + +/** + * Impersonation targets the public API accepts, minus aliases that resolve to + * another target. `null` means there is no set to count and the client gives a + * `profilesLabel` instead. + */ +const profileCounts = { + // The browser list is a TS union, so the .d.ts is its only machine-readable + // form. `chrome`/`firefox`/`okhttp` alias the newest of their family. + async impit() { + const dts = await readFile(join(here, 'node_modules/impit/index.d.ts'), 'utf8'); + const union = /export type Browser =([^;]+);/.exec(dts); + if (!union) throw new Error('could not find the Browser union in impit/index.d.ts'); + const names = [...union[1].matchAll(/'([^']+)'/g)].map((m) => m[1]); + if (names.length === 0) throw new Error('impit Browser union parsed as empty'); + return names.filter((name) => /\d/.test(name)).length; + }, +}; + +const CLIENTS = [ + { + key: 'impit', + label: 'impit', + repo: null, + backend: 'Rust', + profiles: profileCounts.impit, + async setup(url) { + const { Impit } = await import('impit'); + const client = new Impit({ browser: 'chrome', ignoreTlsErrors: true }); + return { + request: async () => { + const response = await client.fetch(url); + return { body: await response.text(), alpn: response.headers.get('x-alpn') }; + }, + }; + }, + }, + { + key: 'got-scraping', + label: 'got-scraping', + repo: 'https://github.com/apify/got-scraping', + backend: 'Node.js TLS', + // chrome, firefox and safari, from the module-private `knownCiphers` in + // got-scraping's bundle. Nothing exposes it at runtime. + profiles: () => 3, + async setup(url) { + const { gotScraping } = await import('got-scraping'); + const client = gotScraping.extend({ + https: { rejectUnauthorized: false }, + retry: { limit: 0 }, + headerGeneratorOptions: { browsers: [{ name: 'chrome' }] }, + }); + return { + request: async () => { + const response = await client(url); + return { body: response.body, alpn: response.headers['x-alpn'] }; + }, + }; + }, + }, + { + key: 'cycletls', + label: 'cycletls', + repo: 'https://github.com/Danny-Dasilva/CycleTLS', + backend: 'Go subprocess', + // Configured with a raw JA3 string, so there is no fixed set to count. + profiles: () => null, + profilesLabel: 'raw JA3', + async setup(url) { + const initCycleTLS = (await import('cycletls')).default; + const client = await initCycleTLS(); + return { + request: async () => { + const response = await client.get(url, { + ja3: CHROME_JA3, + userAgent: CHROME_UA, + insecureSkipVerify: true, + }); + return { body: await response.text(), alpn: response.headers['X-Alpn']?.[0] }; + }, + teardown: () => client.exit(), + }; + }, + }, + { + key: 'undici', + label: 'undici', + repo: null, + backend: 'Node.js', + baseline: true, + profiles: () => null, + async setup(url) { + const { Agent, request } = await import('undici'); + const dispatcher = new Agent({ connect: { rejectUnauthorized: false }, allowH2: true }); + return { + request: async () => { + const response = await request(url, { dispatcher }); + return { body: await response.body.text(), alpn: response.headers['x-alpn'] }; + }, + teardown: () => dispatcher.close(), + }; + }, + }, +]; + +function startServer(bodyBytes) { + const child = spawn(process.execPath, [join(here, '..', 'server.mjs')], { + env: { ...process.env, PORT: '0', BODY_BYTES: String(bodyBytes) }, + stdio: ['ignore', 'pipe', 'inherit'], + }); + return new Promise((resolve, reject) => { + let buffered = ''; + child.stdout.on('data', (chunk) => { + buffered += chunk; + const newline = buffered.indexOf('\n'); + if (newline !== -1) resolve({ child, url: buffered.slice(0, newline) }); + }); + child.on('exit', (code) => reject(new Error(`server exited with ${code} before listening`))); + }); +} + +const options = parseArgs(process.argv.slice(2), { + requests: 2000, + runs: 11, + warmup: 200, + bodyBytes: 1024, + out: join(here, '..', 'results-node.json'), + only: '', +}); + +const selected = options.only + ? CLIENTS.filter((client) => options.only.split(',').includes(client.key)) + : CLIENTS; +if (selected.length === 0) throw new Error(`--only matched no client: ${options.only}`); + +const { child, url } = await startServer(options.bodyBytes); +const results = []; +const failures = []; + +// One long-lived dispatcher keeps the stats connection out of every client's count. +const { Agent, request } = await import('undici'); +const statsDispatcher = new Agent({ connect: { rejectUnauthorized: false } }); +const readStats = async () => { + const response = await request(new URL('/__stats', url), { dispatcher: statsDispatcher }); + return response.body.json(); +}; +await readStats(); + +try { + for (const client of selected) { + process.stderr.write(`${client.key}: `); + let teardown; + try { + const setup = await client.setup(url); + teardown = setup.teardown; + + const probe = await setup.request(); + if (probe.body.length !== options.bodyBytes) { + throw new Error(`expected a ${options.bodyBytes} byte body, got ${probe.body.length}`); + } + + const before = await readStats(); + const timings = await measure(setup.request, options); + const after = await readStats(); + + const version = await packageVersion(client.key); + results.push({ + key: client.key, + label: client.label, + repo: client.repo, + backend: client.backend, + baseline: client.baseline ?? false, + version, + alpn: probe.alpn ?? null, + profiles: await client.profiles(), + profilesLabel: client.profilesLabel ?? null, + sizeBytes: await installSize(client.key, version), + connections: after.connections - before.connections, + ...timings, + }); + const { rps, connections } = results.at(-1); + process.stderr.write(`${rps.toFixed(0)} req/s over ${probe.alpn}, ${connections} connection(s)\n`); + } catch (error) { + failures.push(`${client.key}: ${error.message}`); + process.stderr.write(`FAILED (${error.message})\n`); + } finally { + await teardown?.(); + } + } +} finally { + await statsDispatcher.close(); + child.kill(); +} + +await writeFile(options.out, `${JSON.stringify({ + ecosystem: 'node', + runtime: `Node.js ${process.version}`, + platform: `${platform()}-${arch()}`, + measuredAt: new Date().toISOString(), + options: { + requests: options.requests, + runs: options.runs, + warmup: options.warmup, + bodyBytes: options.bodyBytes, + }, + results, +}, null, 2)}\n`); + +process.stderr.write(`wrote ${options.out}\n`); +if (failures.length > 0) { + process.stderr.write(`${failures.length} client(s) failed:\n${failures.join('\n')}\n`); + process.exitCode = 1; +} diff --git a/benchmarks/node/package.json b/benchmarks/node/package.json new file mode 100644 index 00000000..d9f5e452 --- /dev/null +++ b/benchmarks/node/package.json @@ -0,0 +1,15 @@ +{ + "name": "impit-comparison-benchmark", + "private": true, + "type": "module", + "description": "Throughput comparison of Node.js browser-impersonation HTTP clients. Deliberately unpinned so every run resolves the latest published versions.", + "scripts": { + "bench": "node bench.mjs" + }, + "dependencies": { + "cycletls": "latest", + "got-scraping": "latest", + "impit": "latest", + "undici": "latest" + } +} diff --git a/benchmarks/python/bench.py b/benchmarks/python/bench.py new file mode 100644 index 00000000..f1cf9e99 --- /dev/null +++ b/benchmarks/python/bench.py @@ -0,0 +1,362 @@ +"""Throughput comparison of Python browser-impersonation HTTP clients. + +Runs every client against the shared local HTTP/2 origin in ../server.mjs and +writes ../results-python.json. See ../README.md for how the numbers are taken. +""" + +from __future__ import annotations + +import argparse +import json +import os +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import time +import typing +from collections.abc import Callable, Iterable +from dataclasses import dataclass +from datetime import datetime, timezone +from importlib.metadata import version as installed_version +from pathlib import Path +from typing import Any + +HERE = Path(__file__).resolve().parent +SERVER = HERE.parent / 'server.mjs' + +# Node's `os.arch()` vocabulary, so both reports name the same machine the same way. +ARCH_ALIASES = {'x86_64': 'x64', 'AMD64': 'x64', 'aarch64': 'arm64'} + +CHROME_PROFILES = { + 'impit': 'chrome', + 'primp': 'chrome_146', + 'tls_client': 'chrome_131', + 'curl_cffi': 'chrome', +} + + +@dataclass +class Client: + key: str + """Distribution name on PyPI.""" + label: str + backend: str + setup: Callable[[str], tuple[Callable[[], tuple[bytes, str | None]], Callable[[], None] | None]] + profiles: Callable[[], int | None] + repo: str | None = None + baseline: bool = False + profiles_label: str | None = None + """Shown in the Profiles cell when there is no set to count.""" + + +def _versioned(names: Iterable[str]) -> list[str]: + """Drop unversioned aliases such as impit's `chrome`, which means "newest Chrome".""" + return [name for name in names if re.search(r'\d', name)] + + +def impit_profiles() -> int: + import impit + + return len(_versioned(typing.get_args(impit.Browser))) + + +def rnet_profiles() -> int: + import rnet + + return len([name for name in dir(rnet.Impersonate) if not name.startswith('_')]) + + +def tls_client_profiles() -> int: + from tls_client.settings import ClientIdentifiers + + # Every identifier is a distinct target; the unversioned ones are app + # fingerprints (nike, zalando, ...), not aliases. + return len(typing.get_args(ClientIdentifiers)) + + +def curl_cffi_profiles() -> int: + from curl_cffi.requests import impersonate as ci + + names = set(typing.get_args(ci.BrowserTypeLiteral)) + # REAL_TARGET_MAP holds the "newest of this family" aliases. + names -= set(ci.REAL_TARGET_MAP) + # curl_cffi also kept the pre-rename spellings ("safari18_0") alongside the + # current ones ("safari180"); un-underscoring the version reaches the target. + deduped = {re.sub(r'(\d)_(\d)', r'\1\2', name) for name in names} + return len(names & deduped) + + +def setup_impit(url: str): + import impit + + client = impit.Client(browser=CHROME_PROFILES['impit'], verify=False) + + def request(): + response = client.get(url) + return response.content, response.headers.get('x-alpn') + + return request, None + + +def setup_rnet(url: str): + import rnet + + # rnet has no unversioned alias, so pick the newest Chrome it ships. + newest_chrome = max( + (name for name in dir(rnet.Impersonate) if name.startswith('Chrome')), + key=lambda name: int(name.removeprefix('Chrome')), + ) + client = rnet.BlockingClient(impersonate=getattr(rnet.Impersonate, newest_chrome), verify=False) + + def request(): + response = client.get(url) + alpn = response.headers.get('x-alpn') + return response.bytes(), alpn.decode() if isinstance(alpn, bytes) else alpn + + return request, None + + +def setup_primp(url: str): + import primp + + client = primp.Client(impersonate=CHROME_PROFILES['primp'], verify=False) + if client.impersonate != CHROME_PROFILES['primp']: + raise RuntimeError(f'primp fell back to {client.impersonate!r}; the profile name needs updating') + + def request(): + response = client.get(url) + return response.content, response.headers.get('x-alpn') + + return request, None + + +def setup_tls_client(url: str): + import tls_client + + session = tls_client.Session(client_identifier=CHROME_PROFILES['tls_client']) + + def request(): + response = session.get(url, insecure_skip_verify=True) + return response.content, response.headers.get('X-Alpn') + + return request, session.close + + +def setup_curl_cffi(url: str): + from curl_cffi import requests + + session = requests.Session(impersonate=CHROME_PROFILES['curl_cffi'], verify=False) + + def request(): + response = session.get(url) + return response.content, response.headers.get('x-alpn') + + return request, session.close + + +def setup_httpx(url: str): + import httpx + + client = httpx.Client(verify=False, http2=True) + + def request(): + response = client.get(url) + return response.content, response.headers.get('x-alpn') + + return request, client.close + + +CLIENTS = [ + Client( + key='rnet', + label='rnet', + repo='https://github.com/0x676e67/rnet', + backend='Rust', + setup=setup_rnet, + profiles=rnet_profiles, + ), + Client( + key='primp', + label='primp', + repo='https://github.com/deedy5/primp', + backend='Rust', + setup=setup_primp, + # primp does not expose its profile list, and an unknown name falls back to + # a random profile rather than erroring, so there is nothing to count. + profiles=lambda: None, + profiles_label='n/a', + ), + Client( + key='impit', + label='impit', + backend='Rust', + setup=setup_impit, + profiles=impit_profiles, + ), + Client( + key='tls-client', + label='tls-client', + repo='https://github.com/FlorianREGAZ/Python-Tls-Client', + backend='Go', + setup=setup_tls_client, + profiles=tls_client_profiles, + ), + Client( + key='curl_cffi', + label='curl_cffi', + repo='https://github.com/lexiforest/curl_cffi', + backend='C (libcurl)', + setup=setup_curl_cffi, + profiles=curl_cffi_profiles, + ), + Client( + key='httpx', + label='httpx', + backend='Python', + baseline=True, + setup=setup_httpx, + profiles=lambda: None, + ), +] + + +def start_server(body_bytes: int) -> tuple[subprocess.Popen, str]: + node = shutil.which('node') + if node is None: + raise RuntimeError('node is needed to run the benchmark origin server') + process = subprocess.Popen( + [node, str(SERVER)], + env={**os.environ, 'PORT': '0', 'BODY_BYTES': str(body_bytes)}, + stdout=subprocess.PIPE, + text=True, + ) + url = process.stdout.readline().strip() + if not url: + process.kill() + raise RuntimeError('the origin server exited before it printed its URL') + return process, url + + +def wheel_size(pkg: str, version: str) -> int: + """Size of the wheel pip installs for this interpreter and platform.""" + with tempfile.TemporaryDirectory() as target: + subprocess.run( + [sys.executable, '-m', 'pip', 'download', '--no-deps', '--only-binary', ':all:', + '--quiet', '--dest', target, f'{pkg}=={version}'], + check=True, + capture_output=True, + ) + wheels = list(Path(target).glob('*.whl')) + if len(wheels) != 1: + raise RuntimeError(f'expected one wheel for {pkg}, got {[w.name for w in wheels]}') + return wheels[0].stat().st_size + + +def measure(request, *, requests: int, runs: int, warmup: int) -> dict[str, float]: + """`runs` batches of `requests` sequential calls; see ../harness.mjs for the rationale.""" + for _ in range(warmup): + request() + + rates = [] + for _ in range(runs): + started = time.perf_counter() + for _ in range(requests): + request() + rates.append(requests / (time.perf_counter() - started)) + rates.sort() + return {'rps': rates[-1], 'rpsMedian': rates[len(rates) // 2], 'rpsWorst': rates[0]} + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--requests', type=int, default=2000) + parser.add_argument('--runs', type=int, default=11) + parser.add_argument('--warmup', type=int, default=200) + parser.add_argument('--body-bytes', type=int, default=1024) + parser.add_argument('--out', type=Path, default=HERE.parent / 'results-python.json') + parser.add_argument('--only', default='') + args = parser.parse_args() + + selected = [c for c in CLIENTS if not args.only or c.key in args.only.split(',')] + if not selected: + raise SystemExit(f'--only matched no client: {args.only}') + + import httpx + + process, url = start_server(args.body_bytes) + stats_client = httpx.Client(verify=False) + read_stats = lambda: stats_client.get(f'{url}__stats').json() # noqa: E731 + read_stats() + + results: list[dict[str, Any]] = [] + failures: list[str] = [] + try: + for client in selected: + print(f'{client.key}: ', end='', flush=True, file=sys.stderr) + teardown = None + try: + request, teardown = client.setup(url) + body, alpn = request() + if len(body) != args.body_bytes: + raise RuntimeError(f'expected a {args.body_bytes} byte body, got {len(body)}') + + before = read_stats() + timings = measure(request, requests=args.requests, runs=args.runs, warmup=args.warmup) + after = read_stats() + + version = installed_version(client.key) + results.append({ + 'key': client.key, + 'label': client.label, + 'repo': client.repo, + 'backend': client.backend, + 'baseline': client.baseline, + 'version': version, + 'alpn': alpn, + 'profiles': client.profiles(), + 'profilesLabel': client.profiles_label, + 'sizeBytes': wheel_size(client.key, version), + 'connections': after['connections'] - before['connections'], + **timings, + }) + print( + f'{results[-1]["rps"]:.0f} req/s over {alpn}, ' + f'{results[-1]["connections"]} connection(s)', + file=sys.stderr, + ) + except Exception as exc: # noqa: BLE001 + failures.append(f'{client.key}: {exc}') + print(f'FAILED ({exc})', file=sys.stderr) + finally: + if teardown is not None: + teardown() + finally: + stats_client.close() + process.kill() + + args.out.write_text(json.dumps({ + 'ecosystem': 'python', + 'runtime': f'CPython {platform.python_version()}', + 'platform': f'{sys.platform}-{ARCH_ALIASES.get(platform.machine(), platform.machine())}', + 'measuredAt': datetime.now(timezone.utc).isoformat(), + 'options': { + 'requests': args.requests, + 'runs': args.runs, + 'warmup': args.warmup, + 'bodyBytes': args.body_bytes, + }, + 'results': results, + }, indent=2) + '\n') + + print(f'wrote {args.out}', file=sys.stderr) + if failures: + print(f'{len(failures)} client(s) failed:', *failures, sep='\n', file=sys.stderr) + return 1 + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/benchmarks/python/requirements.txt b/benchmarks/python/requirements.txt new file mode 100644 index 00000000..2859871a --- /dev/null +++ b/benchmarks/python/requirements.txt @@ -0,0 +1,7 @@ +# Deliberately unpinned: every run resolves the latest published versions. +curl_cffi +httpx[http2] +impit +primp +rnet +tls-client diff --git a/benchmarks/server.mjs b/benchmarks/server.mjs new file mode 100644 index 00000000..db88345e --- /dev/null +++ b/benchmarks/server.mjs @@ -0,0 +1,93 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, readFileSync } from 'node:fs'; +import { createSecureServer } from 'node:http2'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const here = dirname(fileURLToPath(import.meta.url)); +const certDir = join(here, '.cert'); +const keyPath = join(certDir, 'key.pem'); +const certPath = join(certDir, 'cert.pem'); + +function ensureCert() { + if (existsSync(keyPath) && existsSync(certPath)) return; + mkdirSync(certDir, { recursive: true }); + execFileSync('openssl', [ + 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-sha256', + '-days', '365', + '-subj', '/CN=localhost', + '-addext', 'subjectAltName=DNS:localhost,IP:127.0.0.1', + '-keyout', keyPath, + '-out', certPath, + ], { stdio: 'ignore' }); +} + +// Precomputed so the server does no per-request work beyond the write. +function jsonBody(bytes) { + const envelope = Buffer.byteLength('{"filler":""}'); + return Buffer.from(`{"filler":"${'x'.repeat(bytes - envelope)}"}`); +} + +export const BODY_BYTES = 1024; + +export function startServer({ port = 0, bodyBytes = BODY_BYTES } = {}) { + ensureCert(); + const body = jsonBody(bodyBytes); + const server = createSecureServer({ + key: readFileSync(keyPath), + cert: readFileSync(certPath), + allowHTTP1: true, + ALPNProtocols: ['h2', 'http/1.1'], + // Clients that RST_STREAM every response (got's http2-wrapper does) burn + // Node's Rapid-Reset budget after ~1000 requests and take a GOAWAY mid-run. + streamResetBurst: Number.MAX_SAFE_INTEGER, + streamResetRate: Number.MAX_SAFE_INTEGER, + }); + + // Only connections that carried benchmark traffic count, so the benchmark's own + // polling of /__stats stays out of a client's total. + const stats = { connections: 0, requests: 0 }; + const counted = new WeakSet(); + + server.on('request', (req, res) => { + if (req.url === '/__stats') { + const json = Buffer.from(JSON.stringify(stats)); + res.writeHead(200, { 'content-type': 'application/json', 'content-length': json.length }); + res.end(json); + return; + } + + const connection = req.stream?.session ?? req.socket; + if (!counted.has(connection)) { + counted.add(connection); + stats.connections += 1; + } + + stats.requests += 1; + res.writeHead(200, { + 'content-type': 'application/json', + 'content-length': body.length, + 'x-alpn': req.httpVersion === '2.0' ? 'h2' : `http/${req.httpVersion}`, + }); + res.end(body); + }); + + // Clients drop their connection at the end of a run; that must not kill the server. + server.on('session', (session) => session.on('error', () => {})); + server.on('clientError', () => {}); + server.on('sessionError', () => {}); + + return new Promise((resolve) => { + server.listen(port, '127.0.0.1', () => { + resolve({ server, url: `https://localhost:${server.address().port}/`, certPath }); + }); + }); +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const { url } = await startServer({ + port: Number(process.env.PORT ?? 8443), + bodyBytes: Number(process.env.BODY_BYTES ?? BODY_BYTES), + }); + process.stdout.write(`${url}\n`); +} diff --git a/benchmarks/update-readme.mjs b/benchmarks/update-readme.mjs new file mode 100644 index 00000000..2317032c --- /dev/null +++ b/benchmarks/update-readme.mjs @@ -0,0 +1,88 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { formatMB, parseArgs } from './harness.mjs'; + +const here = dirname(fileURLToPath(import.meta.url)); + +const START = ''; +const END = ''; + +const options = parseArgs(process.argv.slice(2), { + readme: join(here, '..', 'README.md'), + node: join(here, 'results-node.json'), + python: join(here, 'results-python.json'), +}); + +async function load(path) { + const report = JSON.parse(await readFile(path, 'utf8')); + if (report.results.length === 0) throw new Error(`${path} contains no results`); + // The caption speaks for every row, so a client that fell back to HTTP/1.1 has + // to stop the run rather than end up mislabelled. + const odd = report.results.find((result) => result.alpn !== 'h2'); + if (odd) throw new Error(`${odd.key} negotiated ${odd.alpn}, not h2`); + return report; +} + +const [python, node] = await Promise.all([load(options.python), load(options.node)]); + +if (JSON.stringify(python.options) !== JSON.stringify(node.options)) { + throw new Error('the two reports were taken with different parameters; rerun both'); +} + +function table(report, sizeHeading) { + const ordered = [...report.results] + .sort((a, b) => (a.baseline - b.baseline) || (b.rpsMedian - a.rpsMedian)); + const rows = ordered.map((result) => { + const name = result.repo ? `[\`${result.label}\`](${result.repo})` : `\`${result.label}\``; + return [ + result.baseline ? `${name} (no impersonation)` : (result.repo ? name : `**${name}**`), + result.rpsMedian.toFixed(0), + formatMB(result.sizeBytes), + result.profiles ?? result.profilesLabel ?? '—', + result.backend, + ]; + }); + return [ + `| Package | req/s | ${sizeHeading} | Profiles | Backend |`, + '| --- | --- | --- | --- | --- |', + ...rows.map((cells) => `| ${cells.join(' | ')} |`), + ].join('\n'); +} + +const { requests, runs, bodyBytes } = python.options; + +const body = [ + '### Comparison', + '', + `Median of ${runs} runs of ${requests} sequential requests to a local HTTP/2 server, ` + + `${bodyBytes / 1024} KiB JSON responses over one warm connection. \`Profiles\` counts the ` + + 'impersonation targets each API exposes.', + '', + '**Python**', + '', + table(python, 'Wheel'), + '', + '**Node.js**', + '', + table(node, 'Install'), + '', + `Measured by [\`benchmarks/\`](benchmarks) on ${node.platform}, ${python.measuredAt.slice(0, 10)}.` + + ' Rerun it on your own hardware.', +].join('\n'); + +const readme = await readFile(options.readme, 'utf8'); +const start = readme.indexOf(START); +const end = readme.indexOf(END); +if (start === -1 || end === -1) { + throw new Error(`${options.readme} is missing the ${START} / ${END} markers`); +} + +const updated = `${readme.slice(0, start + START.length)}\n${body}\n${readme.slice(end)}`; +if (updated === readme) { + process.stderr.write('README is already up to date\n'); +} else { + await writeFile(options.readme, updated); + process.stderr.write(`updated ${options.readme}\n`); +}