Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion packages/graph/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ The operations (`tour`, `entrypoints`, `lookup`, `trace`, `details`, `overview`,

## Benchmark

![Common prompt median token use on Codex GPT-5.4 Mini](https://ttsc.dev/benchmark/graph-common-codex-gpt-5.4-mini.svg)
![Common prompt median token use on Codex GPT-5.4 Mini](https://ttsc.dev/benchmark/svg/graph-common-codex-gpt-5.4-mini.svg)

Across eight real repositories, `@ttsc/graph` holds a flat, low median token cost while the alternatives swing with repository size.

Expand Down
130 changes: 130 additions & 0 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

5 changes: 4 additions & 1 deletion website/.gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,10 @@ node_modules
out
.pnpm-store
.DS_Store
public/benchmark/graph-*.svg
# Generated benchmark charts (build/graph-benchmark-svg.cjs; SVGs via
# `pnpm prepare`, PNGs via `pnpm build`)
public/benchmark/svg/
public/benchmark/png/
public/compiler/index.js
public/compiler/*.map
public/compiler/playground.wasm
Expand Down
36 changes: 30 additions & 6 deletions website/build/graph-benchmark-svg.cjs
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
const fs = require("fs");
const path = require("path");

const { renderPng } = require("./svg-to-png.cjs");

const ROOT = path.resolve(__dirname, "..");
const INPUT = path.join(ROOT, "public", "benchmark", "graph.json");
const OUT_DIR = path.join(ROOT, "public", "benchmark");
const SVG_DIR = path.join(OUT_DIR, "svg");
const PNG_DIR = path.join(OUT_DIR, "png");

const COLORS = {
background: "#05070b",
Expand Down Expand Up @@ -59,9 +63,12 @@ const cap = (s) => s.charAt(0).toUpperCase() + s.slice(1);
// Every chart is derived from graph.json so it never drifts from the published
// numbers. For each (harness, model, prompt family) present in the data we emit
// one grouped chart across every repo, plus one single-repo chart per repo.
// File names:
// grouped: graph-<family>-<harness>-<modelVersion>.svg
// single: graph-<repo>-<family>-<harness>-<modelVersion>.svg
// Each SVG (benchmark/svg/) also gets a 2x PNG sibling (benchmark/png/) for
// embeds that reject SVG (dev.to, most social cards). PNG export runs with
// --png (`pnpm build`); `pnpm prepare` emits SVGs only. File names:
// grouped: graph-<family>-<harness>-<modelVersion>.<ext>
// single: graph-<repo>-<family>-<harness>-<modelVersion>.<ext>
const EXPORT_PNG = process.argv.includes("--png");
const report = JSON.parse(fs.readFileSync(INPUT, "utf8"));
const allCells = report.agent?.cells ?? [];

Expand All @@ -78,8 +85,12 @@ for (const cell of allCells) {
}
}

fs.mkdirSync(OUT_DIR, { recursive: true });
fs.mkdirSync(SVG_DIR, { recursive: true });
fs.mkdirSync(PNG_DIR, { recursive: true });
let written = 0;
// --png only re-renders charts whose SVG content changed (or whose PNG is
// missing).
const pngQueue = [];
for (const combo of combos.values()) {
const cells = allCells.filter(
(cell) =>
Expand Down Expand Up @@ -114,15 +125,28 @@ for (const combo of combos.values()) {
);
}
}
const pngs = writePngs();
console.log(
`[build:graph-svg] wrote ${written} chart(s) to ${path.relative(ROOT, OUT_DIR)}`,
`[build:graph-svg] wrote ${written} chart(s)${EXPORT_PNG ? ` and ${pngs} png(s)` : ""} to ${path.relative(ROOT, OUT_DIR)}`,
);

function writeSvg(name, svg) {
fs.writeFileSync(path.join(OUT_DIR, name), `${svg}\n`);
const file = path.join(SVG_DIR, name);
const content = `${svg}\n`;
const changed =
!fs.existsSync(file) || fs.readFileSync(file, "utf8") !== content;
if (changed) fs.writeFileSync(file, content);
const pngFile = path.join(PNG_DIR, name.replace(/\.svg$/, ".png"));
if (changed || !fs.existsSync(pngFile)) pngQueue.push(file);
written += 1;
}

function writePngs() {
if (!EXPORT_PNG || pngQueue.length === 0) return 0;
for (const svgFile of pngQueue) renderPng(svgFile, { outDir: PNG_DIR });
return pngQueue.length;
}

function buildRows(input) {
const byRepo = groupBy(input, (cell) => cell.repo);
return [...byRepo.entries()]
Expand Down
114 changes: 34 additions & 80 deletions website/build/svg-to-png.cjs
Original file line number Diff line number Diff line change
@@ -1,99 +1,55 @@
const fs = require("fs");
const os = require("os");
const path = require("path");
const { execFileSync } = require("child_process");
const { Resvg } = require("@resvg/resvg-js");

const ROOT = path.resolve(__dirname, "..");
const OUT_DIR = path.join(__dirname, "png-out");
const SCALE = 2;
const DEFAULT_OUT_DIR = path.join(__dirname, "png-out");
const DEFAULT_SCALE = 2;

// The two charts embedded in the dev.to blog post. dev.to only accepts PNG
// uploads, so these occasionally need re-exporting from the published SVGs.
const DEFAULT_SVGS = [
"public/benchmark/graph-common-codex-gpt-5.5.svg",
"public/benchmark/graph-zod-common-codex-gpt-5.5.svg",
"public/benchmark/svg/graph-common-codex-gpt-5.5.svg",
"public/benchmark/svg/graph-zod-common-codex-gpt-5.5.svg",
].map((rel) => path.join(ROOT, rel));

// Locally installed headless browsers, in preference order. Chrome first, Edge
// as a fallback; both render SVG identically via the same --screenshot flag.
const BROWSERS = [
"C:\\Program Files\\Google\\Chrome\\Application\\chrome.exe",
"C:\\Program Files (x86)\\Microsoft\\Edge\\Application\\msedge.exe",
];

const inputs = process.argv.slice(2);
const svgPaths = (inputs.length > 0 ? inputs : DEFAULT_SVGS).map((p) =>
path.resolve(p),
);
const browser = findBrowser();

fs.mkdirSync(OUT_DIR, { recursive: true });
for (const svgPath of svgPaths) {
const out = renderPng(svgPath);
console.log(`[svg-to-png] wrote ${out.file} (${out.width}x${out.height})`);
if (require.main === module) {
const inputs = process.argv.slice(2);
const svgPaths = (inputs.length > 0 ? inputs : DEFAULT_SVGS).map((p) =>
path.resolve(p),
);
for (const svgPath of svgPaths) {
const out = renderPng(svgPath);
console.log(`[svg-to-png] wrote ${out.file} (${out.width}x${out.height})`);
}
}

// Wrap the SVG in a viewport-sized HTML page (white background, no margins) so
// Chrome screenshots exactly the intrinsic dimensions, then scale by SCALE via
// --force-device-scale-factor for a crisp 2x export.
function renderPng(svgPath) {
// resvg rasterizes in-process (no browser); `scale` zooms the intrinsic
// width/height for a crisp 2x export. Text uses whatever system font resolves
// from the chart's font-family stack (DejaVu Sans on Linux, Arial on Windows).
function renderPng(svgPath, options = {}) {
const outDir = options.outDir ?? DEFAULT_OUT_DIR;
const scale = options.scale ?? DEFAULT_SCALE;
if (!fs.existsSync(svgPath)) throw new Error(`SVG not found: ${svgPath}`);

const svg = fs.readFileSync(svgPath, "utf8");
const { width, height } = readSvgSize(svg);
const outFile = path.join(OUT_DIR, `${path.basename(svgPath, ".svg")}.png`);
fs.mkdirSync(outDir, { recursive: true });
const outFile = path.join(outDir, `${path.basename(svgPath, ".svg")}.png`);

const inner = svg.replace(/^\s*<\?xml[^>]*\?>\s*/i, "");
const html = `<!doctype html>
<html>
<head>
<meta charset="utf-8" />
<style>
html, body { margin: 0; padding: 0; background: #ffffff; }
svg { display: block; width: ${width}px; height: ${height}px; }
</style>
</head>
<body>${inner}</body>
</html>`;

const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "svg-to-png-"));
const htmlFile = path.join(tmpDir, "wrapper.html");
fs.writeFileSync(htmlFile, html);
try {
execFileSync(
browser,
[
"--headless=new",
"--disable-gpu",
"--hide-scrollbars",
"--default-background-color=FFFFFFFF",
`--force-device-scale-factor=${SCALE}`,
`--screenshot=${outFile}`,
`--window-size=${width},${height}`,
toFileUrl(htmlFile),
],
{ stdio: "ignore" },
);
} finally {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
const rendered = new Resvg(svg, {
fitTo: { mode: "zoom", value: scale },
background: "#ffffff",
}).render();
fs.writeFileSync(outFile, rendered.asPng());

const rendered = readPngSize(outFile);
const expected = { width: width * SCALE, height: height * SCALE };
if (rendered.width !== expected.width || rendered.height !== expected.height)
const size = readPngSize(outFile);
const expected = { width: width * scale, height: height * scale };
if (size.width !== expected.width || size.height !== expected.height)
throw new Error(
`${outFile}: expected ${expected.width}x${expected.height}, got ${rendered.width}x${rendered.height}`,
`${outFile}: expected ${expected.width}x${expected.height}, got ${size.width}x${size.height}`,
);
return { file: outFile, ...rendered };
}

function findBrowser() {
const found = BROWSERS.find((bin) => fs.existsSync(bin));
if (!found)
throw new Error(
`No headless browser found. Checked:\n ${BROWSERS.join("\n ")}`,
);
return found;
return { file: outFile, ...size };
}

// Intrinsic size from the root <svg> width/height attributes, falling back to
Expand Down Expand Up @@ -123,7 +79,7 @@ function numAttr(tag, name) {
}

// PNG dimensions live in the IHDR chunk: bytes 16-19 width, 20-23 height, both
// big-endian. Reading them proves the screenshot came out at the exact 2x size.
// big-endian. Reading them proves the export came out at the exact 2x size.
function readPngSize(file) {
const buf = fs.readFileSync(file);
const signature = "89504e470d0a1a0a";
Expand All @@ -132,6 +88,4 @@ function readPngSize(file) {
return { width: buf.readUInt32BE(16), height: buf.readUInt32BE(20) };
}

function toFileUrl(file) {
return `file:///${file.replace(/\\/g, "/")}`;
}
module.exports = { renderPng };
Loading
Loading