diff --git a/.aider.tags.cache.v4/cache.db b/.aider.tags.cache.v4/cache.db new file mode 100644 index 0000000000..df2580ceea Binary files /dev/null and b/.aider.tags.cache.v4/cache.db differ diff --git a/analyze.js b/analyze.js new file mode 100644 index 0000000000..5d50c37037 --- /dev/null +++ b/analyze.js @@ -0,0 +1,137 @@ +#!/usr/bin/env node + +/** + * analyze.js + * + * Reimplementation of analyze.sh in JavaScript. + * This script analyzes benchmark result files and generates summary statistics. + * + * Usage: + * node analyze.js ... + * + * The input files should be newline-delimited JSON benchmark results. + */ + +import fs from 'fs'; +import path from 'path'; + +function parseBenchmarkLine(line) { + try { + return JSON.parse(line); + } catch (e) { + return null; + } +} + +function formatValue(value, unit) { + if (unit === 'ns' && value >= 1000) { + return `${(value / 1000).toFixed(2)} μs`; + } + if (unit === 'μs' && value >= 1000) { + return `${(value / 1000).toFixed(2)} ms`; + } + if (unit === 'ms' && value >= 1000) { + return `${(value / 1000).toFixed(2)} s`; + } + return `${value.toFixed(2)} ${unit}`; +} + +function calculatePercentageChange(oldValue, newValue) { + if (oldValue === 0) return 0; + return ((newValue - oldValue) / oldValue) * 100; +} + +function generateComparisonTableMarkdown(oldBenchmarks, newBenchmarks) { + let table = ''; + table += '| Benchmark | Base | Change | Percentage Change |\n'; + table += '|-----------|------|--------|-------------------|\n'; + + for (let i = 0; i < oldBenchmarks.length; i++) { + const old = oldBenchmarks[i]; + const newB = newBenchmarks[i]; + const oldEstimate = formatValue(old.typical.estimate, old.typical.unit); + const newEstimate = formatValue(newB.typical.estimate, newB.typical.unit); + const percentageChange = calculatePercentageChange(old.typical.estimate, newB.typical.estimate); + + table += `| ${old.id} | ${oldEstimate} | ${newEstimate} | ${percentageChange.toFixed(2)}% |\n`; + } + return table; +} + +function performCheck(oldBenchmarks, newBenchmarks) { + const threshold = 10.0; + const exceeding = []; + + for (let i = 0; i < oldBenchmarks.length; i++) { + const old = oldBenchmarks[i]; + const newB = newBenchmarks[i]; + const change = calculatePercentageChange(old.typical.estimate, newB.typical.estimate); + if (change > threshold) { + exceeding.push({ id: old.id, change }); + } + } + + if (exceeding.length > 0) { + console.error('Benchmarks exceeding the 10% change threshold:'); + for (const bench of exceeding) { + console.error(` \x1b[31m${bench.id}: ${bench.change.toFixed(2)}%\x1b[0m`); + } + process.exit(1); + } +} + +async function main() { + const args = process.argv.slice(2); + if (args.length < 2 || args.length > 3) { + console.error('Usage: analyze.js [check|table]'); + process.exit(1); + } + + const oldFile = args[0]; + const newFile = args[1]; + const mode = args[2] || null; + + if (!fs.existsSync(oldFile)) { + console.error(`Old file not found: ${oldFile}`); + process.exit(1); + } + if (!fs.existsSync(newFile)) { + console.error(`New file not found: ${newFile}`); + process.exit(1); + } + + const oldContent = fs.readFileSync(oldFile, 'utf-8'); + const newContent = fs.readFileSync(newFile, 'utf-8'); + + const oldBenchmarks = oldContent + .split('\n') + .map(parseBenchmarkLine) + .filter(Boolean); + + const newBenchmarks = newContent + .split('\n') + .map(parseBenchmarkLine) + .filter(Boolean); + + if (oldBenchmarks.length !== newBenchmarks.length) { + console.error('Mismatch in the number of benchmarks between old and new files'); + process.exit(1); + } + + if (mode === 'check') { + performCheck(oldBenchmarks, newBenchmarks); + } else { + const table = generateComparisonTableMarkdown(oldBenchmarks, newBenchmarks); + const outputFile = path.join('benches', 'benchmark.md'); + fs.writeFileSync(outputFile, table); + console.log(table); + if (mode === null) { + performCheck(oldBenchmarks, newBenchmarks); + } + } +} + +main().catch((err) => { + console.error(err); + process.exit(1); +});