Move benchmark top-level

Author: paulmillr

benchmark/decode-quality.ts

@@ -0,0 +1,95 @@
+import { readdirSync, statSync } from 'node:fs';
+import { dirname, join as pjoin } from 'node:path';
+import { performance } from 'node:perf_hooks';
+import { fileURLToPath } from 'node:url';
+import decodeQR from '../src/decode.ts';
+// Reuse the curated expectations from decode tests; should.runWhen keeps tests idle on import.
+import { DECODED, DECODE_VECTOR_EXCLUDE } from '../test/decode.test.ts';
+import { isDecodeImage, readImage } from '../test/utils.ts';
+
+const _dirname = dirname(fileURLToPath(import.meta.url));
+const DETECTION_PATH = pjoin(_dirname, '..', 'test', 'vectors', 'boofcv-v3', 'detection');
+
+const listFiles = (path, isDir = false) =>
+  readdirSync(path)
+    .filter((i) => statSync(`${path}/${i}`).isDirectory() === isDir)
+    .sort();
+
+const percent = (value, total) => (total === 0 ? 'n/a' : `${((100 * value) / total).toFixed(1)}%`);
+const millis = (value) => `${Math.round(value)}ms`;
+
+const select = (envName, values) => {
+  const raw = process.env[envName];
+  if (!raw) return values;
+  const selected = raw
+    .split(',')
+    .map((s) => s.trim())
+    .filter(Boolean);
+  for (const value of selected) {
+    if (!values.includes(value)) throw new Error(`unknown ${envName} value=${value}`);
+  }
+  return selected;
+};
+
+function vectorFiles(category) {
+  const dir = pjoin(DETECTION_PATH, category);
+  return listFiles(dir)
+    .filter(isDecodeImage)
+    .filter((file) => !DECODE_VECTOR_EXCLUDE.includes(`${category}/${file}`))
+    .map((file) => ({
+      category,
+      file,
+      path: pjoin('detection', category, file),
+      expected: DECODED[category]?.[file],
+    }));
+}
+
+function runCategory(files) {
+  const stats = {
+    files: 0,
+    expected: 0,
+    matched: 0,
+    wrong: 0,
+    missed: 0,
+    unknownDecoded: 0,
+    errors: 0,
+    ms: 0,
+  };
+  const started = performance.now();
+  for (const vector of files) {
+    stats.files++;
+    if (vector.expected) stats.expected++;
+    let decoded;
+    try {
+      decoded = decodeQR(readImage(vector.path), { moreEffort: true });
+    } catch {
+      stats.errors++;
+    }
+    if (vector.expected) {
+      if (vector.expected.includes(decoded)) stats.matched++;
+      else if (decoded === undefined) stats.missed++;
+      else stats.wrong++;
+    } else if (decoded !== undefined) {
+      stats.unknownDecoded++;
+    }
+  }
+  stats.ms = performance.now() - started;
+  return stats;
+}
+
+function printRow(category, stats) {
+  const msPerImage = stats.files === 0 ? 0 : stats.ms / stats.files;
+  const quality = stats.matched + stats.unknownDecoded;
+  console.log(`${category},${percent(quality, stats.files)},${millis(msPerImage)}`);
+}
+
+function main() {
+  const categories = select('QR_QUALITY_CATEGORIES', listFiles(DETECTION_PATH, true));
+  for (const category of categories) {
+    const files = vectorFiles(category);
+    if (files.length === 0) continue;
+    printRow(category, runCategory(files));
+  }
+}
+
+main();

benchmark/index.ts

@@ -0,0 +1,34 @@
+import { bench } from '@paulmillr/jsbt/bench.js';
+import { deepStrictEqual } from 'node:assert';
+import decodeQR from '../src/decode.ts';
+import { Bitmap, encodeQR } from '../src/index.ts';
+
+const section = (name) => console.log(`\n# ${name}`);
+
+function imageFromText(text, opts = {}) {
+  const { scale = 4, ...qrOpts } = opts;
+  const raw = encodeQR(text, 'raw', { border: 4, ecc: 'medium', ...qrOpts });
+  return new Bitmap(raw.length, raw).scale(scale).toImage();
+}
+
+const smallText = 'HELLO WORLD';
+const largeText = 'H'.repeat(768);
+const smallImage = imageFromText(smallText, { version: 1, scale: 4 });
+const largeImage = imageFromText(largeText, { scale: 2 });
+
+async function main() {
+  section('encode');
+  await bench('ascii small', () => encodeQR(smallText, 'ascii', { ecc: 'medium' }));
+  await bench('gif small', () => encodeQR(smallText, 'gif', { ecc: 'medium' }));
+  await bench('svg small', () => encodeQR(smallText, 'svg', { ecc: 'medium' }));
+
+  section('encode large');
+  await bench('raw large', () => encodeQR(largeText, 'raw', { ecc: 'medium' }));
+  await bench('ascii large', () => encodeQR(largeText, 'ascii', { ecc: 'medium' }));
+
+  section('decode generated');
+  await bench('v1 scale4', () => deepStrictEqual(decodeQR(smallImage), smallText));
+  await bench('large scale2', () => deepStrictEqual(decodeQR(largeImage), largeText));
+}
+
+main();

test/benchmark/index.ts benchmark/thirdparty/index.tstest/benchmark/index.ts renamed to benchmark/thirdparty/index.ts

@@ -12,8 +12,11 @@ import jsqr from 'jsqr';
 import { fileURLToPath } from 'node:url';
 import * as qrcodeGenerator from 'qrcode-generator';
 
+const _dirname = dirname(fileURLToPath(import.meta.url));
 const decodeExp = 'https://www.surveymonkey.com/s/TheClubatLAS_T3';
-const decodeJPG = jpeg.decode(readFileSync('../vectors/boofcv-v3/detection/blurred/image007.jpg'));
+const decodeJPG = jpeg.decode(
+  readFileSync(_dirname + '/../../test/vectors/boofcv-v3/detection/blurred/image007.jpg')
+);
 
 // Compared to other JS libraries:
 // - Don't work: [jsQR](https://github.com/cozmo/jsQR) is dead, [zxing-js](https://github.com/zxing-js/) is [dead](https://github.com/zxing-js/library/commit/b797504c25454db32aa2db410e6502b6db12a401), [qr-scanner](https://github.com/nimiq/qr-scanner/) uses jsQR, doesn't work outside of browser, [qcode-decoder](https://github.com/cirocosta/qcode-decoder) broken version of jsQR, doesn't work outside of browser
@@ -97,7 +100,7 @@ async function main() {
 
   section('Decoding quality');
   const _dirname = dirname(fileURLToPath(import.meta.url));
-  const DETECTION_PATH = pjoin(_dirname, '..', 'vectors', 'boofcv-v3', 'detection');
+  const DETECTION_PATH = pjoin(_dirname, '..', '..', 'test', 'vectors', 'boofcv-v3', 'detection');
 
   for (const category of listFiles(DETECTION_PATH, true)) {
     const DIR_PATH = `${DETECTION_PATH}/${category}`;

test/benchmark/package.json benchmark/thirdparty/package.jsontest/benchmark/package.json renamed to benchmark/thirdparty/package.json

@@ -6,7 +6,7 @@
   "main": "index.js",
   "type": "module",
   "scripts": {
-    "bench": "node index.ts"
+    "benchmark": "node index.ts"
   },
   "keywords": [],
   "author": "",

package.json

@@ -35,7 +35,9 @@
     "build": "tsc",
     "bundle": "npx --no @paulmillr/jsbt bundle test/build",
     "check": "npx --no @paulmillr/jsbt check",
-    "bench": "cd test/benchmark && npm ci && node index.ts",
+    "benchmark": "node benchmark/index.ts",
+    "benchmark:quality": "node benchmark/decode-quality.ts",
+    "benchmark:thirdparty": "cd benchmark/thirdparty && npm ci && node index.ts",
     "format": "prettier --write src",
     "test": "node --no-warnings test/index.ts",
     "test:bun": "bun test/index.ts",