Merge branch 'main' into pdubroy/cstview-unify

Author: pdubroy

packages/compiler/scripts/bench.js

@@ -5,7 +5,7 @@ import {run, bench, group, summary} from 'mitata';
 import * as ohm from 'ohm-js';
 
 import * as es5js from '../../../examples/ecmascript/index.js';
-import {toWasmGrammar} from '../test/_helpers.js';
+import {legacyGrammarToWasm} from '../test/_helpers.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const datadir = join(__dirname, '../test/data');
@@ -96,15 +96,18 @@ group('JSON', () => {
   // Note: we are deliberately creating one instance of the matcher that's
   // reused. This takes advantage of JIT tier-up, and approximates usage
   // in a long-running process, e.g. LSP server.
-  liquidHtmlWasm = await toWasmGrammar(
+  liquidHtmlWasm = await legacyGrammarToWasm(
     liquid.LiquidHTML,
     readFileSync(join(__dirname, '../build/liquid-html.wasm'))
   );
-  es5Wasm = await toWasmGrammar(
+  es5Wasm = await legacyGrammarToWasm(
     es5js.grammar,
     readFileSync(join(__dirname, '../build/es5.wasm'))
   );
-  jsonWasm = await toWasmGrammar(json, readFileSync(join(__dirname, '../build/json.wasm')));
+  jsonWasm = await legacyGrammarToWasm(
+    json,
+    readFileSync(join(__dirname, '../build/json.wasm'))
+  );
 
   await run();
 })();

packages/compiler/scripts/parseLiquid.js

@@ -17,13 +17,14 @@ import * as ohm from 'ohm-js-legacy';
 import {Bench} from 'tinybench';
 
 import {Compiler} from '../src/Compiler.ts';
-import {unparse, toWasmGrammar} from '../test/_helpers.js';
+import {unparse} from '../test/_helpers.js';
 import {CstView} from '../../runtime/src/cstView.ts';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const datadir = join(__dirname, '../test/data');
 
-const liquid = ohm.grammars(readFileSync(join(datadir, 'liquid-html.ohm'), 'utf8'));
+const grammarSource = readFileSync(join(datadir, 'liquid-html.ohm'), 'utf8');
+const liquid = ohm.grammars(grammarSource);
 
 // Parse flags and positional args.
 const flags = new Set(process.argv.slice(2).filter(a => a.startsWith('--')));
@@ -78,9 +79,10 @@ const pattern = positionalArgs[0];
 
   // Compile to Wasm.
   const compileStart = bench.now();
-  const modBytes = new Compiler(liquid.LiquidHTML).compile();
+  const allBytes = compileGrammars(grammarSource);
   const compileTime = bench.now() - compileStart;
-  const g = await toWasmGrammar(liquid.LiquidHTML, {modBytes});
+  const modBytes = allBytes.LiquidHTML;
+  const g = await Grammar.instantiate(modBytes);
   const {exports} = g._instance;
   let peakWasmHeapBytes = 0;
   let peakWasmMemoryBytes = 0;

packages/compiler/scripts/parsePipRequirements.js

@@ -11,12 +11,15 @@ import process from 'node:process';
 import {fileURLToPath} from 'node:url';
 import * as ohm from 'ohm-js-legacy';
 
-import {unparse, toWasmGrammar} from '../test/_helpers.js';
+import {Grammar} from 'ohm-js';
+import {compile} from '../src/api.ts';
+import {unparse} from '../test/_helpers.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const datadir = join(__dirname, '../test/data');
 
-const grammar = ohm.grammar(readFileSync(join(datadir, 'pep-508.ohm'), 'utf8'));
+const grammarSource = readFileSync(join(datadir, 'pep-508.ohm'), 'utf8');
+const grammar = ohm.grammar(grammarSource);
 const input = readFileSync(join(datadir, 'requirements_all.txt'), 'utf8');
 
 (async function main() {
@@ -32,7 +35,8 @@ const input = readFileSync(join(datadir, 'requirements_all.txt'), 'utf8');
   assert.equal(r.succeeded(), true, `JS parse failed: ${r.shortMessage}`);
 
   // --- Wasm ---
-  const g = await toWasmGrammar(grammar);
+  const modBytes = compile(grammarSource);
+  const g = await Grammar.instantiate(modBytes);
   const {exports} = g._instance;
 
   const wasmStart = performance.now();

packages/compiler/test/_helpers.js

@@ -2,12 +2,58 @@
 
 import {Compiler} from '../src/Compiler.ts';
 import {Grammar} from 'ohm-js';
+import {grammars as parseGrammars} from '../src/parseGrammars.ts';
 
 const DEBUG = process.env.OHM_DEBUG === '1';
 
-export async function toWasmGrammar(grammar, {modBytes, ...compilerOpts} = {}) {
+// Compile and load a Wasm grammar from source, using the Wasm metagrammar
+// path (no legacy ohm-js). For single-grammar sources.
+export async function compileAndLoad(source, compilerOpts = {}) {
+  const ns = parseGrammars(source);
+  const names = Object.keys(ns);
+  const g = ns[names[names.length - 1]];
+  return _compileAndInstantiate(g, compilerOpts);
+}
+
+// Compile and load all grammars from a multi-grammar source, returning
+// a {name: Grammar} map. Uses the Wasm metagrammar path.
+export async function compileAndLoadAll(source, compilerOpts = {}) {
+  const ns = parseGrammars(source);
+  const result = {};
+  for (const [name, g] of Object.entries(ns)) {
+    result[name] = await _compileAndInstantiate(g, compilerOpts);
+  }
+  return result;
+}
+
+async function _compileAndInstantiate(parsedGrammar, compilerOpts) {
+  const compiler = new Compiler(parsedGrammar, compilerOpts);
+  const bytes = compiler.compile();
+
+  const wasmGrammar = new Grammar();
+
+  let depth = 0;
+  let debugImports = {};
+  if (DEBUG) {
+    debugImports = compiler.getDebugImports((label, ret) => {
+      const result = ret === 0 ? 'FAIL' : 'SUCCESS';
+      const indented = s => new Array(depth).join('  ') + s;
+      const pos = wasmGrammar._instance.exports.pos.value;
+      if (label.startsWith('BEGIN')) depth += 1;
+      const tail = label.startsWith('END') ? ` -> ${result}` : '';
+      // eslint-disable-next-line no-console
+      console.log(`pos: ${pos} ${indented(label)}${tail}`);
+      if (label.startsWith('END')) depth -= 1;
+    });
+  }
+  return wasmGrammar._instantiate(bytes, debugImports);
+}
+
+// Takes a *legacy* ohm-js Grammar object and compiles it to Wasm.
+// Use this only in tests that deliberately compare Wasm vs legacy behavior.
+export async function legacyGrammarToWasm(grammar, compilerOpts = {}) {
   const compiler = new Compiler(grammar, compilerOpts);
-  const bytes = modBytes ?? compiler.compile();
+  const bytes = compiler.compile();
 
   const wasmGrammar = new Grammar();
 

packages/compiler/test/_test-v24.js

@@ -1,11 +1,9 @@
 // eslint-disable-next-line ava/no-ignored-test-files -- dynamically imported from test-wasm.js
 import test from 'ava';
-import * as ohm from 'ohm-js-legacy';
-
-import {toWasmGrammar} from './_helpers.js';
+import {compileAndLoad} from './_helpers.js';
 
 test('nested matching with `using`', async t => {
-  const g = await toWasmGrammar(ohm.grammar('G { Start = letter+ | digit+ }'));
+  const g = await compileAndLoad('G { Start = letter+ | digit+ }');
 
   {
     using outer = g.match('abc');

packages/compiler/test/test-cst-compat.js

@@ -10,7 +10,7 @@ import {readFileSync} from 'node:fs';
 import * as ohm from 'ohm-js-legacy';
 import {grammar as v18Grammar, grammars as v18Grammars} from 'ohm-js-legacy/v18';
 
-import {matchWithInput, scriptRel, toWasmGrammar} from './_helpers.js';
+import {matchWithInput, scriptRel, legacyGrammarToWasm} from './_helpers.js';
 
 // --- Serializer: reduces a CST node to a plain JSON "shape" ---
 
@@ -51,7 +51,7 @@ function serializeCst(node) {
 const arithmeticSrc = readFileSync(scriptRel('../../ohm-js/test/data/arithmetic.ohm'), 'utf8');
 
 test.failing('compat: arithmetic', async t => {
-  const wasmG = await toWasmGrammar(ohm.grammar(arithmeticSrc));
+  const wasmG = await legacyGrammarToWasm(ohm.grammar(arithmeticSrc));
   const v18G = v18Grammar(arithmeticSrc);
 
   for (const input of ['1', '10 + 20', '1+276*(3+4)', '(10+ 999)- 1 +222']) {
@@ -65,7 +65,7 @@ test.failing('compat: arithmetic', async t => {
 const liquidHtmlSrc = readFileSync(scriptRel('data/liquid-html.ohm'), 'utf8');
 
 test.failing('compat: liquid-html', async t => {
-  const wasmG = await toWasmGrammar(ohm.grammars(liquidHtmlSrc).LiquidHTML);
+  const wasmG = await legacyGrammarToWasm(ohm.grammars(liquidHtmlSrc).LiquidHTML);
   const v18G = v18Grammars(liquidHtmlSrc).LiquidHTML;
 
   const inputs = [

packages/compiler/test/test-debug.js

@@ -1,11 +1,10 @@
 import test from 'ava';
-import * as ohm from 'ohm-js-legacy';
 
 import {getMatchStats} from 'ohm-js/unstableDebug';
-import {toWasmGrammar} from './_helpers.js';
+import {compileAndLoad} from './_helpers.js';
 
 test('getMatchStats basic', async t => {
-  const g = await toWasmGrammar(ohm.grammar('G { start = "a" "b" }'));
+  const g = await compileAndLoad('G { start = "a" "b" }');
   g.match('ab').use(r => {
     t.true(r.succeeded());
     const stats = getMatchStats(r);
@@ -27,8 +26,8 @@ test('getMatchStats basic', async t => {
 });
 
 test('getMatchStats with alternatives', async t => {
-  const g = await toWasmGrammar(
-    ohm.grammar('G { start = big | small\n  big = "x" "y" "z"\n  small = "a" }')
+  const g = await compileAndLoad(
+    'G { start = big | small\n  big = "x" "y" "z"\n  small = "a" }'
   );
   g.match('a').use(r => {
     t.true(r.succeeded());

packages/compiler/test/test-es5.js

@@ -1,27 +1,37 @@
 import test from 'ava';
+import {readFileSync} from 'node:fs';
 import {readFile} from 'node:fs/promises';
 import {dirname, join} from 'node:path';
 import {performance} from 'node:perf_hooks';
 import {fileURLToPath} from 'node:url';
 
 import * as es5js from '../../../examples/ecmascript/index.js';
-import {matchWithInput, unparse, toWasmGrammar} from './_helpers.js';
+import {compileAndLoadAll, matchWithInput, unparse, legacyGrammarToWasm} from './_helpers.js';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const datadir = join(__dirname, 'data');
 
 const html5shivPath = join(datadir, '_html5shiv-3.7.3.js');
+const es5GrammarSource = readFileSync(
+  join(__dirname, '../../../examples/ecmascript/src/es5.ohm'),
+  'utf8'
+);
+
+async function loadES5() {
+  const grammars = await compileAndLoadAll(es5GrammarSource);
+  return grammars.ES5;
+}
 
 test('basic es5 examples', async t => {
-  const g = await toWasmGrammar(es5js.grammar);
+  const g = await loadES5();
   t.is(matchWithInput(g, 'x = 3;'), 1);
   t.is(matchWithInput(g, 'function foo() { return 1; }'), 1);
 });
 
 test('html5shiv', async t => {
   const source = await readFile(html5shivPath, 'utf8');
 
-  const g = await toWasmGrammar(es5js.grammar);
+  const g = await legacyGrammarToWasm(es5js.grammar);
   let start = performance.now();
   es5js.grammar.match(source);
   t.log(`html5shiv (Ohm) match time: ${(performance.now() - start).toFixed(2)}ms`);
@@ -50,12 +60,12 @@ test('unparsing', async t => {
     /\d+/.test("123") && console.log(counter());
   `;
 
-  const g = await toWasmGrammar(es5js.grammar);
+  const g = await loadES5();
   t.is(matchWithInput(g, source), 1);
   t.is(unparse(g).trimEnd(), source.trimEnd());
 });
 
 test('matching at end', async t => {
-  const g = await toWasmGrammar(es5js.grammar);
+  const g = await loadES5();
   t.false(g.match('', 'letter').use(r => r.succeeded()));
 });

packages/compiler/test/test-failurePos.js

@@ -4,7 +4,7 @@ import fc from 'fast-check';
 import {readFileSync} from 'node:fs';
 import {grammars, grammar} from 'ohm-js-legacy/v18';
 
-import {scriptRel, toWasmGrammar} from './_helpers.js';
+import {scriptRel, legacyGrammarToWasm} from './_helpers.js';
 
 const grammarSource = readFileSync(scriptRel('data/liquid-html.ohm'), 'utf8');
 const ns = grammars(grammarSource);
@@ -79,7 +79,7 @@ function sameFailurePos(wasmGrammar) {
 }
 
 test('failure pos (fast-check)', async t => {
-  const g = await toWasmGrammar(ns.LiquidHTML);
+  const g = await legacyGrammarToWasm(ns.LiquidHTML);
   const details = fc.check(sameFailurePos(g), {
     includeErrorInReport: true,
     interruptAfterTimeLimit: 1000,
@@ -94,7 +94,7 @@ test('failure pos: basic 1', async t => {
       Start = number+
       number = digit+
     }`);
-  const wasmGrammar = await toWasmGrammar(g);
+  const wasmGrammar = await legacyGrammarToWasm(g);
 
   t.is(failurePos(g, 'a'), 0);
   t.is(failurePos(wasmGrammar, 'a'), 0);
@@ -113,7 +113,7 @@ test('failure pos: basic 2', async t => {
             | number
         number = digit+
       }`);
-  const wasmGrammar = await toWasmGrammar(g);
+  const wasmGrammar = await legacyGrammarToWasm(g);
 
   t.is(failurePos(g, '99 + 66'), 7);
   t.is(failurePos(wasmGrammar, '99 + 66'), 7);
@@ -125,7 +125,7 @@ test('failure pos: basic 3', async t => {
       Start = letter letter
       space := "/*" (~"*/" any)* "*/"
     }`);
-  const wasmGrammar = await toWasmGrammar(g);
+  const wasmGrammar = await legacyGrammarToWasm(g);
 
   t.is(failurePos(wasmGrammar, '99'), 0);
 });
@@ -137,7 +137,7 @@ test('failure pos: lookahead', async t => {
         start = ~(anyTwo "!") "a" "b"
         anyTwo = any any
       }`);
-    const wasmGrammar = await toWasmGrammar(g);
+    const wasmGrammar = await legacyGrammarToWasm(g);
 
     // Original Ohm behaviour is to ignore failures inside the lookahead, so
     // it produces 'Expected "a"' at pos 0.
@@ -153,7 +153,7 @@ test('failure pos: memoization', async t => {
         start = ~anyTwo anyTwo
         anyTwo = any any
       }`);
-    const wasmGrammar = await toWasmGrammar(g);
+    const wasmGrammar = await legacyGrammarToWasm(g);
 
     // Original Ohm behaviour is to ignore failures inside the lookahead, so
     // it produces 'Expected "a"' at pos 0.
@@ -168,7 +168,7 @@ test('failure pos: space skipping', async t => {
       Start = digit digit
       space += "/*" (~"*/" any)* "*/" -- comment
     }`);
-  const wasmGrammar = await toWasmGrammar(g);
+  const wasmGrammar = await legacyGrammarToWasm(g);
 
   // Failure inside space skipping should be ignored.
   t.is(failurePos(g, '9 /* bad'), 2);
@@ -183,7 +183,7 @@ test('failure pos is always after space skipping', async t => {
             | "3." twice<"b">
       twice<x> = x x
     }`);
-  const wasmGrammar = await toWasmGrammar(g);
+  const wasmGrammar = await legacyGrammarToWasm(g);
 
   // Regular terminal
   t.is(failurePos(g, '1. c'), 3);
@@ -204,7 +204,7 @@ test('failure pos is always after space skipping', async t => {
 });
 
 test('fast-check zoo', async t => {
-  const wasmGrammar = await toWasmGrammar(ns.LiquidHTML);
+  const wasmGrammar = await legacyGrammarToWasm(ns.LiquidHTML);
 
   const input = '< {% if swatch_value %}';
   t.is(failurePos(wasmGrammar, input), failurePos(ns.LiquidHTML, input));