Merge remote-tracking branch 'origin/main'

Author: pdubroy

packages/compiler/Makefile

@@ -50,6 +50,39 @@ build/json.wasm: src/cli.ts ../lang-json/json.ohm
 build/liquid-html.wasm: src/cli.ts test/data/liquid-html.ohm
 	$(NODE) src/cli.ts -g LiquidHTML -o build/liquid-html.wasm test/data/liquid-html.ohm
 
+JAVY ?= javy
+
+# --- Javy: self-contained Wasm build of the compiler ---
+#
+# Javy (https://github.com/bytecodealliance/javy) compiles JavaScript into a
+# standalone Wasm module by embedding QuickJS. This lets us run the Ohm
+# compiler in any environment with a Wasm runtime, without requiring Node.js.
+#
+# How it works:
+#   1. tsdown bundles the compiler into a single IIFE (build/compiler-javy.iife.js),
+#      using ohm-js-legacy for grammar parsing (pure JS, no Wasm dependency)
+#      and stubbing out ohm-js v18 (which requires WebAssembly).
+#   2. Javy compiles the IIFE into a Wasm module (build/compiler.wasm).
+#
+# The resulting module reads an Ohm grammar from stdin and writes compiled
+# Wasm bytes to stdout. Errors are written to stderr.
+#
+# Prerequisites: `javy` CLI (https://github.com/bytecodealliance/javy/releases)
+# Testing requires `wasmtime` (https://wasmtime.dev/).
+.PHONY: compiler-wasm
+compiler-wasm: build/compiler.wasm
+
+build/compiler-javy.iife.js: src/javy-entry.ts src/javy-api.ts src/javy-ohm-js-stub.ts \
+    build/ohmRuntime.wasm_sections.ts build/ohm-grammar-wasm.ts $(SRC_TS_FILES) tsdown.javy.config.ts
+	pnpm tsdown --config tsdown.javy.config.ts
+
+build/compiler.wasm: build/compiler-javy.iife.js
+	$(JAVY) build $< -o $@
+
+.PHONY: test-compiler-wasm
+test-compiler-wasm: build/compiler.wasm
+	pnpm ava test/_test-javy.js
+
 .PHONY: go-test-es5
 go-test-es5: build/es5.wasm
 	cd test/go && go test -v

packages/compiler/scripts/parseLiquid.js

@@ -19,7 +19,7 @@ import {Bench} from 'tinybench';
 import {Grammar} from 'ohm-js';
 import {compileGrammars} from '../src/api.ts';
 import {unparse} from '../test/_helpers.js';
-import {createReader} from '../../runtime/src/cstReader.ts';
+import {createReader, CstNodeType} from '../../runtime/src/cstReader.ts';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 const datadir = join(__dirname, '../test/data');
@@ -36,7 +36,6 @@ const positionalArgs = process.argv.slice(2).filter(a => !a.startsWith('--'));
 const smallSize = flags.has('--small-size');
 const includeUnparse = flags.has('--include-unparse');
 const useCstReader = flags.has('--cst-reader');
-const useCstReaderPacked = flags.has('--cst-reader-packed');
 
 // Get pattern from command line arguments
 const pattern = positionalArgs[0];
@@ -105,34 +104,12 @@ const pattern = positionalArgs[0];
     opts
   );
 
-  // Walk CST using CstReader (raw handles), collecting terminal text.
-  function unparseCstReaderRaw(matchResult) {
+  // Walk CST using CstReader, collecting terminal text.
+  function unparseCstReader(matchResult) {
     const reader = createReader(matchResult);
-    const inp = reader.input;
-    let ans = '';
-    function walk(handle, startIdx) {
-      if (reader.isTerminal(handle)) {
-        ans += inp.slice(startIdx, startIdx + reader.matchLength(handle));
-        return;
-      }
-      reader.forEachChild(
-        handle,
-        (child, _leadingSpaces, offset) => {
-          walk(child, startIdx + offset);
-        },
-        startIdx
-      );
-    }
-    walk(reader.rootHandle, reader.rootStartIdx);
-    return ans;
-  }
-
-  // Walk CST using CstReader (handles with startIdx), collecting terminal text.
-  function unparseCstReaderPacked(matchResult) {
-    const reader = createReader(matchResult, {packStartIdx: true});
     let ans = '';
     function walk(handle) {
-      if (reader.isTerminal(handle)) {
+      if (reader.type(handle) === CstNodeType.TERMINAL) {
         ans += reader.sourceString(handle);
         return;
       }
@@ -146,11 +123,7 @@ const pattern = positionalArgs[0];
 
   const wasmLabel = includeUnparse ? 'Wasm parse+unparse' : 'Wasm parse';
   bench.add(
-    useCstReaderPacked
-      ? `${wasmLabel} (CstReader packed)`
-      : useCstReader
-        ? `${wasmLabel} (CstReader)`
-        : wasmLabel,
+    useCstReader ? `${wasmLabel} (CstReader)` : wasmLabel,
     () => {
       let overriddenDuration = 0;
       for (const {input} of files) {
@@ -167,11 +140,7 @@ const pattern = positionalArgs[0];
             peakWasmMemoryBytes,
             exports.memory.buffer.byteLength
           );
-          return useCstReaderPacked
-            ? unparseCstReaderPacked(m)
-            : useCstReader
-              ? unparseCstReaderRaw(m)
-              : unparse(g);
+          return useCstReader ? unparseCstReader(m) : unparse(g);
         });
         if (includeUnparse) overriddenDuration += bench.now() - start;
       }

packages/compiler/src/javy-api.ts

@@ -0,0 +1,20 @@
+// Javy-specific compile API. The normal compile path (parseGrammars.ts) uses
+// the Wasm-compiled Ohm meta-grammar to parse grammars, but QuickJS/Javy
+// doesn't have a WebAssembly runtime. So this module uses ohm-js-legacy's
+// pure-JS grammar parser instead, the same approach as scripts/bootstrap.ts.
+
+import {grammars} from 'ohm-js-legacy';
+
+import {Compiler} from './Compiler.ts';
+
+export function compile(source: string, grammarName?: string): Uint8Array {
+  const ns = grammars(source);
+  let g;
+  if (grammarName) {
+    g = ns[grammarName];
+    if (!g) throw new Error(`Grammar '${grammarName}' not found`);
+  } else {
+    g = Object.values(ns).at(-1);
+  }
+  return new Compiler(g, {debug: false}).compile();
+}

packages/compiler/src/javy-entry.ts

@@ -0,0 +1,52 @@
+// Entry point for the Javy-compiled Wasm module.
+// Protocol: stdin = UTF-8 grammar source, stdout = compiled Wasm bytes.
+
+import {compile} from './javy-api.ts';
+
+declare const Javy: {
+  IO: {
+    readSync(fd: number, buf: Uint8Array): number;
+    writeSync(fd: number, buf: Uint8Array): void;
+  };
+};
+
+function readStdin(): string {
+  const chunks: Uint8Array[] = [];
+  const buf = new Uint8Array(4096);
+  while (true) {
+    const n = Javy.IO.readSync(0, buf);
+    if (n === 0) break;
+    chunks.push(buf.slice(0, n));
+  }
+  let total = 0;
+  for (const c of chunks) total += c.length;
+  const result = new Uint8Array(total);
+  let offset = 0;
+  for (const c of chunks) {
+    result.set(c, offset);
+    offset += c.length;
+  }
+  return new TextDecoder().decode(result);
+}
+
+function stderr(msg: string): void {
+  Javy.IO.writeSync(2, new TextEncoder().encode(msg));
+}
+
+// Parse an optional "#grammarName <name>" directive from the first line of input.
+// If present, returns [grammarName, remainingSource]; otherwise [undefined, source].
+function parseHeader(input: string): [string | undefined, string] {
+  const match = input.match(/^#grammarName (\S+)\n([\s\S]*)$/);
+  if (match) return [match[1], match[2]];
+  return [undefined, input];
+}
+
+try {
+  const raw = readStdin();
+  const [grammarName, source] = parseHeader(raw);
+  const wasmBytes = compile(source, grammarName);
+  Javy.IO.writeSync(1, wasmBytes);
+} catch (e: any) {
+  const msg = e?.message ?? String(e);
+  stderr(msg + '\n');
+}

packages/compiler/src/javy-ohm-js-stub.ts

@@ -0,0 +1,10 @@
+// Stub for ohm-js (v18) — used in the Javy build where WebAssembly is not
+// available. Compiler.ts transitively imports parseGrammars.ts which imports
+// Grammar from ohm-js, but this code path is never reached when grammars are
+// parsed via ohm-js-legacy.
+
+export class Grammar {
+  constructor() {
+    throw new Error('ohm-js v18 Grammar is not available in this environment');
+  }
+}

packages/compiler/src/leb128.ts

@@ -0,0 +1,32 @@
+// Pure JS LEB128 decoding, replacing @thi.ng/leb128 (which uses Wasm internally).
+// API matches @thi.ng/leb128: (data: Uint8Array, offset?: number) => [bigint, number]
+
+export function decodeULEB128(data: Uint8Array, offset: number = 0): [bigint, number] {
+  let result = 0n;
+  let shift = 0n;
+  let pos = offset;
+  while (true) {
+    const byte = data[pos++];
+    result |= BigInt(byte & 0x7f) << shift;
+    if ((byte & 0x80) === 0) break;
+    shift += 7n;
+  }
+  return [result, pos - offset];
+}
+
+export function decodeSLEB128(data: Uint8Array, offset: number = 0): [bigint, number] {
+  let result = 0n;
+  let shift = 0n;
+  let pos = offset;
+  let byte: number;
+  do {
+    byte = data[pos++];
+    result |= BigInt(byte & 0x7f) << shift;
+    shift += 7n;
+  } while ((byte & 0x80) !== 0);
+  // Sign extend if the highest bit of the last byte is set.
+  if (shift < 64n && (byte & 0x40) !== 0) {
+    result |= -(1n << shift);
+  }
+  return [result, pos - offset];
+}

packages/compiler/src/rewriteFuncIdx.ts

@@ -2,7 +2,7 @@
 // for additional imports in the dest module. Used by Compiler.ts at compile
 // time to adjust prebuilt code for extra imports (e.g. debug imports).
 
-import {decodeULEB128, decodeSLEB128} from '@thi.ng/leb128';
+import {decodeULEB128, decodeSLEB128} from './leb128.ts';
 import * as w from '@wasmgroundup/emit';
 
 // For sanity checking, assume that the number of locals is never

packages/compiler/test/_test-javy.js

@@ -0,0 +1,68 @@
+// Tests for the Javy-compiled Wasm module (build/compiler.wasm).
+// Run via `make test-compiler-wasm`.
+
+/* eslint-disable ava/no-ignored-test-files */
+
+import test from 'ava';
+import {execFileSync} from 'node:child_process';
+import {URL} from 'node:url';
+import {Grammar} from 'ohm-js';
+
+import {compile} from '../src/javy-api.ts';
+
+const COMPILER_WASM = new URL('../build/compiler.wasm', import.meta.url).pathname;
+
+function javyCompile(grammarSource, grammarName) {
+  const input = grammarName ? `#grammarName ${grammarName}\n${grammarSource}` : grammarSource;
+  return execFileSync('wasmtime', [COMPILER_WASM], {
+    input,
+    maxBuffer: 1024 * 1024,
+  });
+}
+
+test('javy: simple grammar', t => {
+  const bytes = javyCompile('G { start = "hello" "world" }');
+  const g = new Grammar(bytes);
+  g.match('helloworld').use(r => t.false(r.failed()));
+  g.match('goodbye').use(r => t.true(r.failed()));
+});
+
+test('javy: arithmetic grammar', t => {
+  const source = `Arithmetic {
+  Exp = AddExp
+  AddExp = AddExp "+" MulExp  -- plus
+         | AddExp "-" MulExp  -- minus
+         | MulExp
+  MulExp = MulExp "*" PriExp  -- times
+         | MulExp "/" PriExp  -- divide
+         | PriExp
+  PriExp = "(" Exp ")"  -- paren
+         | number
+  number = digit+
+}`;
+  const bytes = javyCompile(source);
+  const g = new Grammar(bytes);
+  g.match('1+2*3').use(r => t.false(r.failed()));
+  g.match('(1+2)*3').use(r => t.false(r.failed()));
+  g.match('hello').use(r => t.true(r.failed()));
+});
+
+test('javy: output matches Node compiler', t => {
+  const source = 'G { start = "a" | "b" | "c" }';
+  const javyBytes = javyCompile(source);
+  const nodeBytes = compile(source);
+  t.deepEqual(new Uint8Array(javyBytes), nodeBytes);
+});
+
+test('javy: #grammarName selects grammar', t => {
+  const source = 'A { start = "a" }\nB { start = "b" }';
+  const bytesA = javyCompile(source, 'A');
+  const gA = new Grammar(bytesA);
+  gA.match('a').use(r => t.false(r.failed()));
+  gA.match('b').use(r => t.true(r.failed()));
+
+  const bytesB = javyCompile(source, 'B');
+  const gB = new Grammar(bytesB);
+  gB.match('b').use(r => t.false(r.failed()));
+  gB.match('a').use(r => t.true(r.failed()));
+});