compiler: add support for standalone wasm build using javy (#604)

Author: pdubroy

packages/compiler/Makefile

@@ -50,6 +50,39 @@ build/json.wasm: src/cli.ts ../lang-json/json.ohm
 build/liquid-html.wasm: src/cli.ts test/data/liquid-html.ohm
 	$(NODE) src/cli.ts -g LiquidHTML -o build/liquid-html.wasm test/data/liquid-html.ohm
 
+JAVY ?= javy
+
+# --- Javy: self-contained Wasm build of the compiler ---
+#
+# Javy (https://github.com/bytecodealliance/javy) compiles JavaScript into a
+# standalone Wasm module by embedding QuickJS. This lets us run the Ohm
+# compiler in any environment with a Wasm runtime, without requiring Node.js.
+#
+# How it works:
+#   1. tsdown bundles the compiler into a single IIFE (build/compiler-javy.iife.js),
+#      using ohm-js-legacy for grammar parsing (pure JS, no Wasm dependency)
+#      and stubbing out ohm-js v18 (which requires WebAssembly).
+#   2. Javy compiles the IIFE into a Wasm module (build/compiler.wasm).
+#
+# The resulting module reads an Ohm grammar from stdin and writes compiled
+# Wasm bytes to stdout. Errors are written to stderr.
+#
+# Prerequisites: `javy` CLI (https://github.com/bytecodealliance/javy/releases)
+# Testing requires `wasmtime` (https://wasmtime.dev/).
+.PHONY: compiler-wasm
+compiler-wasm: build/compiler.wasm
+
+build/compiler-javy.iife.js: src/javy-entry.ts src/javy-api.ts src/javy-ohm-js-stub.ts \
+    build/ohmRuntime.wasm_sections.ts build/ohm-grammar-wasm.ts $(SRC_TS_FILES) tsdown.javy.config.ts
+	pnpm tsdown --config tsdown.javy.config.ts
+
+build/compiler.wasm: build/compiler-javy.iife.js
+	$(JAVY) build $< -o $@
+
+.PHONY: test-compiler-wasm
+test-compiler-wasm: build/compiler.wasm
+	pnpm ava test/_test-javy.js
+
 .PHONY: go-test-es5
 go-test-es5: build/es5.wasm
 	cd test/go && go test -v

packages/compiler/src/javy-api.ts

@@ -0,0 +1,20 @@
+// Javy-specific compile API. The normal compile path (parseGrammars.ts) uses
+// the Wasm-compiled Ohm meta-grammar to parse grammars, but QuickJS/Javy
+// doesn't have a WebAssembly runtime. So this module uses ohm-js-legacy's
+// pure-JS grammar parser instead, the same approach as scripts/bootstrap.ts.
+
+import {grammars} from 'ohm-js-legacy';
+
+import {Compiler} from './Compiler.ts';
+
+export function compile(source: string, grammarName?: string): Uint8Array {
+  const ns = grammars(source);
+  let g;
+  if (grammarName) {
+    g = ns[grammarName];
+    if (!g) throw new Error(`Grammar '${grammarName}' not found`);
+  } else {
+    g = Object.values(ns).at(-1);
+  }
+  return new Compiler(g, {debug: false}).compile();
+}

packages/compiler/src/javy-entry.ts

@@ -0,0 +1,52 @@
+// Entry point for the Javy-compiled Wasm module.
+// Protocol: stdin = UTF-8 grammar source, stdout = compiled Wasm bytes.
+
+import {compile} from './javy-api.ts';
+
+declare const Javy: {
+  IO: {
+    readSync(fd: number, buf: Uint8Array): number;
+    writeSync(fd: number, buf: Uint8Array): void;
+  };
+};
+
+function readStdin(): string {
+  const chunks: Uint8Array[] = [];
+  const buf = new Uint8Array(4096);
+  while (true) {
+    const n = Javy.IO.readSync(0, buf);
+    if (n === 0) break;
+    chunks.push(buf.slice(0, n));
+  }
+  let total = 0;
+  for (const c of chunks) total += c.length;
+  const result = new Uint8Array(total);
+  let offset = 0;
+  for (const c of chunks) {
+    result.set(c, offset);
+    offset += c.length;
+  }
+  return new TextDecoder().decode(result);
+}
+
+function stderr(msg: string): void {
+  Javy.IO.writeSync(2, new TextEncoder().encode(msg));
+}
+
+// Parse an optional "#grammarName <name>" directive from the first line of input.
+// If present, returns [grammarName, remainingSource]; otherwise [undefined, source].
+function parseHeader(input: string): [string | undefined, string] {
+  const match = input.match(/^#grammarName (\S+)\n([\s\S]*)$/);
+  if (match) return [match[1], match[2]];
+  return [undefined, input];
+}
+
+try {
+  const raw = readStdin();
+  const [grammarName, source] = parseHeader(raw);
+  const wasmBytes = compile(source, grammarName);
+  Javy.IO.writeSync(1, wasmBytes);
+} catch (e: any) {
+  const msg = e?.message ?? String(e);
+  stderr(msg + '\n');
+}

packages/compiler/src/javy-ohm-js-stub.ts

@@ -0,0 +1,10 @@
+// Stub for ohm-js (v18) — used in the Javy build where WebAssembly is not
+// available. Compiler.ts transitively imports parseGrammars.ts which imports
+// Grammar from ohm-js, but this code path is never reached when grammars are
+// parsed via ohm-js-legacy.
+
+export class Grammar {
+  constructor() {
+    throw new Error('ohm-js v18 Grammar is not available in this environment');
+  }
+}

packages/compiler/src/leb128.ts

@@ -0,0 +1,32 @@
+// Pure JS LEB128 decoding, replacing @thi.ng/leb128 (which uses Wasm internally).
+// API matches @thi.ng/leb128: (data: Uint8Array, offset?: number) => [bigint, number]
+
+export function decodeULEB128(data: Uint8Array, offset: number = 0): [bigint, number] {
+  let result = 0n;
+  let shift = 0n;
+  let pos = offset;
+  while (true) {
+    const byte = data[pos++];
+    result |= BigInt(byte & 0x7f) << shift;
+    if ((byte & 0x80) === 0) break;
+    shift += 7n;
+  }
+  return [result, pos - offset];
+}
+
+export function decodeSLEB128(data: Uint8Array, offset: number = 0): [bigint, number] {
+  let result = 0n;
+  let shift = 0n;
+  let pos = offset;
+  let byte: number;
+  do {
+    byte = data[pos++];
+    result |= BigInt(byte & 0x7f) << shift;
+    shift += 7n;
+  } while ((byte & 0x80) !== 0);
+  // Sign extend if the highest bit of the last byte is set.
+  if (shift < 64n && (byte & 0x40) !== 0) {
+    result |= -(1n << shift);
+  }
+  return [result, pos - offset];
+}

packages/compiler/src/rewriteFuncIdx.ts

@@ -2,7 +2,7 @@
 // for additional imports in the dest module. Used by Compiler.ts at compile
 // time to adjust prebuilt code for extra imports (e.g. debug imports).
 
-import {decodeULEB128, decodeSLEB128} from '@thi.ng/leb128';
+import {decodeULEB128, decodeSLEB128} from './leb128.ts';
 import * as w from '@wasmgroundup/emit';
 
 // For sanity checking, assume that the number of locals is never

packages/compiler/test/_test-javy.js

@@ -0,0 +1,68 @@
+// Tests for the Javy-compiled Wasm module (build/compiler.wasm).
+// Run via `make test-compiler-wasm`.
+
+/* eslint-disable ava/no-ignored-test-files */
+
+import test from 'ava';
+import {execFileSync} from 'node:child_process';
+import {URL} from 'node:url';
+import {Grammar} from 'ohm-js';
+
+import {compile} from '../src/javy-api.ts';
+
+const COMPILER_WASM = new URL('../build/compiler.wasm', import.meta.url).pathname;
+
+function javyCompile(grammarSource, grammarName) {
+  const input = grammarName ? `#grammarName ${grammarName}\n${grammarSource}` : grammarSource;
+  return execFileSync('wasmtime', [COMPILER_WASM], {
+    input,
+    maxBuffer: 1024 * 1024,
+  });
+}
+
+test('javy: simple grammar', t => {
+  const bytes = javyCompile('G { start = "hello" "world" }');
+  const g = new Grammar(bytes);
+  g.match('helloworld').use(r => t.false(r.failed()));
+  g.match('goodbye').use(r => t.true(r.failed()));
+});
+
+test('javy: arithmetic grammar', t => {
+  const source = `Arithmetic {
+  Exp = AddExp
+  AddExp = AddExp "+" MulExp  -- plus
+         | AddExp "-" MulExp  -- minus
+         | MulExp
+  MulExp = MulExp "*" PriExp  -- times
+         | MulExp "/" PriExp  -- divide
+         | PriExp
+  PriExp = "(" Exp ")"  -- paren
+         | number
+  number = digit+
+}`;
+  const bytes = javyCompile(source);
+  const g = new Grammar(bytes);
+  g.match('1+2*3').use(r => t.false(r.failed()));
+  g.match('(1+2)*3').use(r => t.false(r.failed()));
+  g.match('hello').use(r => t.true(r.failed()));
+});
+
+test('javy: output matches Node compiler', t => {
+  const source = 'G { start = "a" | "b" | "c" }';
+  const javyBytes = javyCompile(source);
+  const nodeBytes = compile(source);
+  t.deepEqual(new Uint8Array(javyBytes), nodeBytes);
+});
+
+test('javy: #grammarName selects grammar', t => {
+  const source = 'A { start = "a" }\nB { start = "b" }';
+  const bytesA = javyCompile(source, 'A');
+  const gA = new Grammar(bytesA);
+  gA.match('a').use(r => t.false(r.failed()));
+  gA.match('b').use(r => t.true(r.failed()));
+
+  const bytesB = javyCompile(source, 'B');
+  const gB = new Grammar(bytesB);
+  gB.match('b').use(r => t.false(r.failed()));
+  gB.match('a').use(r => t.true(r.failed()));
+});

packages/compiler/test/test-leb128.js

@@ -0,0 +1,110 @@
+// Cross-check local leb128.ts against @thi.ng/leb128.
+
+import test from 'ava';
+import * as thi from '@thi.ng/leb128';
+
+import {decodeULEB128, decodeSLEB128} from '../src/leb128.ts';
+
+function encodeULEB128(value) {
+  const bytes = [];
+  let v = BigInt(value);
+  do {
+    let byte = Number(v & 0x7fn);
+    v >>= 7n;
+    if (v !== 0n) byte |= 0x80;
+    bytes.push(byte);
+  } while (v !== 0n);
+  return new Uint8Array(bytes);
+}
+
+function encodeSLEB128(value) {
+  const bytes = [];
+  let v = BigInt(value);
+  let more = true;
+  while (more) {
+    let byte = Number(v & 0x7fn);
+    v >>= 7n;
+    if ((v === 0n && (byte & 0x40) === 0) || (v === -1n && (byte & 0x40) !== 0)) {
+      more = false;
+    } else {
+      byte |= 0x80;
+    }
+    bytes.push(byte);
+  }
+  return new Uint8Array(bytes);
+}
+
+const uleb128Cases = [
+  0n,
+  1n,
+  63n, // max 1-byte value (6 bits + no sign bit concern for unsigned)
+  127n, // 0x7f — largest value fitting in 7 bits
+  128n, // 0x80 — first 2-byte value
+  255n,
+  256n,
+  16383n, // max 2-byte value
+  16384n, // first 3-byte value
+  624485n, // classic Wikipedia example
+  2097151n, // max 3-byte value
+  (1n << 32n) - 1n, // u32 max
+  1n << 32n, // just over u32
+  (1n << 53n) - 1n, // Number.MAX_SAFE_INTEGER as bigint
+];
+
+const sleb128Cases = [
+  0n,
+  1n,
+  -1n,
+  63n, // max positive 1-byte value
+  64n, // first positive 2-byte value (bit 6 = sign)
+  -64n, // min negative 1-byte value
+  -65n, // first negative 2-byte value
+  127n,
+  -128n,
+  128n,
+  -129n,
+  8191n, // max positive 2-byte value
+  -8192n, // min negative 2-byte value
+  (1n << 31n) - 1n, // i32 max
+  -(1n << 31n), // i32 min
+];
+
+for (const value of uleb128Cases) {
+  test(`decodeULEB128: ${value}`, t => {
+    const encoded = encodeULEB128(value);
+    const [localVal, localLen] = decodeULEB128(encoded);
+    const [thiVal, thiLen] = thi.decodeULEB128(encoded);
+    t.is(localVal, thiVal);
+    t.is(localLen, thiLen);
+  });
+}
+
+for (const value of sleb128Cases) {
+  test(`decodeSLEB128: ${value}`, t => {
+    const encoded = encodeSLEB128(value);
+    const [localVal, localLen] = decodeSLEB128(encoded);
+    const [thiVal, thiLen] = thi.decodeSLEB128(encoded);
+    t.is(localVal, thiVal);
+    t.is(localLen, thiLen);
+  });
+}
+
+test('decodeULEB128: with offset', t => {
+  const prefix = new Uint8Array([0xde, 0xad]);
+  const encoded = encodeULEB128(624485n);
+  const buf = new Uint8Array([...prefix, ...encoded]);
+  const [localVal, localLen] = decodeULEB128(buf, 2);
+  const [thiVal, thiLen] = thi.decodeULEB128(buf, 2);
+  t.is(localVal, thiVal);
+  t.is(localLen, thiLen);
+});
+
+test('decodeSLEB128: with offset', t => {
+  const prefix = new Uint8Array([0xde, 0xad, 0xbe]);
+  const encoded = encodeSLEB128(-123456n);
+  const buf = new Uint8Array([...prefix, ...encoded]);
+  const [localVal, localLen] = decodeSLEB128(buf, 3);
+  const [thiVal, thiLen] = thi.decodeSLEB128(buf, 3);
+  t.is(localVal, thiVal);
+  t.is(localLen, thiLen);
+});

packages/compiler/tsdown.javy.config.ts

@@ -0,0 +1,40 @@
+import {dirname, resolve} from 'node:path';
+import {fileURLToPath} from 'node:url';
+import {defineConfig} from 'tsdown';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+export default defineConfig({
+  entry: {'compiler-javy': 'src/javy-entry.ts'},
+  format: 'iife',
+  outDir: 'build',
+  clean: false,
+  sourcemap: false,
+  dts: false,
+  external: [],
+  noExternal: [/.*/],
+  inlineOnly: false,
+  alias: {
+    'ohm-js': resolve(__dirname, 'src/javy-ohm-js-stub.ts'),
+  },
+  // Polyfill atob — not available in QuickJS but used by base64-encoded
+  // Wasm sections. Must run before the IIFE body.
+  banner: `
+var atob = function(encoded) {
+  var chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
+  var result = '';
+  encoded = encoded.replace(/=+$/, '');
+  for (var i = 0; i < encoded.length; ) {
+    var a = chars.indexOf(encoded[i++]);
+    var b = i < encoded.length ? chars.indexOf(encoded[i++]) : 0;
+    var c = i < encoded.length ? chars.indexOf(encoded[i++]) : -1;
+    var d = i < encoded.length ? chars.indexOf(encoded[i++]) : -1;
+    var bits = (a << 18) | (b << 12) | ((c === -1 ? 0 : c) << 6) | (d === -1 ? 0 : d);
+    result += String.fromCharCode((bits >> 16) & 0xff);
+    if (c !== -1) result += String.fromCharCode((bits >> 8) & 0xff);
+    if (d !== -1) result += String.fromCharCode(bits & 0xff);
+  }
+  return result;
+};
+`,
+});