diff --git a/packages/miniohm-js/index.js b/packages/miniohm-js/index.js index ea11ede6..f403fa84 100644 --- a/packages/miniohm-js/index.js +++ b/packages/miniohm-js/index.js @@ -127,6 +127,10 @@ export class WasmMatcher { buf[written] = 0xff; // Mark end of input with an invalid UTF-8 character. return written; } + + getRightmostFailurePosition() { + return this._instance.exports.rightmostFailurePos.value; + } } class CstNode { @@ -169,7 +173,7 @@ class CstNode { get children() { const children = []; for (let i = 0; i < this.count; i++) { - const slotOffset = this._base + 12 + i * 4; + const slotOffset = this._base + 16 + i * 4; children.push( new CstNode(this._ruleNames, this._view, this._view.getUint32(slotOffset, true)), ); diff --git a/packages/wasm/TODO.md b/packages/wasm/TODO.md index 9ee042ac..baae2154 100644 --- a/packages/wasm/TODO.md +++ b/packages/wasm/TODO.md @@ -1,7 +1,7 @@ ## TODOs - [x] Include a map of rule name to ruleId in the module. -- [ ] Implicit space skipping +- [x] Implicit space skipping - [ ] Error handling - [x] NonterminalNodes should keep track of the rule - [ ] When iteration contains a sequence, the children are flattened into the iter node. @@ -9,20 +9,29 @@ - [x] Parameterized rules with >3 params - [x] Parameters that aren't terminals - [x] Memoization for parameterized rules -- [ ] Avoid unnecessary dispatch in generalized rules -- [ ] Avoid duplicate lifted rules. - [x] Support direct left recursion. -- [ ] Handle left recursion detection at grammar parse time. - [x] Separate API for _creating_ the Wasm module from the WasmMatcher interface. - [x] Implement a proper CLI. + +Cleanups: + +- [ ] Handle left recursion detection at grammar parse time. - [ ] Handle non-memoization of inline rules at grammar parse time +- [ ] Move to a failureOffset in memo entries +- [ ] Add assertions for any known input size limitations. + +Optimizations: + +- [ ] Avoid unnecessary dispatch in generalized rules +- [ ] Avoid duplicate lifted rules. +- [ ] Compressed (32-bit) header for Nonterminal nodes in common case +- [ ] Compressed (inline 32-bit) repr for Terminal nodes +- [ ] Proper preallocated nodes (incl. failurePos) for common cases ## Limitations - The input is assumed to be no bigger than 64k. - For the memo table, we assume that there are no more than 256 rules in the grammar. -- Parameterized rules only support up to 3 parameters, and no memoization. - - Parameters must be terminals. ## Unanswered questions diff --git a/packages/wasm/package.json b/packages/wasm/package.json index 99fde259..bf2ffc7f 100644 --- a/packages/wasm/package.json +++ b/packages/wasm/package.json @@ -26,6 +26,7 @@ "assemblyscript": "^0.27.36", "ava": "^6.2.0", "esbuild": "^0.25.5", + "fast-check": "^4.2.0", "fast-glob": "^3.3.3", "liquid-html-parser": "link:@shopify/liquid-html-parser", "mitata": "^1.0.34", diff --git a/packages/wasm/runtime/ohmRuntime.ts b/packages/wasm/runtime/ohmRuntime.ts index 7269e36d..c1802e84 100644 --- a/packages/wasm/runtime/ohmRuntime.ts +++ b/packages/wasm/runtime/ohmRuntime.ts @@ -1,4 +1,4 @@ -type Result = i32; +type ApplyResult = bool; declare function fillInputBuffer(offset: i32, maxLen: i32): i32; declare function printI32(val: i32): void; @@ -13,27 +13,57 @@ declare function isRuleSyntactic(ruleId: i32): bool; @inline const STACK_START_OFFSET: usize = WASM_PAGE_SIZE; @inline const MAX_INPUT_LEN_BYTES: usize = 64 * 1024; -// Note: the rule evaluation functions use a different representation. -// They return non-zero for success and zero for failure. -@inline const EMPTY: Result = 0; -@inline const FAIL: Result = 0xfffffff0; -@inline const UNUSED_LR_BOMB: Result = FAIL | 0x1; -@inline const USED_LR_BOMB: Result = FAIL | 0x3; - -@inline const CST_NODE_OVERHEAD: usize = 12; +// CST nodes +@inline const CST_NODE_OVERHEAD: usize = 16; +@inline const NODE_TYPE_TERMINAL: i32 = -1; @inline const NODE_TYPE_ITERATION: i32 = -2; +// Memo table entries +type MemoEntry = i32; + +@inline const EMPTY: MemoEntry = 0; + +// Low bit: failure flag. +// Rest: failurePos (signed int, 31 bits). +@inline const MEMO_FAILURE_FLAG: MemoEntry = 0x1; + +// Not: left recursion bombs never include failurePos. +// We need to be careful that a true failure w/ failurePos can't produce +// the same value. Because failurePos >= -1, we can use -2 and -3. +// TODO: Use failureOffset (unsigned) instead? That's what we do in JS. +@inline const UNUSED_LR_BOMB: MemoEntry = (-2 << 1) | MEMO_FAILURE_FLAG; +@inline const USED_LR_BOMB: MemoEntry = (-3 << 1) | MEMO_FAILURE_FLAG + +// The result of a raw rule evaluation function. +// Low bit: RULE_EVAL_SUCCESS_FLAG +// Rest: failurePos (signed int, 31 bits). +type RuleEvalResult = i32; + +@inline const RULE_EVAL_SUCCESS_FLAG = 1; + // Shared globals let pos: i32 = 0; + +// The rightmost position at which a leaf (Terminal, etc.) failed to match. +let rightmostFailurePos: i32 = 0; + let sp: usize = 0; let bindings: Array = new Array(); -@inline function memoTableGet(memoPos: usize, ruleId: i32): Result { - return load(memoPos * MEMO_COL_SIZE_BYTES + ruleId * sizeof(), MEMO_START_OFFSET); +@inline function max(a: T, b: T): T { + return a > b ? a : b; +} + +@inline function memoEntryForFailure(failurePos: i32): MemoEntry { + return (failurePos << 1) | MEMO_FAILURE_FLAG; +} + +@inline function memoTableGet(memoPos: usize, ruleId: i32): MemoEntry { + return load(memoPos * MEMO_COL_SIZE_BYTES + ruleId * sizeof(), MEMO_START_OFFSET); } -@inline function memoTableSet(memoPos: usize, ruleId: i32, value: Result): void { - store(memoPos * MEMO_COL_SIZE_BYTES + ruleId * sizeof(), value, MEMO_START_OFFSET); +@inline function memoTableSet(memoPos: usize, ruleId: i32, value: MemoEntry): void { + store(memoPos * MEMO_COL_SIZE_BYTES + ruleId * sizeof(), value, MEMO_START_OFFSET); } @inline function cstGetCount(ptr: usize): i32 { @@ -60,28 +90,27 @@ let bindings: Array = new Array(); store(ptr, t, 8); } -@inline function memoizeResult(memoPos: usize, ruleId: i32, result: Result): void { - memoTableSet(memoPos, ruleId, result); +@inline function cstGetFailurePos(ptr: usize): i32 { + return load(ptr, 12); } -@inline function isFailure(result: Result): bool { - return result < 0; +@inline function cstSetFailurePos(ptr: usize, pos: i32): void { + store(ptr, pos, 12); } -function useMemoizedResult(ruleId: i32, result: Result): Result { - if (result === UNUSED_LR_BOMB) { - memoTableSet(pos, ruleId, USED_LR_BOMB); - return 0; - } else if (isFailure(result)) { - return 0; +function useMemoizedResult(ruleId: i32, result: MemoEntry): ApplyResult { + if (result & MEMO_FAILURE_FLAG) { + if (result === UNUSED_LR_BOMB) { + memoTableSet(pos, ruleId, USED_LR_BOMB); + } else { + rightmostFailurePos = max(rightmostFailurePos, result >> 1); + } + return false; } pos += cstGetMatchLength(result); + rightmostFailurePos = max(rightmostFailurePos, cstGetFailurePos(result)); bindings.push(result); - return result; -} - -function hasMemoizedResult(ruleId: i32): boolean { - return memoTableGet(pos, ruleId) !== 0; + return true; } @inline function maybeSkipSpaces(ruleId: i32): void { @@ -93,12 +122,18 @@ function hasMemoizedResult(ruleId: i32): boolean { } } -export function match(startRuleId: i32): Result { - // (Re-)initialize globals, clear memo table. +function resetParsingState(): void { pos = 0; + rightmostFailurePos = -1; sp = STACK_START_OFFSET; + heap.reset(); + bindings = new Array(); memory.fill(MEMO_START_OFFSET, 0, MEMO_COL_SIZE_BYTES * MAX_INPUT_LEN_BYTES); +} + +export function match(startRuleId: i32): ApplyResult { + resetParsingState(); // Get the input and do the match. let inputLen = fillInputBuffer(0, i32(WASM_PAGE_SIZE)); @@ -107,78 +142,103 @@ export function match(startRuleId: i32): Result { const succeeded = evalApply0(startRuleId) !== 0; if (succeeded) { maybeSkipSpaces(startRuleId); + // printI32(heap.alloc(8) - __heap_base); // Print heap usage. + // TODO: Do we need to update rightmostFailurePos here? return inputLen === pos; } - return 0; + + return false; } -@inline function evalRuleBody(ruleId: i32): Result { - return call_indirect(ruleId); +@inline function evalRuleBody(ruleId: i32): RuleEvalResult { + return call_indirect(ruleId); } -export function evalApplyGeneralized(ruleId: i32, caseIdx: i32): Result { +// Extracts the local failure position from a RuleEvalResult. +// If it's greater than the global rightmostFailurePos, it updates it. +// Returns the local failure position. +@inline function maybeUpdateRightmostFailurePos(result: RuleEvalResult): i32 { + const failurePos = result >> 1; + rightmostFailurePos = max(rightmostFailurePos, failurePos); + return failurePos; +} + +// Evaluates a generalized rule. Identical to evalApplyNoMemo0, but includes +// the caseIdx. +export function evalApplyGeneralized(ruleId: i32, caseIdx: i32): ApplyResult { const origPos = pos; const origNumBindings = bindings.length; - if (call_indirect(ruleId, caseIdx)) { - return newNonterminalNode(origPos, pos, ruleId, origNumBindings); + const result = call_indirect(ruleId, caseIdx) + const failurePos = maybeUpdateRightmostFailurePos(result); + if (result & RULE_EVAL_SUCCESS_FLAG) { + newNonterminalNode(origPos, pos, ruleId, origNumBindings, failurePos); + return true; } - return 0; + return false; } -export function evalApplyNoMemo0(ruleId: i32): Result { +export function evalApplyNoMemo0(ruleId: i32): ApplyResult { const origPos = pos; const origNumBindings = bindings.length; - if (evalRuleBody(ruleId)) { - return newNonterminalNode(origPos, pos, ruleId, origNumBindings); + let result = evalRuleBody(ruleId); + const failurePos = maybeUpdateRightmostFailurePos(result); + if (result & RULE_EVAL_SUCCESS_FLAG) { + newNonterminalNode(origPos, pos, ruleId, origNumBindings, failurePos); + return true; } - return 0; + return false; } -export function evalApply0(ruleId: i32): Result { - let result = memoTableGet(pos, ruleId); - if (result !== 0) { - return useMemoizedResult(ruleId, result); +export function evalApply0(ruleId: i32): ApplyResult { + const memo = memoTableGet(pos, ruleId); + if (memo !== 0) { + return useMemoizedResult(ruleId, memo); } const origPos = pos; - let origNumBindings = bindings.length; - memoizeResult(origPos, ruleId, UNUSED_LR_BOMB); - let succeeded: i32 = evalRuleBody(ruleId); + const origNumBindings = bindings.length; + memoTableSet(origPos, ruleId, UNUSED_LR_BOMB); + + const result = evalRuleBody(ruleId); + const failurePos = maybeUpdateRightmostFailurePos(result); // Straight failure — record a clean failure in the memo table. - if (!succeeded) { - memoizeResult(origPos, ruleId, FAIL); - return 0; + if ((result & RULE_EVAL_SUCCESS_FLAG) == 0) { + memoTableSet(origPos, ruleId, memoEntryForFailure(failurePos)); + return false; } if (memoTableGet(origPos, ruleId) === USED_LR_BOMB) { - return handleLeftRecursion(origPos, ruleId, origNumBindings); + return handleLeftRecursion(origPos, ruleId, origNumBindings, failurePos); } - // No left recursion — memoize and return. - result = newNonterminalNode(origPos, pos, ruleId, origNumBindings); - memoizeResult(origPos, ruleId, result); - return result; + const node = newNonterminalNode(origPos, pos, ruleId, origNumBindings, failurePos); + memoTableSet(origPos, ruleId, node); + return true; } -export function handleLeftRecursion(origPos: usize, ruleId: i32, origNumBindings: i32): Result { +export function handleLeftRecursion(origPos: usize, ruleId: i32, origNumBindings: i32, failurePos: i32): ApplyResult { let maxPos: i32; - let result: Result; - let succeeded: i32; + let node: usize; + let succeeded: bool; do { // The current result is the best one -- record it. maxPos = pos; - result = newNonterminalNode(origPos, pos, ruleId, origNumBindings); - memoizeResult(origPos, ruleId, result); + rightmostFailurePos = max(rightmostFailurePos, failurePos); + node = newNonterminalNode(origPos, pos, ruleId, origNumBindings, failurePos); + memoTableSet(origPos, ruleId, node); // Reset and try to improve on the current best. pos = origPos; bindings.length = origNumBindings; - succeeded = evalRuleBody(ruleId); + const result = evalRuleBody(ruleId); + succeeded = (result & RULE_EVAL_SUCCESS_FLAG) != 0; + failurePos = result >> 1; } while (succeeded && pos > maxPos); pos = maxPos; + bindings.length = origNumBindings + 1; - bindings[origNumBindings] = result; + bindings[origNumBindings] = node; return succeeded; } @@ -186,19 +246,21 @@ export function newTerminalNode(startIdx: i32, endIdx: i32): usize { const ptr = heap.alloc(CST_NODE_OVERHEAD); cstSetCount(ptr, 0); cstSetMatchLength(ptr, endIdx - startIdx); - cstSetType(ptr, -1); + cstSetType(ptr, NODE_TYPE_TERMINAL); + cstSetFailurePos(ptr, 0); bindings.push(ptr); return ptr; } // Create an internal (non-leaf) node (IterationNode or NonterminalNode). -@inline function newNonLeafNodeWithType(startIdx: i32, endIdx: i32, type: i32, origNumBindings: i32): usize { +@inline function newNonLeafNode(startIdx: i32, endIdx: i32, type: i32, origNumBindings: i32, failurePos: i32): usize { const bindingsLen = bindings.length; const numChildren = bindingsLen - origNumBindings; const ptr = heap.alloc(CST_NODE_OVERHEAD + numChildren * 4); cstSetCount(ptr, numChildren); cstSetMatchLength(ptr, endIdx - startIdx); cstSetType(ptr, type); + cstSetFailurePos(ptr, failurePos); for (let i = 0; i < numChildren; i++) { store(ptr + CST_NODE_OVERHEAD + i * 4, bindings[bindingsLen - numChildren + i]); } @@ -207,12 +269,12 @@ export function newTerminalNode(startIdx: i32, endIdx: i32): usize { return ptr; } -export function newNonterminalNode(startIdx: i32, endIdx: i32, ruleId: i32, origNumBindings: i32): usize { - return newNonLeafNodeWithType(startIdx, endIdx, ruleId, origNumBindings); +export function newNonterminalNode(startIdx: i32, endIdx: i32, ruleId: i32, origNumBindings: i32, failurePos: i32): usize { + return newNonLeafNode(startIdx, endIdx, ruleId, origNumBindings, failurePos); } export function newIterationNode(startIdx: i32, endIdx: i32, origNumBindings: i32): usize { - return newNonLeafNodeWithType(startIdx, endIdx, NODE_TYPE_ITERATION, origNumBindings); + return newNonLeafNode(startIdx, endIdx, NODE_TYPE_ITERATION, origNumBindings, -1); } export function getBindingsLength(): i32 { diff --git a/packages/wasm/src/index.js b/packages/wasm/src/index.js index ac7a2531..471c110a 100644 --- a/packages/wasm/src/index.js +++ b/packages/wasm/src/index.js @@ -69,6 +69,8 @@ function isSyntacticRule(ruleName) { return ruleName[0] === ruleName[0].toUpperCase(); } +const asciiChars = Array.from({length: 128}).map((_, i) => String.fromCharCode(i)); + class IndexedSet { constructor() { this._map = new Map(); @@ -281,32 +283,33 @@ class Assembler { this._code.push(...checkNoUndefined(bytes.flat(Infinity))); } - block(bt, bodyThunk) { - this._blockOnly(bt); + block(bt, bodyThunk, label = '') { + this._blockOnly(bt, label); bodyThunk(); this._endBlock(); } // Prefer to use `block`, but for some cases it's more convenient to emit // the block and the end separately. - _blockOnly(bt) { - this.emit(w.instr.block, bt); - this._blockStack.push('block'); + // Note: `label` (if specified) is not unique (e.g., 'pexprEnd'). + _blockOnly(bt, label) { + this.emit(instr.block, bt); + this._blockStack.push(label ? `block:${label}` : 'block'); } // This should always be paired with `blockOnly`. _endBlock() { - const what = this._blockStack.pop(); + const what = this._blockStack.pop().split(':')[0]; assert(what === 'block', 'Invalid endBlock'); - this.emit(w.instr.end); + this.emit(instr.end); } loop(bt, bodyThunk) { - this.emit(w.instr.loop, bt); + this.emit(instr.loop, bt); this._blockStack.push('loop'); bodyThunk(); this._blockStack.pop(); - this.emit(w.instr.end); + this.emit(instr.end); } if(bt, bodyThunk) { @@ -314,15 +317,15 @@ class Assembler { } ifElse(bt, thenThunk, elseThunk = undefined) { - this.emit(w.instr.if, bt); + this.emit(instr.if, bt); this._blockStack.push('if'); thenThunk(); if (elseThunk) { - this.emit(w.instr.else); + this.emit(instr.else); elseThunk(); } this._blockStack.pop(); - this.emit(w.instr.end); + this.emit(instr.end); } ifFalse(bt, bodyThunk) { @@ -354,6 +357,10 @@ class Assembler { this.emit(instr.i32.mul); } + i32Eq() { + this.emit(instr.i32.eq); + } + i32Ne() { this.emit(instr.i32.ne); } @@ -388,32 +395,30 @@ class Assembler { } break(depth) { - const what = this._blockStack.at(-(depth + 1)); + const what = this._blockStack.at(-(depth + 1)).split(':')[0]; assert(what === 'block' || what === 'if', 'Invalid break'); this.emit(instr.br, w.labelidx(depth)); } // Conditional break -- emits a `br_if` for the given depth. condBreak(depth) { - const what = this._blockStack.at(-(depth + 1)); + const what = this._blockStack.at(-(depth + 1)).split(':')[0]; assert(what === 'block' || what === 'if', 'Invalid condBreak'); this.emit(instr.br_if, w.labelidx(depth)); } continue(depth) { - const what = this._blockStack.at(-(depth + 1)); + const what = this._blockStack.at(-(depth + 1)).split(':')[0]; assert(what === 'loop', 'Invalid continue'); this.emit(instr.br, w.labelidx(depth)); } brTable(labels, defaultLabelidx) { - this.emit(w.instr.br_table, w.vec(labels), defaultLabelidx); + this.emit(instr.br_table, w.vec(labels), defaultLabelidx); } return() { - const what = this._blockStack[0]; - assert(what === 'block', 'Invalid return'); - this.emit(w.instr.return); + this.emit(instr.return); } // Emit a dense jump table (switch-like) using br_table. @@ -475,13 +480,17 @@ class Assembler { this.localSet('ret'); } - pushStackFrame() { + pushStackFrame(saveThunk) { this.globalGet('sp'); this.i32Const(Assembler.STACK_FRAME_SIZE_BYTES); this.i32Sub(); this.globalSet('sp'); - this.savePos(); - this.saveNumBindings(); + if (saveThunk) { + saveThunk(); + } else { + this.savePos(); + this.saveNumBindings(); + } } popStackFrame() { @@ -538,6 +547,45 @@ class Assembler { } } + saveFailurePos() { + this.globalGet('sp'); + this.localGet('failurePos'); + this.i32Store(); + } + + restoreFailurePos() { + this.globalGet('sp'); + this.i32Load(); + this.localSet('failurePos'); + } + + saveGlobalFailurePos() { + this.globalGet('sp'); + this.globalGet('rightmostFailurePos'); + this.i32Store(4); + } + + restoreGlobalFailurePos() { + this.globalGet('sp'); + this.i32Load(4); + this.globalSet('rightmostFailurePos'); + } + + updateGlobalFailurePos() { + // rightmostFailurePos = max(rightmostFailurePos, failurePos) + this.i32Max( + () => this.globalGet('rightmostFailurePos'), + () => this.localGet('failurePos'), + ); + this.globalSet('rightmostFailurePos'); + } + + updateLocalFailurePos(origPosThunk) { + // failurePos = max(failurePos, origPos) + this.i32Max(() => this.localGet('failurePos'), origPosThunk); + this.localSet('failurePos'); + } + // Increment the current input position by 1. // [i32, i32] -> [i32] incPos() { @@ -571,6 +619,33 @@ class Assembler { ); this.localSet('ret'); } + + i32Max(aThunk, bThunk) { + aThunk(); + bThunk(); + aThunk(); + bThunk(); + this.emit(instr.i32.gt_s, instr.select); + } + + // Return the depth of the block with the given label. + depthOf(label) { + const i = this._blockStack.findLastIndex(what => what === `block:${label}`); + assert(i !== -1, `Unknown label: ${label}`); + return this._blockStack.length - i - 1; + } + + ruleEvalReturn() { + // Convert the value in `ret` to a single bit in position 0. + this.localGet('ret'); + this.emit(instr.i32.eqz, instr.i32.eqz); + + // Remaining 32 bits hold the (signed) failurePos. + this.localGet('failurePos'); + this.i32Const(1); + this.emit(instr.i32.shl); + this.emit(instr.i32.or); + } } Assembler.ALIGN_1_BYTE = 0; Assembler.ALIGN_4_BYTES = 2; @@ -597,17 +672,22 @@ export class Compiler { // The rule ID is a 0-based index that's mapped to the name. // It is *not* the same as the function index the rule's eval function. this.ruleIdByName = new IndexedSet(); + + this._specialRules = ['spaces', 'alnum', 'any']; + // Ensure default start rule has id 0; $term, 1; and spaces, 2. this._ensureRuleId(grammar.defaultStartRule); this._ensureRuleId('$term'); - this._ensureRuleId('spaces'); + this._specialRules.forEach(name => { + this._ensureRuleId(name); + }); this.rules = undefined; this._nextLiftedId = 0; // Keeps track of whether we're in a lexical or syntactic context. this._lexContextStack = []; - this._applySpaces = ir.apply('spaces'); + this._applySpacesImplicit = ir.apply('$spaces'); } importCount() { @@ -630,6 +710,7 @@ export class Compiler { } liftPExpr(exp, isSyntactic) { + assert(EMIT_GENERALIZED_RULES, 'Lifting only happens w/ generalized rules'); assert(!(exp instanceof pexprs.Terminal)); // Note: the same expression might appear in more than one place, and @@ -711,6 +792,7 @@ export class Compiler { // (global $runtime/ohmRuntime/bindings (mut i32) (i32.const 0)) // (global $~lib/memory/__heap_base i32 (i32.const 1179884)) asm.addGlobal('pos', w.valtype.i32, w.mut.var, () => asm.i32Const(0)); + asm.addGlobal('rightmostFailurePos', w.valtype.i32, w.mut.var, () => asm.i32Const(-1)); asm.addGlobal('sp', w.valtype.i32, w.mut.var, () => asm.i32Const(0)); asm.addGlobal('__Runtime.Stub', w.valtype.i32, w.mut.const, () => asm.i32Const(0)); asm.addGlobal('__Runtime.Minimal', w.valtype.i32, w.mut.const, () => asm.i32Const(1)); @@ -741,26 +823,18 @@ export class Compiler { const {grammar} = this; const lookUpRule = name => { - const isSyntactic = isSyntacticRule(name); - if (name in grammar.rules) { - return {...grammar.rules[name], isSyntactic}; - } - if (grammar.superGrammar) { - return lookUpRule(name, grammar.superGrammar); - } + assert(name in grammar.rules); + return {...grammar.rules[name], isSyntactic: isSyntacticRule(name)}; }; - // Begin with all the rules in the grammar + spaces. + // Begin with all the rules in the grammar + all "special" rules. const rules = Object.entries(this.grammar.rules).map(([name, info]) => { const isSyntactic = isSyntacticRule(name); return [name, {...info, isSyntactic}]; }); - rules.push([ - 'spaces', - { - ...lookUpRule('spaces'), - }, - ]); + this._specialRules.forEach(name => { + rules.push([name, {...lookUpRule(name)}]); + }); const liftedTerminals = new IndexedSet(); @@ -853,14 +927,19 @@ export class Compiler { asm.addFunction(`$${name}`, [w.valtype.i32], [w.valtype.i32], () => { asm.addLocal('ret', w.valtype.i32); asm.addLocal('tmp', w.valtype.i32); + asm.addLocal('failurePos', w.valtype.i32); + asm.i32Const(-1); + asm.localSet('failurePos'); const values = this.liftedTerminals.values(); asm.switch( w.blocktype.empty, () => asm.localGet('__arg0'), values.length, - (i, depth) => this.emitTerminal(ir.terminal(values[i]), depth), - () => asm.emit(w.instr.unreachable), + i => this.emitTerminal(ir.terminal(values[i])), + () => asm.emit(instr.unreachable), ); + // Note: unlike a regular rule evaluation, this function just returns + // the raw result of PExpr evaluation. asm.localGet('ret'); }); this.endLexContext(); @@ -884,14 +963,40 @@ export class Compiler { if (ruleInfo.patterns) { paramTypes = [w.valtype.i32]; } + // const preHook = () => { + // if (['alnum'].includes(name)) { + // this.emitSingleCharFastPath('alnum'); + // } + // }; + + const restoreFailurePos = name === this._applySpacesImplicit.ruleName; + this.beginLexContext(!ruleInfo.isSyntactic); asm.addFunction(`$${name}`, paramTypes, [w.valtype.i32], () => { asm.addLocal('ret', w.valtype.i32); asm.addLocal('tmp', w.valtype.i32); + asm.addLocal('failurePos', w.valtype.i32); + asm.globalGet('rightmostFailurePos'); + asm.localSet('failurePos'); + + // TODO: Find a simpler way to do this. + if (restoreFailurePos) { + asm.addLocal('origFailurePos', w.valtype.i32); + asm.globalGet('rightmostFailurePos'); + asm.localSet('origFailurePos'); + } + asm.emit(`BEGIN eval:${name}`); this.emitPExpr(ruleInfo.body); + + if (restoreFailurePos) { + asm.localGet('origFailurePos'); + asm.dup(); + asm.globalSet('rightmostFailurePos'); + asm.localSet('failurePos'); + } + asm.ruleEvalReturn(); asm.emit(`END eval:${name}`); - asm.localGet('ret'); }); this.endLexContext(); return this.asm._functionDecls.at(-1); @@ -950,9 +1055,16 @@ export class Compiler { }, }); specialize(ir.apply(this.grammar.defaultStartRule)); - specialize(ir.apply('spaces')); + this._specialRules.forEach(name => { + specialize(ir.apply(name)); + }); this.rules = newRules; + // Make a special rule for implicit space skipping, with the same body + // as the real `spaces` rule. + this._ensureRuleId('$spaces', {notMemoized: true}); + newRules.set('$spaces', getNotNull(newRules, 'spaces')); + if (EMIT_GENERALIZED_RULES) { const insertDispatches = (exp, patterns) => ir.rewrite(exp, { @@ -1093,7 +1205,10 @@ export class Compiler { let pushArg = []; if (x.startsWith('END')) { decl.paramTypes = [w.valtype.i32]; - pushArg = [instr.local.get, w.localidx(0)]; + // We want to pass 'ret', but to figure out its index, we need to + // account for the number of parameters. + const retIdx = entry.paramTypes.length; + pushArg = [instr.local.get, w.localidx(retIdx)]; } // …and replace the string with a call to that function. @@ -1152,27 +1267,31 @@ export class Compiler { patterns.length, handleCase, () => { - asm.emit('herre'); - asm.emit(w.instr.unreachable); + asm.emit(instr.unreachable); }, ); } // Contract: emitPExpr always means we're going deeper in the PExpr tree. - emitPExpr(exp) { + emitPExpr(exp, {preHook, postHook} = {}) { const {asm} = this; + const allowFastApply = !preHook && !postHook; + // Note that after specializeApplications, there are two classes of rule: // - specialized rules, which contain no Params, and only have // applications without args // - generalized rules, which may contain Params and apps w/ args. + assert(!(exp.type === 'Apply' && exp.children.length > 0)); - if (exp.type === 'Apply' && exp.children.length === 0) { + if (exp.type === 'Apply' && allowFastApply) { + asm.emit(`BEGIN apply:${exp.ruleName}`); this.emitApply(exp); + asm.emit(`END apply:${exp.ruleName}`); return; } if (exp.type === 'ApplyGeneralized') { - assert(EMIT_GENERALIZED_RULES); + assert(EMIT_GENERALIZED_RULES && allowFastApply); this.emitApplyGeneralized(exp); return; } @@ -1184,30 +1303,37 @@ export class Compiler { // Wrap the body in a block, which is useful for two reasons: // - it allows early returns. // - it makes sure that the generated code doesn't have stack effects. - asm.block(w.blocktype.empty, () => { - // prettier-ignore - switch (exp.type) { - case 'Alt': this.emitAlt(exp); break; - case 'Any': this.emitAny(); break; - case 'Dispatch': this.emitDispatch(exp); break; - case 'End': this.emitEnd(); break; - case 'Lex': this.emitLex(exp); break; - case 'LiftedTerminal': this.emitApplyTerm(exp); break; - case 'Lookahead': this.emitLookahead(exp, true); break; - case 'Not': this.emitLookahead(exp, false); break; - case 'Seq': this.emitSeq(exp); break; - case 'Star': this.emitStar(exp); break; - case 'Opt': this.emitOpt(exp); break; - case 'Range': this.emitRange(exp); break; - case 'Plus': this.emitPlus(exp); break; - case 'Terminal': this.emitTerminal(exp); break; - case 'UnicodeChar': this.emitUnicodeChar(exp); break; - case 'Param': - // Fall through (Params should not exist at codegen time). - default: - throw new Error(`not handled: ${exp.type}`); - } - }); + asm.block( + w.blocktype.empty, + () => { + if (preHook) preHook(); + + // prettier-ignore + switch (exp.type) { + case 'Alt': this.emitAlt(exp); break; + case 'Any': this.emitAny(); break; + case 'Dispatch': this.emitDispatch(exp); break; + case 'End': this.emitEnd(); break; + case 'Lex': this.emitLex(exp); break; + case 'LiftedTerminal': this.emitApplyTerm(exp); break; + case 'Lookahead': this.emitLookahead(exp); break; + case 'Not': this.emitNot(exp); break; + case 'Seq': this.emitSeq(exp); break; + case 'Star': this.emitStar(exp); break; + case 'Opt': this.emitOpt(exp); break; + case 'Range': this.emitRange(exp); break; + case 'Plus': this.emitPlus(exp); break; + case 'Terminal': this.emitTerminal(exp); break; + case 'UnicodeChar': this.emitUnicodeChar(exp); break; + case 'Param': + // Fall through (Params should not exist at codegen time). + default: + throw new Error(`not handled: ${exp.type}`); + } + }, + 'pexprEnd', + ); + if (postHook) postHook(); asm.popStackFrame(); asm.emit(`END ${debugLabel}`); } @@ -1218,7 +1344,7 @@ export class Compiler { for (const term of exp.children) { this.emitPExpr(term); asm.localGet('ret'); - asm.condBreak(0); // return if succeeded + asm.condBreak(asm.depthOf('pexprEnd')); asm.restorePos(); asm.restoreBindingsLength(); } @@ -1227,19 +1353,30 @@ export class Compiler { emitAny() { const {asm} = this; - this.maybeEmitSpaceSkipping(); - asm.i32Const(0xff); - asm.nextCharCode(); - asm.i32Ne(); - asm.maybeReturnTerminalNodeWithSavedPos(); + this.wrapTerminalLike(() => { + asm.i32Const(0xff); + asm.nextCharCode(); + asm.i32Eq(); + asm.condBreak(asm.depthOf('failure')); + }); } emitApplyTerm({terminalId}) { const {asm} = this; - this.maybeEmitSpaceSkipping(); + + // Save the original position. + asm.globalGet('pos'); + asm.localSet('tmp'); + + // Call the terminal rule, and use its result as ours. asm.i32Const(terminalId); - asm.emit(w.instr.call, this.ruleEvalFuncIdx('$term')); - asm.localSet('ret'); + asm.emit(instr.call, this.ruleEvalFuncIdx('$term')); + asm.localTee('ret'); + + // Update the failure position if necessary. + asm.ifFalse(w.blocktype.empty, () => { + asm.updateLocalFailurePos(() => asm.localGet('tmp')); + }); } // Emit an application of the generalized version of a parameterized rule. @@ -1255,31 +1392,36 @@ export class Compiler { emitApply(exp) { assert(exp.children.length === 0); - if (exp !== this._applySpaces) { - this.maybeEmitSpaceSkipping(); // Avoid infinite recursion. + // Avoid infinite recursion. + if (exp !== this._applySpacesImplicit) { + this.maybeEmitSpaceSkipping(); } const {asm} = this; asm.i32Const(this.ruleId(exp.ruleName)); + // TODO: Should lifted expressions be memoized? // TODO: Handle this at grammar parse time, not here. if (exp.ruleName.includes('_')) { asm.callPrebuiltFunc('evalApplyNoMemo0'); } else { asm.callPrebuiltFunc('evalApply0'); } + // The application may have updated rightmostFailurePos; if so, we may + // need to update the local failure position. + asm.updateLocalFailurePos(() => asm.globalGet('rightmostFailurePos')); asm.localSet('ret'); } emitEnd() { const {asm} = this; - - this.maybeEmitSpaceSkipping(); - asm.i32Const(0xff); - // Careful! We shouldn't move the pos here. Or does it matter? - asm.currCharCode(); - asm.emit(instr.i32.eq); - asm.maybeReturnTerminalNodeWithSavedPos(); + this.wrapTerminalLike(() => { + asm.i32Const(0xff); + // Careful! We shouldn't move the pos here. Or does it matter? + asm.currCharCode(); + asm.i32Ne(); + asm.condBreak(asm.depthOf('failure')); + }); } emitFail() { @@ -1294,16 +1436,34 @@ export class Compiler { this._lexContextStack.pop(); } - emitLookahead({child}, shouldMatch = true) { + emitLookahead({child}) { const {asm} = this; // TODO: Should positive lookahead record a CST? this.emitPExpr(child); - if (!shouldMatch) { - asm.localGet('ret'); - asm.emit(instr.i32.eqz); - asm.localSet('ret'); - } + asm.restoreBindingsLength(); + asm.restorePos(); + } + + emitNot({child}) { + const {asm} = this; + + // Push an inner stack frame with the failure positions. + asm.pushStackFrame(() => { + asm.saveFailurePos(); + asm.saveGlobalFailurePos(); + }); + this.emitPExpr(child); + + // Invert the result. + asm.localGet('ret'); + asm.emit(instr.i32.eqz); + asm.localSet('ret'); + + // Restore all global and local state. + asm.restoreGlobalFailurePos(); + asm.restoreFailurePos(); + asm.popStackFrame(); // Pop inner frame. asm.restoreBindingsLength(); asm.restorePos(); } @@ -1331,28 +1491,25 @@ export class Compiler { emitRange(exp) { assert(exp.lo.length === 1 && exp.hi.length === 1); - const lo = exp.lo.charCodeAt(0); const hi = exp.hi.charCodeAt(0); // TODO: Do we disallow 0xff in the range? const {asm} = this; - this.maybeEmitSpaceSkipping(); - asm.nextCharCode(); - - // if (c > hi) return 0; - asm.dup(); - asm.i32Const(hi); - asm.emit(instr.i32.gt_u); - asm.if(w.blocktype.empty, () => { - asm.setRet(0); - asm.break(1); + this.wrapTerminalLike(() => { + asm.nextCharCode(); + + // if (c > hi) return 0; + asm.dup(); + asm.i32Const(hi); + asm.emit(instr.i32.gt_u); + asm.condBreak(asm.depthOf('failure')); + + // if (c >= lo) return 0; + asm.i32Const(lo); + asm.emit(instr.i32.lt_u); + asm.condBreak(asm.depthOf('failure')); }); - - // if (c >= lo) - asm.i32Const(lo); - asm.emit(instr.i32.ge_u); - asm.maybeReturnTerminalNodeWithSavedPos(); } emitSeq({children}) { @@ -1368,14 +1525,14 @@ export class Compiler { this.emitPExpr(c); asm.localGet('ret'); asm.emit(instr.i32.eqz); - asm.condBreak(0); + asm.condBreak(asm.depthOf('pexprEnd')); } } maybeEmitSpaceSkipping() { if (IMPLICIT_SPACE_SKIPPING && !this.inLexicalContext()) { this.asm.emit('BEGIN space skipping'); - this.emitApply(this._applySpaces); + this.emitApply(this._applySpacesImplicit); this.asm.emit('END space skipping'); } } @@ -1386,17 +1543,21 @@ export class Compiler { // We push another stack frame because we need to save and restore // the position just before the last (failed) expression. asm.pushStackFrame(); - asm.block(w.blocktype.empty, () => { - asm.loop(w.blocktype.empty, () => { - asm.savePos(); - asm.saveNumBindings(); - this.emitPExpr(child); - asm.localGet('ret'); - asm.emit(instr.i32.eqz); - asm.condBreak(1); - asm.continue(0); - }); - }); + asm.block( + w.blocktype.empty, + () => { + asm.loop(w.blocktype.empty, () => { + asm.savePos(); + asm.saveNumBindings(); + this.emitPExpr(child); + asm.localGet('ret'); + asm.emit(instr.i32.eqz); + asm.condBreak(asm.depthOf('done')); + asm.continue(0); + }); + }, + 'done', + ); asm.restorePos(); asm.restoreBindingsLength(); asm.popStackFrame(); @@ -1405,57 +1566,89 @@ export class Compiler { asm.localSet('ret'); } - emitTerminal({value}, depth = 0) { - // TODO: - // - proper UTF-8! - // - handle longer terminals with a loop - // - SIMD + wrapTerminalLike(thunk) { + const {asm} = this; + this.maybeEmitSpaceSkipping(); + asm.block( + w.blocktype.empty, + () => { + asm.block( + w.blocktype.empty, + () => { + thunk(); + asm.newTerminalNodeWithSavedPos(); + asm.localSet('ret'); + asm.break(asm.depthOf('_success')); + }, + 'failure', + ); + asm.updateLocalFailurePos(() => asm.getSavedPos()); + asm.setRet(0); + }, + '_success', + ); + } + + emitTerminal({value}) { const {asm} = this; asm.emit(JSON.stringify(value)); - this.maybeEmitSpaceSkipping(); - for (const c of [...value]) { - // Compare next char - asm.i32Const(c.charCodeAt(0)); - asm.currCharCode(); - asm.emit(instr.i32.ne); - asm.if(w.blocktype.empty, () => { - asm.setRet(0); - asm.break(depth + 1); - }); - asm.incPos(); - } - asm.newTerminalNodeWithSavedPos(); - asm.localSet('ret'); + this.wrapTerminalLike(() => { + // TODO: + // - proper UTF-8! + // - handle longer terminals with a loop + // - SIMD + for (const c of [...value]) { + asm.i32Const(c.charCodeAt(0)); + asm.currCharCode(); + asm.emit(instr.i32.ne); + asm.condBreak(asm.depthOf('failure')); + asm.incPos(); + } + }); } emitUnicodeChar(exp) { const {asm} = this; - this.maybeEmitSpaceSkipping(); - - const handleDefault = () => { - // TODO: Implement the slow case by calling out to the host. - asm.emit(w.instr.unreachable); - }; // TODO: Add support for more categories, by calling out to the host. assert(['Ll', 'Lu', 'Ltmo'].includes(exp.category)); - const caseCb = (i, depth) => { - const c = String.fromCharCode(i); - if ( - (exp.category === 'Lu' && 'A' <= c && c <= 'Z') || - (exp.category === 'Ll' && 'a' <= c && c <= 'z') - ) { - asm.incPos(); - asm.newTerminalNodeWithSavedPos(); - asm.localSet('ret'); - } else { - asm.setRet(0); - } - asm.break(depth); - }; - asm.switch(w.blocktype.empty, () => asm.currCharCode(), 128, caseCb, handleDefault); + const makeLabels = () => + asciiChars.map(c => { + const isLowercase = 'a' <= c && c <= 'z'; + const isUppercase = 'A' <= c && c <= 'Z'; + if ((exp.category === 'Lu' && isUppercase) || (exp.category === 'Ll' && isLowercase)) { + return w.labelidx(asm.depthOf('innerSuccess')); + } + return w.labelidx(asm.depthOf('failure')); + }); + this.wrapTerminalLike(() => { + asm.block( + w.blocktype.empty, + () => { + asm.block( + w.blocktype.empty, + () => { + asm.currCharCode(); + asm.brTable(makeLabels(), w.labelidx(asm.depthOf('default'))); + }, + 'default', + ); + // Check for 0xff (end) + asm.currCharCode(); + asm.i32Const(0xff); + asm.i32Eq(); + asm.condBreak(asm.depthOf('failure')); + + // Otherwise, trap. + // TODO: Replace this with a proper, out-of-line implementation. + asm.emit(instr.unreachable); + }, + 'innerSuccess', + ); + asm.incPos(); + }); } } diff --git a/packages/wasm/src/ir.ts b/packages/wasm/src/ir.ts index 468abea5..8940e2b5 100644 --- a/packages/wasm/src/ir.ts +++ b/packages/wasm/src/ir.ts @@ -348,7 +348,7 @@ export function toString(exp: Expr): string { case 'Terminal': return JSON.stringify(exp.value); case 'UnicodeChar': - return `$unicodeChar<${JSON.stringify(exp.value)}>`; + return `$unicodeChar<${JSON.stringify(exp.category)}>`; case 'Dispatch': return `$dispatch`; // TODO: Improve this. case 'Lex': diff --git a/packages/wasm/test/_helpers.js b/packages/wasm/test/_helpers.js index 064008c1..6240ade1 100644 --- a/packages/wasm/test/_helpers.js +++ b/packages/wasm/test/_helpers.js @@ -1,4 +1,4 @@ -/* global process */ +/* global process, URL */ import {WasmMatcher} from '@ohm-js/miniohm-js'; @@ -29,3 +29,5 @@ export async function wasmMatcherForGrammar(grammar, modBytes = undefined) { } return m._instantiate(bytes, debugImports); } + +export const scriptRel = relPath => new URL(relPath, import.meta.url); diff --git a/packages/wasm/test/data/swatch.liquid b/packages/wasm/test/data/swatch.liquid index c8eb5cd1..c052d965 100644 --- a/packages/wasm/test/data/swatch.liquid +++ b/packages/wasm/test/data/swatch.liquid @@ -1,6 +1,32 @@ +{% comment %} + Renders a swatch component. + Accepts: + - swatch: {Object} a swatch object + - shape: {String} swatch shape. Accepts 'square', defaults to circle. + + Usage: + {% render 'swatch', + swatch: value.swatch + shape: 'square' + %} +{% endcomment %} + +{%- liquid + assign swatch_value = null + if swatch.image + assign image_url = swatch.image | image_url: width: 50 + assign swatch_value = 'url(' | append: image_url | append: ')' + assign swatch_focal_point = swatch.image.presentation.focal_point + elsif swatch.color + assign swatch_value = 'rgb(' | append: swatch.color.rgb | append: ')' + endif +-%} + diff --git a/packages/wasm/test/test-failurePos.js b/packages/wasm/test/test-failurePos.js new file mode 100644 index 00000000..919bba45 --- /dev/null +++ b/packages/wasm/test/test-failurePos.js @@ -0,0 +1,179 @@ +import test from 'ava'; +import assert from 'node:assert/strict'; +import fc from 'fast-check'; +import {readFileSync} from 'node:fs'; +import * as ohm from 'ohm-js'; + +import {scriptRel, wasmMatcherForGrammar} from './_helpers.js'; + +const grammarSource = readFileSync(scriptRel('data/liquid-html.ohm'), 'utf8'); +const ns = ohm.grammars(grammarSource); + +function failurePos(matcher, input) { + matcher.setInput(input); + const result = matcher.match(); + // TODO: Unify the APIs. + if (typeof result === 'number') { + assert.equal(result, 0); + return matcher.getRightmostFailurePosition(); + } + assert.equal(result.failed(), true); + return result.getRightmostFailurePosition(); +} + +/* eslint-disable max-len */ +const validInput = `{% comment %} + Renders a swatch component. + Accepts: + - swatch: {Object} a swatch object + - shape: {String} swatch shape. Accepts 'square', defaults to circle. + + Usage: + {% render 'swatch', + swatch: value.swatch + shape: 'square' + %} +{% endcomment %} + +{%- liquid + assign swatch_value = null + if swatch.image + assign image_url = swatch.image | image_url: width: 50 + assign swatch_value = 'url(' | append: image_url | append: ')' + assign swatch_focal_point = swatch.image.presentation.focal_point + elsif swatch.color + assign swatch_value = 'rgb(' | append: swatch.color.rgb | append: ')' + endif +-%} + +`; +/* eslint-enable max-len */ + +// Take some valid input, randomly corrupt it, and then check that the +// rightmostFailurePosition is the same as the JS parser reports. +const checkFailurePos = matcher => + fc.property( + fc.nat(), // Position to corrupt + fc.integer({min: 1, max: 20}), // Number of characters to corrupt + (posSeed, numChars) => { + const pos = posSeed % Math.max(1, validInput.length - numChars); + + // Remove a slice of random amount of characters + const newInput = validInput.slice(0, pos) + validInput.slice(pos + numChars); + + matcher.setInput(newInput); + fc.pre(matcher.match() === 0); + + return ( + matcher.getRightmostFailurePosition() === + ns.LiquidHTML.match(newInput).getRightmostFailurePosition() + ); + }, + ); + +// eslint-disable-next-line ava/no-skip-test +test('failure pos (fast-check)', async t => { + const m = await wasmMatcherForGrammar(ns.LiquidHTML); + t.notThrows(() => fc.assert(checkFailurePos(m), {numRuns: 50})); +}); + +test('failure pos: basic 1', async t => { + const g = ohm.grammar(` + G { + Start = number+ + number = digit+ + }`); + const jsMatcher = g.matcher(); + const wasmMatcher = await wasmMatcherForGrammar(g); + + t.is(failurePos(jsMatcher, 'a'), 0); + t.is(failurePos(wasmMatcher, 'a'), 0); + + t.is(failurePos(jsMatcher, '123a'), 3); + t.is(failurePos(wasmMatcher, '123a'), 3); + + t.is(failurePos(jsMatcher, '1 99a'), 4); + t.is(failurePos(wasmMatcher, '1 99a'), 4); +}); + +test('failure pos: basic 2', async t => { + const g = ohm.grammar(` + G { + Exp = number "+" number ";" -- plus + | number + number = digit+ + }`); + const jsMatcher = g.matcher(); + const wasmMatcher = await wasmMatcherForGrammar(g); + + t.is(failurePos(jsMatcher, '99 + 66'), 7); + t.is(failurePos(wasmMatcher, '99 + 66'), 7); +}); + +test('failure pos: basic 3', async t => { + const g = ohm.grammar(` + G { + Start = letter letter + space := "/*" (~"*/" any)* "*/" + }`); + const wasmMatcher = await wasmMatcherForGrammar(g); + + t.is(failurePos(wasmMatcher, '99'), 0); +}); + +test('failure pos: lookahead', async t => { + { + const g = ohm.grammar(` + G { + start = ~(anyTwo "!") "a" "b" + anyTwo = any any + }`); + const jsMatcher = g.matcher(); + const wasmMatcher = await wasmMatcherForGrammar(g); + + // Original Ohm behaviour is to ignore failures inside the lookahead, so + // it produces 'Expected "a"' at pos 0. + t.is(failurePos(jsMatcher, '99'), 0); + t.is(failurePos(wasmMatcher, '99'), 0); + } +}); + +test('failure pos: memoization', async t => { + { + const g = ohm.grammar(` + G { + start = ~anyTwo anyTwo + anyTwo = any any + }`); + const jsMatcher = g.matcher(); + const wasmMatcher = await wasmMatcherForGrammar(g); + + // Original Ohm behaviour is to ignore failures inside the lookahead, so + // it produces 'Expected "a"' at pos 0. + t.is(failurePos(jsMatcher, '9'), 1); + t.is(failurePos(wasmMatcher, '9'), 1); + } +}); + +test('failure pos: space skipping', async t => { + { + const g = ohm.grammar(` + G { + Start = digit digit + space += "/*" (~"*/" any)* "*/" -- comment + }`); + const jsMatcher = g.matcher(); + const wasmMatcher = await wasmMatcherForGrammar(g); + + // Failure inside space skipping should be ignored. + t.is(failurePos(jsMatcher, '9 /* bad'), 2); + t.is(failurePos(wasmMatcher, '9 /* bad'), 2); + } +}); diff --git a/packages/wasm/test/test-wasm.js b/packages/wasm/test/test-wasm.js index be399709..efd7a83c 100644 --- a/packages/wasm/test/test-wasm.js +++ b/packages/wasm/test/test-wasm.js @@ -506,8 +506,8 @@ test('real-world grammar', async t => { Msgs = Msg* Msg = description? spaces (Head spaces Params spaces) - lower := "a".."z" - upper := "A".."Z" + // Required until unicodeLtmo is implemented. + letter := lower | upper description = "#" (~nl any)* nl? Head = msgTarget spaces msgName @@ -848,8 +848,12 @@ test('specialized rule names', t => { compiler.normalize(); t.deepEqual([...compiler.rules.keys()].sort(), [ + '$spaces', + 'alnum', + 'any', 'commaSep', 'commaSep>', + 'digit', 'emptyListOf', 'emptyListOf,$term$1>', 'exclaimed', @@ -860,8 +864,10 @@ test('specialized rule names', t => { 'flip,hello>', 'hello', 'hello2', + 'letter', 'listOf', 'listOf,$term$1>', + 'lower', 'nonemptyListOf', 'nonemptyListOf,$term$1>', 'one', @@ -870,6 +876,8 @@ test('specialized rule names', t => { 'start', 'three', 'two', + 'unicodeLtmo', + 'upper', ]); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 66acc883..67a2af53 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -300,6 +300,9 @@ importers: esbuild: specifier: ^0.25.5 version: 0.25.5 + fast-check: + specifier: ^4.2.0 + version: 4.2.0 fast-glob: specifier: ^3.3.3 version: 3.3.3 @@ -1512,6 +1515,10 @@ packages: resolution: {integrity: sha512-11Ndz7Nv+mvAC1j0ktTa7fAb0vLyGGX+rMHNBYQviQDGU0Hw7lhctJANqbPhu9nV9/izT/IntTgZ7Im/9LJs9g==} engines: {'0': node >=0.6.0} + fast-check@4.2.0: + resolution: {integrity: sha512-buxrKEaSseOwFjt6K1REcGMeFOrb0wk3cXifeMAG8yahcE9kV20PjQn1OdzPGL6OBFTbYXfjleNBARf/aCfV1A==} + engines: {node: '>=12.17.0'} + fast-deep-equal@3.1.3: resolution: {integrity: sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==} @@ -2257,6 +2264,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@7.0.1: + resolution: {integrity: sha512-oTUZM/NAZS8p7ANR3SHh30kXB+zK2r2BPcEn/awJIbOvq82WoMN4p62AWWp3Hhw50G0xMsw1mhIBLqHw64EcNQ==} + qs@6.5.3: resolution: {integrity: sha512-qxXIEh4pCGfHICj1mAJQ2/2XVZkjCDTcEgfoSQxc/fYivUZxTkk7L3bDBJSoNrEzXI17oUO5Dp07ktqE5KzczA==} engines: {node: '>=0.6'} @@ -4038,6 +4048,10 @@ snapshots: extsprintf@1.3.0: {} + fast-check@4.2.0: + dependencies: + pure-rand: 7.0.1 + fast-deep-equal@3.1.3: {} fast-diff@1.3.0: {} @@ -4739,6 +4753,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@7.0.1: {} + qs@6.5.3: {} querystringify@2.2.0: {}