Skip to content

Commit 66c28b7

Browse files
pdubroypdubroy
authored
runtime: CST reader cleanups (get rid of non-packed version) (#606)
1 parent 0151629 commit 66c28b7

3 files changed

Lines changed: 334 additions & 203 deletions

File tree

‎packages/compiler/scripts/parseLiquid.js‎

Lines changed: 6 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ import {Bench} from 'tinybench';
1919
import {Grammar} from 'ohm-js';
2020
import {compileGrammars} from '../src/api.ts';
2121
import {unparse} from '../test/_helpers.js';
22-
import {createReader} from '../../runtime/src/cstReader.ts';
22+
import {createReader, CstNodeType} from '../../runtime/src/cstReader.ts';
2323

2424
const __dirname = dirname(fileURLToPath(import.meta.url));
2525
const datadir = join(__dirname, '../test/data');
@@ -36,7 +36,6 @@ const positionalArgs = process.argv.slice(2).filter(a => !a.startsWith('--'));
3636
const smallSize = flags.has('--small-size');
3737
const includeUnparse = flags.has('--include-unparse');
3838
const useCstReader = flags.has('--cst-reader');
39-
const useCstReaderPacked = flags.has('--cst-reader-packed');
4039

4140
// Get pattern from command line arguments
4241
const pattern = positionalArgs[0];
@@ -105,34 +104,12 @@ const pattern = positionalArgs[0];
105104
opts
106105
);
107106

108-
// Walk CST using CstReader (raw handles), collecting terminal text.
109-
function unparseCstReaderRaw(matchResult) {
107+
// Walk CST using CstReader, collecting terminal text.
108+
function unparseCstReader(matchResult) {
110109
const reader = createReader(matchResult);
111-
const inp = reader.input;
112-
let ans = '';
113-
function walk(handle, startIdx) {
114-
if (reader.isTerminal(handle)) {
115-
ans += inp.slice(startIdx, startIdx + reader.matchLength(handle));
116-
return;
117-
}
118-
reader.forEachChild(
119-
handle,
120-
(child, _leadingSpaces, offset) => {
121-
walk(child, startIdx + offset);
122-
},
123-
startIdx
124-
);
125-
}
126-
walk(reader.rootHandle, reader.rootStartIdx);
127-
return ans;
128-
}
129-
130-
// Walk CST using CstReader (handles with startIdx), collecting terminal text.
131-
function unparseCstReaderPacked(matchResult) {
132-
const reader = createReader(matchResult, {packStartIdx: true});
133110
let ans = '';
134111
function walk(handle) {
135-
if (reader.isTerminal(handle)) {
112+
if (reader.type(handle) === CstNodeType.TERMINAL) {
136113
ans += reader.sourceString(handle);
137114
return;
138115
}
@@ -146,11 +123,7 @@ const pattern = positionalArgs[0];
146123

147124
const wasmLabel = includeUnparse ? 'Wasm parse+unparse' : 'Wasm parse';
148125
bench.add(
149-
useCstReaderPacked
150-
? `${wasmLabel} (CstReader packed)`
151-
: useCstReader
152-
? `${wasmLabel} (CstReader)`
153-
: wasmLabel,
126+
useCstReader ? `${wasmLabel} (CstReader)` : wasmLabel,
154127
() => {
155128
let overriddenDuration = 0;
156129
for (const {input} of files) {
@@ -167,11 +140,7 @@ const pattern = positionalArgs[0];
167140
peakWasmMemoryBytes,
168141
exports.memory.buffer.byteLength
169142
);
170-
return useCstReaderPacked
171-
? unparseCstReaderPacked(m)
172-
: useCstReader
173-
? unparseCstReaderRaw(m)
174-
: unparse(g);
143+
return useCstReader ? unparseCstReader(m) : unparse(g);
175144
});
176145
if (includeUnparse) overriddenDuration += bench.now() - start;
177146
}

‎packages/compiler/test/test-cstReader.js‎

Lines changed: 274 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,274 @@
1+
import test from 'ava';
2+
3+
import {createReader, CstNodeType, NULL_HANDLE} from '../../runtime/src/cstReader.ts';
4+
import {compileAndLoad, matchWithInput} from './_helpers.js';
5+
6+
test('root node basics', async t => {
7+
const g = await compileAndLoad('G { start = "ab" "cd" }');
8+
t.is(matchWithInput(g, 'abcd'), 1);
9+
10+
g.match('abcd').use(mr => {
11+
const reader = createReader(mr);
12+
t.is(reader.type(reader.root), CstNodeType.NONTERMINAL);
13+
t.is(reader.matchLength(reader.root), 4);
14+
t.is(reader.ctorName(reader.root), 'start');
15+
t.is(reader.childCount(reader.root), 2);
16+
t.is(reader.input, 'abcd');
17+
t.is(reader.sourceString(reader.root), 'abcd');
18+
t.is(reader.startIdx(reader.root), 0);
19+
});
20+
});
21+
22+
test('terminal children', async t => {
23+
const g = await compileAndLoad('G { start = "ab" "cd" }');
24+
g.match('abcd').use(mr => {
25+
const reader = createReader(mr);
26+
const children = [];
27+
reader.forEachChild(reader.root, (child, leadingSpaces, startIdx, index) => {
28+
children.push({child, leadingSpaces, startIdx, index});
29+
});
30+
t.is(children.length, 2);
31+
32+
// First child: "ab"
33+
t.is(reader.type(children[0].child), CstNodeType.TERMINAL);
34+
t.is(reader.matchLength(children[0].child), 2);
35+
t.is(reader.ctorName(children[0].child), '_terminal');
36+
t.is(reader.sourceString(children[0].child), 'ab');
37+
t.is(children[0].leadingSpaces, NULL_HANDLE);
38+
t.is(children[0].index, 0);
39+
40+
// Second child: "cd"
41+
t.is(reader.type(children[1].child), CstNodeType.TERMINAL);
42+
t.is(reader.matchLength(children[1].child), 2);
43+
t.is(reader.sourceString(children[1].child), 'cd');
44+
t.is(children[1].index, 1);
45+
});
46+
});
47+
48+
test('nonterminal children', async t => {
49+
const g = await compileAndLoad('G { start = a b\na = "x"\nb = "y" }');
50+
g.match('xy').use(mr => {
51+
const reader = createReader(mr);
52+
const children = [];
53+
reader.forEachChild(reader.root, (child, ls, startIdx, i) => {
54+
children.push({child, ls, startIdx, i});
55+
});
56+
t.is(children.length, 2);
57+
t.is(reader.ctorName(children[0].child), 'a');
58+
t.is(reader.ctorName(children[1].child), 'b');
59+
t.is(reader.type(children[0].child), CstNodeType.NONTERMINAL);
60+
t.is(reader.type(children[1].child), CstNodeType.NONTERMINAL);
61+
t.is(reader.sourceString(children[0].child), 'x');
62+
t.is(reader.sourceString(children[1].child), 'y');
63+
});
64+
});
65+
66+
test('iteration (list) node', async t => {
67+
const g = await compileAndLoad('G { start = "a"* }');
68+
g.match('aaa').use(mr => {
69+
const reader = createReader(mr);
70+
let listHandle;
71+
reader.forEachChild(reader.root, child => {
72+
listHandle = child;
73+
});
74+
t.is(reader.type(listHandle), CstNodeType.LIST);
75+
t.is(reader.ctorName(listHandle), '_list');
76+
t.is(reader.childCount(listHandle), 3);
77+
78+
const items = [];
79+
reader.forEachChild(listHandle, child => {
80+
items.push(reader.sourceString(child));
81+
});
82+
t.deepEqual(items, ['a', 'a', 'a']);
83+
});
84+
});
85+
86+
test('iteration with nonterminals', async t => {
87+
const g = await compileAndLoad('G { start = letter* }');
88+
g.match('abc').use(mr => {
89+
const reader = createReader(mr);
90+
let listHandle;
91+
reader.forEachChild(reader.root, child => {
92+
listHandle = child;
93+
});
94+
t.is(reader.type(listHandle), CstNodeType.LIST);
95+
const items = [];
96+
reader.forEachChild(listHandle, child => {
97+
items.push(reader.sourceString(child));
98+
});
99+
t.is(items.length, 3);
100+
t.deepEqual(items, ['a', 'b', 'c']);
101+
});
102+
});
103+
104+
test('optional node: present', async t => {
105+
const g = await compileAndLoad('G { start = "a"? }');
106+
g.match('a').use(mr => {
107+
const reader = createReader(mr);
108+
let opt;
109+
reader.forEachChild(reader.root, child => {
110+
opt = child;
111+
});
112+
t.is(reader.type(opt), CstNodeType.OPT);
113+
t.is(reader.ctorName(opt), '_opt');
114+
t.is(reader.childCount(opt), 1);
115+
t.is(reader.matchLength(opt), 1);
116+
});
117+
});
118+
119+
test('optional node: absent', async t => {
120+
const g = await compileAndLoad('G { start = "a"? }');
121+
g.match('').use(mr => {
122+
const reader = createReader(mr);
123+
let opt;
124+
reader.forEachChild(reader.root, child => {
125+
opt = child;
126+
});
127+
t.is(reader.type(opt), CstNodeType.OPT);
128+
t.is(reader.childCount(opt), 0);
129+
t.is(reader.matchLength(opt), 0);
130+
});
131+
});
132+
133+
// --- unparse via walk ---
134+
135+
test('unparse: simple terminals', async t => {
136+
const g = await compileAndLoad('G { start = "ab" "cd" }');
137+
g.match('abcd').use(mr => {
138+
const reader = createReader(mr);
139+
let ans = '';
140+
function walk(handle) {
141+
if (reader.type(handle) === CstNodeType.TERMINAL) {
142+
ans += reader.sourceString(handle);
143+
return;
144+
}
145+
reader.forEachChild(handle, child => walk(child));
146+
}
147+
walk(reader.root);
148+
t.is(ans, 'abcd');
149+
});
150+
});
151+
152+
test('unparse: with rule application', async t => {
153+
const g = await compileAndLoad('G { start = a b\na = "x"\nb = "y" }');
154+
g.match('xy').use(mr => {
155+
const reader = createReader(mr);
156+
let ans = '';
157+
function walk(handle) {
158+
if (reader.type(handle) === CstNodeType.TERMINAL) {
159+
ans += reader.sourceString(handle);
160+
return;
161+
}
162+
reader.forEachChild(handle, child => walk(child));
163+
}
164+
walk(reader.root);
165+
t.is(ans, 'xy');
166+
});
167+
});
168+
169+
test('unparse: with nonterminals', async t => {
170+
const g = await compileAndLoad('G { start = a b\na = "hello"\nb = "world" }');
171+
g.match('helloworld').use(mr => {
172+
const reader = createReader(mr);
173+
let ans = '';
174+
function walk(handle) {
175+
if (reader.type(handle) === CstNodeType.TERMINAL) {
176+
ans += reader.sourceString(handle);
177+
return;
178+
}
179+
reader.forEachChild(handle, child => walk(child));
180+
}
181+
walk(reader.root);
182+
t.is(ans, 'helloworld');
183+
});
184+
});
185+
186+
test('unparse: unicode', async t => {
187+
const g = await compileAndLoad('G { start = any* }');
188+
const input = 'Nöö';
189+
g.match(input).use(mr => {
190+
const reader = createReader(mr);
191+
let ans = '';
192+
function walk(handle) {
193+
if (reader.type(handle) === CstNodeType.TERMINAL) {
194+
ans += reader.sourceString(handle);
195+
return;
196+
}
197+
reader.forEachChild(handle, child => walk(child));
198+
}
199+
walk(reader.root);
200+
t.is(ans, input);
201+
});
202+
});
203+
204+
// --- leading spaces ---
205+
206+
test('rootLeadingSpaces: present', async t => {
207+
const g = await compileAndLoad('G { Start = "x" }');
208+
g.match(' x').use(mr => {
209+
const reader = createReader(mr);
210+
t.not(reader.rootLeadingSpaces, NULL_HANDLE);
211+
t.is(reader.matchLength(reader.rootLeadingSpaces), 2);
212+
t.is(reader.ctorName(reader.rootLeadingSpaces), 'spaces');
213+
t.is(reader.type(reader.rootLeadingSpaces), CstNodeType.NONTERMINAL);
214+
t.is(reader.sourceString(reader.rootLeadingSpaces), ' ');
215+
});
216+
});
217+
218+
test('rootLeadingSpaces: absent', async t => {
219+
const g = await compileAndLoad('G { Start = "x" }');
220+
g.match('x').use(mr => {
221+
const reader = createReader(mr);
222+
t.is(reader.rootLeadingSpaces, NULL_HANDLE);
223+
});
224+
});
225+
226+
test('child leadingSpaces in syntactic rule', async t => {
227+
const g = await compileAndLoad('G { Start = "a" "b" }');
228+
g.match('a b').use(mr => {
229+
const reader = createReader(mr);
230+
const spacesInfo = [];
231+
reader.forEachChild(reader.root, (child, leadingSpaces, startIdx, index) => {
232+
spacesInfo.push({
233+
index,
234+
hasSpaces: leadingSpaces !== NULL_HANDLE,
235+
spacesLen: leadingSpaces !== NULL_HANDLE ? reader.matchLength(leadingSpaces) : 0,
236+
spacesStr: leadingSpaces !== NULL_HANDLE ? reader.sourceString(leadingSpaces) : '',
237+
});
238+
});
239+
t.is(spacesInfo.length, 2);
240+
// "a" has no leading spaces
241+
t.false(spacesInfo[0].hasSpaces);
242+
// "b" has 1 leading space
243+
t.true(spacesInfo[1].hasSpaces);
244+
t.is(spacesInfo[1].spacesLen, 1);
245+
t.is(spacesInfo[1].spacesStr, ' ');
246+
});
247+
});
248+
249+
// --- details ---
250+
251+
test('details returns ruleId for nonterminals', async t => {
252+
const g = await compileAndLoad('G { start = a\na = "x" }');
253+
g.match('x').use(mr => {
254+
const reader = createReader(mr);
255+
// Root is 'start', details should be its ruleId (>= 0).
256+
const d = reader.details(reader.root);
257+
t.true(d >= 0);
258+
});
259+
});
260+
261+
// --- edge cases ---
262+
263+
test('childCount is 0 for tagged terminals', async t => {
264+
const g = await compileAndLoad('G { start = "x" }');
265+
g.match('x').use(mr => {
266+
const reader = createReader(mr);
267+
let termChild;
268+
reader.forEachChild(reader.root, child => {
269+
termChild = child;
270+
});
271+
t.is(reader.type(termChild), CstNodeType.TERMINAL);
272+
t.is(reader.childCount(termChild), 0);
273+
});
274+
});

0 commit comments

Comments
 (0)