Skip to content

Commit 853f115

Browse files
johnsoncodehkcursoragent
johnsoncodehk
and
cursoragent
committed
Fix sepBy and FIRST-alt over-accept via checkpoint restore in emitted parsers.
Restore CST state when sep/predAlt arms fail so incomplete pairs and related toy witnesses reject consistently across TS, Go, and Rust; add 25 fixed parity witnesses and Go EntryMeta for non-reuse emit. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent e8ab178 commit 853f115

6 files changed

Lines changed: 137 additions & 20 deletions

File tree

‎src/target-go.ts‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -540,7 +540,11 @@ function stepCond(s: Step, ids: LexIdPlan, ar: ArenaIdPlan): string {
540540
}
541541

542542
function predAltBody(branches: Step[][], ids: LexIdPlan, firsts: FirstSig[] | undefined, ar: ArenaIdPlan): string {
543-
const arms = branches.map((br, i) => `if ${firstCond(firsts![i], 't', ids)} { if ${br.length ? br.map((x) => stepCond(x, ids, ar)).join(' && ') : 'true'} { return true } }`).join(' else ');
543+
// FIRST dispatch still only tries the matching arm; on half-failure restore like non-pred alt.
544+
const arms = branches.map((br, i) => {
545+
const steps = br.length ? br.map((x) => stepCond(x, ids, ar)).join(' && ') : 'true';
546+
return `if ${firstCond(firsts![i], 't', ids)} { save := pos; sb := len(scratch); nb := len(nodes); kb := len(kids); if ${steps} { return true }; pos = save; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb] }`;
547+
}).join(' else ');
544548
return `t := peek(); if t == nil { return false }; ${arms}; return false`;
545549
}
546550

@@ -1895,7 +1899,11 @@ function notBodyW(steps: Step[], ids: LexIdPlan, ar: ArenaIdPlan, recv: string):
18951899
}
18961900
function predAltBodyW(branches: Step[][], ids: LexIdPlan, ar: ArenaIdPlan, firsts: FirstSig[] | undefined, recv: string): string {
18971901
// Same-line `} else if` (native predAltBody joins with ' else '); newlines break Go ASI.
1898-
const arms = branches.map((br, i) => `if ${firstCond(firsts![i], 't', ids)} { if ${br.length ? br.map((x) => stepCondW(x, ids, ar, recv)).join(' && ') : 'true'} { return true } }`).join(' else ');
1902+
// FIRST dispatch; restore pos/scratch/arena on arm half-failure (like altBodyW).
1903+
const arms = branches.map((br, i) => {
1904+
const steps = br.length ? br.map((x) => stepCondW(x, ids, ar, recv)).join(' && ') : 'true';
1905+
return `if ${firstCond(firsts![i], 't', ids)} { sp := p.pos; sb := len(p.scratch); ${arenaCkpt(recv)}; if ${steps} { return true }; p.pos = sp; p.scratch = p.scratch[:sb]; ${arenaRestore(recv)} }`;
1906+
}).join(' else ');
18991907
return `t := p.peekW(); if t == nil { return false }; ${arms}; return false`;
19001908
}
19011909

@@ -2322,11 +2330,13 @@ func (p *${recv}) optW(body func() bool) bool {
23222330
\treturn true
23232331
}
23242332
func (p *${recv}) sepByW(elem func() bool, delim uint16) bool {
2325-
\tif !elem() { return true }
2333+
\tsp0 := p.pos; sb0 := len(p.scratch); ${arenaCkpt(recv).replace('nb, kb', 'nb0, kb0')}
2334+
\tif !elem() { p.pos = sp0; p.scratch = p.scratch[:sb0]; ${arenaRestore(recv).replace('nb', 'nb0').replace('kb', 'kb0')}; return true }
23262335
\tfor {
23272336
\t\tsp := p.pos; sb := len(p.scratch); ${arenaCkpt(recv)}
23282337
\t\tif !p.matchLitW(delim, TT_SKIP_PUNCT) { p.pos = sp; p.scratch = p.scratch[:sb]; ${arenaRestore(recv)}; break }
2329-
\t\tif !elem() { break }
2338+
\t\tsp2 := p.pos; sb2 := len(p.scratch); ${arenaCkpt(recv).replace('nb, kb', 'nb2, kb2')}
2339+
\t\tif !elem() { p.pos = sp2; p.scratch = p.scratch[:sb2]; ${arenaRestore(recv).replace('nb', 'nb2').replace('kb', 'kb2')}; break }
23302340
\t}
23312341
\treturn true
23322342
}
@@ -2398,8 +2408,13 @@ func (b *CstBuilder) Shift(h int32, byteDelta, tokDelta int) int32 {
23982408
}
23992409
`;
24002410

2411+
// EntryMeta is part of parseWithMetaW's signature always; reuse paths define it via
2412+
// rdEntryWithReuse{A,B}. Non-reuse grammars (e.g. toy) still emit DocWith and need the type.
2413+
const entryMetaType = topReuse ? '' : `type EntryMeta struct { TokStart, TokEnd, Ext, Off, End, KidStart, KidCount int }
2414+
`;
2415+
24012416
const parseWithMetaW = `
2402-
func parseWithMetaW(text string, meta []alignMeta, b Builder) (h int32, entries []EntryMeta, entryHs []int32, ok bool) {
2417+
${entryMetaType}func parseWithMetaW(text string, meta []alignMeta, b Builder) (h int32, entries []EntryMeta, entryHs []int32, ok bool) {
24032418
toks := toksFromMeta(text, meta)
24042419
n := len(toks)
24052420
p := &parserW{toks: toks, src: text, b: b, scratch: nil}
@@ -2905,7 +2920,7 @@ func NewDocWith(src string, b Builder) Document {
29052920
func newDocWith(src string, b Builder) *DocWith {
29062921
d := &DocWith{text: src, b: b}
29072922
d.toks = ${initToks}
2908-
h, entries, entryHs, ok := parseWithMetaW(src, d.toks, b)
2923+
${topReuse ? 'h, entries, entryHs, ok' : 'h, _, _, ok'} := parseWithMetaW(src, d.toks, b)
29092924
if ok {
29102925
${initAssign}
29112926
} else {
@@ -2937,7 +2952,7 @@ ${recoverClear}
29372952
}
29382953
}()
29392954
d.toks = ${freshMeta}
2940-
h, ne, nhs, ok := parseWithMetaW(d.text, d.toks, d.b)
2955+
${topReuse ? 'h, ne, nhs, ok' : 'h, _, _, ok'} := parseWithMetaW(d.text, d.toks, d.b)
29412956
if ok {
29422957
${recoverAssign}
29432958
} else {
@@ -3444,11 +3459,13 @@ func opt(body func() bool) bool {
34443459
\tsp := pos; sb := len(scratch); nb := len(nodes); kb := len(kids); if !body() { pos = sp; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb] }; return true
34453460
}
34463461
func sepBy(elem func() bool, delimLid uint16) bool {
3447-
\tif !elem() { return true } // the whole separated list is optional — zero elements is valid
3462+
\tsp0 := pos; sb0 := len(scratch); nb0 := len(nodes); kb0 := len(kids)
3463+
\tif !elem() { pos = sp0; scratch = scratch[:sb0]; nodes = nodes[:nb0]; kids = kids[:kb0]; return true }
34483464
\tfor {
34493465
\t\tsp := pos; sb := len(scratch); nb := len(nodes); kb := len(kids)
34503466
\t\tif !matchLit(delimLid, TT_SKIP_PUNCT) { pos = sp; scratch = scratch[:sb]; nodes = nodes[:nb]; kids = kids[:kb]; break }
3451-
\t\tif !elem() { break } // a trailing delimiter is allowed — keep the pushed delim and stop
3467+
\t\tsp2 := pos; sb2 := len(scratch); nb2 := len(nodes); kb2 := len(kids)
3468+
\t\tif !elem() { pos = sp2; scratch = scratch[:sb2]; nodes = nodes[:nb2]; kids = kids[:kb2]; break } // trailing delim OK — keep delim
34523469
\t}
34533470
\treturn true
34543471
}

‎src/target-rust.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -738,7 +738,11 @@ function notBody(steps: Step[], ids: LexIdPlan, ar: ArenaIdPlan): string {
738738
return `let sp = p.pos; let sb = p.scratch.len(); let ck = p.b.checkpoint(); let m = ${steps.length ? steps.map((x) => stepCondP(x, ids, ar)).join(' && ') : 'true'}; p.pos = sp; p.scratch.truncate(sb); p.b.restore(ck); !m`;
739739
}
740740
function predAltBody(branches: Step[][], ids: LexIdPlan, ar: ArenaIdPlan, firsts?: FirstSig[]): string {
741-
const arms = branches.map((br, i) => ` ${i === 0 ? 'if' : 'else if'} ${firstCond(firsts![i], 't', ids)} { if ${br.length ? br.map((x) => stepCondP(x, ids, ar)).join(' && ') : 'true'} { return true; } }`).join('\n');
741+
// FIRST dispatch; restore pos/scratch/builder on arm half-failure (like altBody).
742+
const arms = branches.map((br, i) => {
743+
const steps = br.length ? br.map((x) => stepCondP(x, ids, ar)).join(' && ') : 'true';
744+
return ` ${i === 0 ? 'if' : 'else if'} ${firstCond(firsts![i], 't', ids)} { let sp = p.pos; let sb = p.scratch.len(); let ck = p.b.checkpoint(); if ${steps} { return true; } p.pos = sp; p.scratch.truncate(sb); p.b.restore(ck); }`;
745+
}).join('\n');
742746
return `let t = match p.peek() { Some(t) => t, None => return false };\n${arms}\n false`;
743747
}
744748

@@ -2613,11 +2617,13 @@ impl<'a, B: Builder> Parser<'a, B> {
26132617
}
26142618
#[inline(always)]
26152619
fn sep_by(&mut self, elem: fn(&mut Parser<'a, B>) -> bool, delim: u16) -> bool {
2616-
if !elem(self) { return true; }
2620+
let sp0 = self.pos; let sb0 = self.scratch.len(); let ck0 = self.b.checkpoint();
2621+
if !elem(self) { self.pos = sp0; self.scratch.truncate(sb0); self.b.restore(ck0); return true; }
26172622
loop {
26182623
let sp = self.pos; let sb = self.scratch.len(); let ck = self.b.checkpoint();
26192624
if !self.match_lit(delim, ${punctId}) { self.pos = sp; self.scratch.truncate(sb); self.b.restore(ck); break; }
2620-
if !elem(self) { break; }
2625+
let sp2 = self.pos; let sb2 = self.scratch.len(); let ck2 = self.b.checkpoint();
2626+
if !elem(self) { self.pos = sp2; self.scratch.truncate(sb2); self.b.restore(ck2); break; }
26212627
}
26222628
true
26232629
}

‎src/target-ts.ts‎

Lines changed: 18 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -523,7 +523,14 @@ function stepCond(s: Step, ids: LexIdPlan, w = false): string {
523523

524524
function predAltBody(branches: Step[][], ids: LexIdPlan, firsts?: FirstSig[], w = false): string {
525525
const sc = (x: Step) => stepCond(x, ids, w);
526-
const arms = branches.map((br, i) => `if (${firstCond(firsts![i], 't', ids)}) { if (${br.length ? br.map(sc).join(' && ') : 'true'}) return true; }`).join(' else ');
526+
// FIRST dispatch still only tries the matching arm; on half-failure restore like non-pred altBody.
527+
const arms = branches.map((br, i) => {
528+
const steps = br.length ? br.map(sc).join(' && ') : 'true';
529+
const body = w
530+
? `{ const sp = pos; const bk = kids.length; const bs = spans.length; if (${steps}) return true; pos = sp; kids.length = bk; spans.length = bs; }`
531+
: `{ const sp = pos; const bk = kids.length; if (${steps}) return true; pos = sp; kids.length = bk; }`;
532+
return `if (${firstCond(firsts![i], 't', ids)}) ${body}`;
533+
}).join(' else ');
527534
return `const t = peek(); if (t === null) return false; ${arms} return false;`;
528535
}
529536

@@ -1176,11 +1183,14 @@ function optW(body: () => boolean, kids: any[], spans: BWSpan[]): boolean {
11761183
const sp = pos; const before = kids.length; const bs = spans.length; if (!body()) { pos = sp; kids.length = before; spans.length = bs; } return true;
11771184
}
11781185
function sepByW(elem: () => boolean, delimLid: number, kids: any[], spans: BWSpan[]): boolean {
1179-
if (!elem()) return true;
1186+
// Match interpreter matchSep: restore on elem failure (zero elems OK; trailing delim keeps delim).
1187+
const sp0 = pos; const before0 = kids.length; const bs0 = spans.length;
1188+
if (!elem()) { pos = sp0; kids.length = before0; spans.length = bs0; return true; }
11801189
for (;;) {
11811190
const sp = pos; const before = kids.length; const bs = spans.length;
11821191
if (!matchLitW(delimLid, '$punct', kids, spans)) { pos = sp; kids.length = before; spans.length = bs; break; }
1183-
if (!elem()) break;
1192+
const sp2 = pos; const before2 = kids.length; const bs2 = spans.length;
1193+
if (!elem()) { pos = sp2; kids.length = before2; spans.length = bs2; break; }
11841194
}
11851195
return true;
11861196
}
@@ -3881,11 +3891,14 @@ function opt(body: () => boolean, kids: Cst[]): boolean {
38813891
const sp = pos; const before = kids.length; if (!body()) { pos = sp; kids.length = before; } return true;
38823892
}
38833893
function sepBy(elem: () => boolean, delimLid: number, kids: Cst[]): boolean {
3884-
if (!elem()) return true; // the whole separated list is optional — zero elements is valid
3894+
// Match interpreter matchSep: restore on elem failure (zero elems OK; trailing delim keeps delim).
3895+
const sp0 = pos; const before0 = kids.length;
3896+
if (!elem()) { pos = sp0; kids.length = before0; return true; }
38853897
for (;;) {
38863898
const sp = pos; const before = kids.length;
38873899
if (!matchLit(delimLid, '$punct', kids)) { pos = sp; kids.length = before; break; }
3888-
if (!elem()) break; // a trailing delimiter is allowed — keep the pushed delim and stop
3900+
const sp2 = pos; const before2 = kids.length;
3901+
if (!elem()) { pos = sp2; kids.length = before2; break; } // trailing delim OK — keep delim, drop elem pollution
38893902
}
38903903
return true;
38913904
}

‎test/fixtures/shape-toy.ts‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -600,8 +600,8 @@ export function buildToyCorpus(seed = 0x5a2_2026): { src: string; source: string
600600
const n = Math.floor(rng() * 5);
601601
return `maybe${n ? ' ' + Array.from({ length: n }, atom).join(' ') : ''}`;
602602
}
603-
// SH3-1b: sep(alt([Ident,':',Number], Number)) — include multi-pair + trailing
604-
// delim; incomplete `id:` forms are CST-over-accept (see SH3-1b reply), not emitted.
603+
// SH3-1b/cst-fix: sep(alt([Ident,':',Number], Number)) — multi-pair + trailing delim.
604+
// Incomplete `id:` forms belong in invalidProgram / reject anchors (CST now restores).
605605
if (r < .90) {
606606
const n = Math.floor(rng() * 4);
607607
const pairs = Array.from({ length: n }, () => {
@@ -631,6 +631,9 @@ export function buildToyCorpus(seed = 0x5a2_2026): { src: string; source: string
631631
'tag x', 'bang !!x', 'unknown unknown;', 'guard ;', 'args(1 2);',
632632
'txn a:b;', 'txn ?', 'line a\nb;', 'line a;', 'noplus 1+;', 'repeat +;',
633633
'pairs(a::1);', 'pairs(,);', 'notany bad;', 'notany worse x;',
634+
// incomplete sep+alt (predAlt half-fail must not over-accept)
635+
'pairs (a : );', 'pairs (1, a : );', 'pairs(a:);', 'pairs(1,a:);',
636+
'pairs(a:1,b:);', 'pairs(a : );', 'pairs(1, a:);',
634637
]);
635638
}
636639
// Planner adversarial 112 cases (depth groups + fragment cross-product).
@@ -656,8 +659,11 @@ export function buildToyCorpus(seed = 0x5a2_2026): { src: string; source: string
656659
// SH3-1b: suppress must not block prec-binary `*` (LED-only exclude)
657660
'noplus 1 * 2;', 'noplus 1 * 2 * 3;', 'noplus (1*2);', 'noplus 1*2;',
658661
'noplus 1/2;', 'noplus 1*2+3;', 'noplus (1*2)*3;', 'noplus 1*(2*3);',
659-
// SH3-1b: well-formed sep+alt (incomplete `pairs(a:)` is CST-over-accept — not here)
662+
// SH3-1b: well-formed sep+alt
660663
'pairs(a:1);', 'pairs(1, a:2);', 'pairs(a:1, 2, b:3);', 'pairs( a : 1 , );',
664+
// cst-fix: incomplete sep+alt must REJECT (predAlt/sepBy restore)
665+
'pairs (a : );', 'pairs (1, a : );', 'pairs(a:);', 'pairs(1,a:);',
666+
'pairs(a:1,b:);', 'pairs(a : );', 'pairs(1, a:);',
661667
// SH2-0b: choice-arm nested groups + multi-stmt
662668
'tag x=(1);', 'tag x=((1));', 'tag y=(((2)));', 'tag z:(3);', 'tag z:((a));',
663669
'tag x:1;tag y=2;', 'bang!x;tag z;', 'tag x=(1);tag y=2;tag z;',

‎test/shape-parity.ts‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1018,6 +1018,40 @@ async function main(): Promise<void> {
10181018
);
10191019
}
10201020

1021+
const cstFixSepAltWitnesses: { src: string; want: 'accept' | 'reject' }[] = [
1022+
{ src: "pairs (a : );", want: 'reject' },
1023+
{ src: "pairs (1, a : );", want: 'reject' },
1024+
{ src: "pairs(a:);", want: 'reject' },
1025+
{ src: "pairs(1,a:);", want: 'reject' },
1026+
{ src: "pairs(a:1,b:);", want: 'reject' },
1027+
{ src: "pairs(a : );", want: 'reject' },
1028+
{ src: "pairs(1, a:);", want: 'reject' },
1029+
{ src: "notany bad;", want: 'reject' },
1030+
{ src: "txn a:b;", want: 'reject' },
1031+
{ src: 'line a\nb;', want: 'reject' },
1032+
{ src: "args(1,,2);", want: 'reject' },
1033+
{ src: "pairs();", want: 'accept' },
1034+
{ src: "pairs(a:1);", want: 'accept' },
1035+
{ src: "pairs(1, a:2);", want: 'accept' },
1036+
{ src: "pairs(a:1,2,b:3,);", want: 'accept' },
1037+
{ src: "pairs( a : 1 , );", want: 'accept' },
1038+
{ src: "maybe;", want: 'accept' },
1039+
{ src: "maybe a 1;", want: 'accept' },
1040+
{ src: "repeat a 1 b 2;", want: 'accept' },
1041+
{ src: "notany good;", want: 'accept' },
1042+
{ src: "txn a:b?;", want: 'accept' },
1043+
{ src: "noplus 1 * 2;", want: 'accept' },
1044+
{ src: "line a b;", want: 'accept' },
1045+
{ src: "args();", want: 'accept' },
1046+
{ src: "args(1,);", want: 'accept' },
1047+
];
1048+
for (const { src, want } of cstFixSepAltWitnesses) {
1049+
const cst = accepts(toyMod, src, false);
1050+
const ast = accepts(toyMod, src, true);
1051+
const ok = want === 'accept' ? cst && ast : !cst && !ast;
1052+
check(ok, `cst-fix sepAlt witness ${want} ${JSON.stringify(src)} (cst=${cst} ast=${ast})`);
1053+
}
1054+
10211055
// ── Guard + capped witnesses (toy) ────────────────────────────────────────
10221056
check(
10231057
accepts(toyMod, 'a::b;', false) && accepts(toyMod, 'a::b;', true),

‎test/shape-rust.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,47 @@ async function main(): Promise<void> {
304304
}
305305
check('SH3-1b noplus suppress/binary TS↔Rust', sh31bBad === 0, `${sh31bNoplus.length - sh31bBad}/${sh31bNoplus.length}`);
306306

307+
const cstFixSepAltWitnesses: { src: string; want: 'accept' | 'reject' }[] = [
308+
{ src: "pairs (a : );", want: 'reject' },
309+
{ src: "pairs (1, a : );", want: 'reject' },
310+
{ src: "pairs(a:);", want: 'reject' },
311+
{ src: "pairs(1,a:);", want: 'reject' },
312+
{ src: "pairs(a:1,b:);", want: 'reject' },
313+
{ src: "pairs(a : );", want: 'reject' },
314+
{ src: "pairs(1, a:);", want: 'reject' },
315+
{ src: "notany bad;", want: 'reject' },
316+
{ src: "txn a:b;", want: 'reject' },
317+
{ src: 'line a\nb;', want: 'reject' },
318+
{ src: "args(1,,2);", want: 'reject' },
319+
{ src: "pairs();", want: 'accept' },
320+
{ src: "pairs(a:1);", want: 'accept' },
321+
{ src: "pairs(1, a:2);", want: 'accept' },
322+
{ src: "pairs(a:1,2,b:3,);", want: 'accept' },
323+
{ src: "pairs( a : 1 , );", want: 'accept' },
324+
{ src: "maybe;", want: 'accept' },
325+
{ src: "maybe a 1;", want: 'accept' },
326+
{ src: "repeat a 1 b 2;", want: 'accept' },
327+
{ src: "notany good;", want: 'accept' },
328+
{ src: "txn a:b?;", want: 'accept' },
329+
{ src: "noplus 1 * 2;", want: 'accept' },
330+
{ src: "line a b;", want: 'accept' },
331+
{ src: "args();", want: 'accept' },
332+
{ src: "args(1,);", want: 'accept' },
333+
];
334+
const cstFixWitnessSrcs = cstFixSepAltWitnesses.map((w) => w.src);
335+
const cstFixLines = runBatch(toyBin, cstFixWitnessSrcs);
336+
let cstFixBad = 0;
337+
for (let i = 0; i < cstFixSepAltWitnesses.length; i++) {
338+
const { src, want } = cstFixSepAltWitnesses[i]!;
339+
const tsCst = toyTs.parse(toyTs.tokenize(src)) !== null;
340+
const tsAst = toyTs.parseAst(src) !== null;
341+
const rustOk = cstFixLines[i]!.startsWith('A ');
342+
const ok = want === 'accept' ? tsCst && tsAst && rustOk : !tsCst && !tsAst && !rustOk;
343+
if (!ok) cstFixBad++;
344+
}
345+
check('cst-fix sepAlt witnesses TS↔Rust', cstFixBad === 0, `${cstFixSepAltWitnesses.length - cstFixBad}/${cstFixSepAltWitnesses.length}`);
346+
347+
307348
let failFast = '';
308349
try {
309350
emitRust(typescriptGrammar, { shape: typescriptShape });

0 commit comments

Comments
 (0)