SH2-0b: fix CST↔AST false divergences; harden toy corpus ≥1200.

Co-authored-by: Cursor <cursoragent@cursor.com>

Author: johnsoncodehk

src/shape-schema.ts

@@ -32,6 +32,8 @@ export type LeafValueShape = {
 };
 export type CustomShape = { kind: 'custom'; fn: string; reason: string };
 export type KeepShape = { kind: 'keep' };
+/** Delegate Pratt atom NUD to an RD rule (e.g. Atom choice → Number|Identifier nodes). */
+export type RuleRefShape = { kind: 'rule'; name: string };
 
 export type ChoiceArm = {
   name: string;
@@ -42,7 +44,7 @@ export type ChoiceShape = { kind: 'choice'; arms: ChoiceArm[] };
 
 export type PrattShape = {
   kind: 'pratt';
-  atom?: LeafValueShape | KeepShape | DropShape | CustomShape;
+  atom?: LeafValueShape | KeepShape | DropShape | CustomShape | RuleRefShape;
   group?: InlineShape | NodeShape | CustomShape;
   nudSeq?: RuleShapeAtom;
   nudCapped?: RuleShapeAtom;

src/target-ts.ts

@@ -2494,8 +2494,12 @@ function emitShapeTypeDecls(ir: ParserIR, shapeIR: ShapeIR): string {
       if (r.name === ir.entry) lines.push(`export type AstRoot = ${name};`);
     } else if (r.shape.kind === 'pratt') {
       const members: string[] = [];
-      const add = (s: RuleShape | undefined) => {
+      const add = (s: RuleShape | { kind: 'rule'; name: string } | undefined) => {
         if (!s) return;
+        if (s.kind === 'rule') {
+          members.push(`${s.name}Shape`);
+          return;
+        }
         if (s.kind === 'node') members.push(s.type);
         else if (s.kind === 'leafValue') {
           const ts = s.fn === 'number' ? 'number' : s.fn === 'bigint' ? 'bigint' : s.fn === 'boolean' ? 'boolean' : 'string';
@@ -3044,7 +3048,12 @@ function emitAstPrattRule(r: PrattRule, sir: ShapeIRRule, ids: LexIdPlan, shapeI
   // ── atom ──────────────────────────────────────────────────────────────────
   ctx.pratt.atom++;
   let atomCode = '';
-  if (ps.atom?.kind === 'custom') {
+  if (ps.atom?.kind === 'rule') {
+    // Delegate to an RD rule that covers the same atom tokens (no runtime custom).
+    atomCode = `  if (${r.name}_ATOM.has(t.kid)) {
+    return parseAst${ps.atom.name}() as ${retType};
+  }`;
+  } else if (ps.atom?.kind === 'custom') {
     atomCode = `  if (${r.name}_ATOM.has(t.kid)) {
     const save = pos; const spOff = t.off; pos++;
     const _lv = t.kid === ${kidOf(ids, 'Number')} ? _shapeLeafNumber(t)
@@ -3077,7 +3086,7 @@ function emitAstPrattRule(r: PrattRule, sir: ShapeIRRule, ids: LexIdPlan, shapeI
         const close = b.steps[b.steps.length - 1]?.t === 'lit' ? lidOf(ids, (b.steps[b.steps.length - 1] as { value: string }).value) : 0;
         groupCode = `  if (t.lid === ${open}) {
     const save = pos;
-    if (!_shapeDropLit(${open})) return null;
+    if (!_shapeDropLit(${open})) { pos = save; return null; }
     const inner = parseAst${r.name}();
     if (inner === null || !_shapeDropLit(${close})) { pos = save; return null; }
     return inner;

test/fixtures/shape-toy.ts

@@ -126,12 +126,8 @@ export const toyShape: ShapeSpec = {
     },
     Expr: {
       kind: 'pratt',
-      // keep yields bare leafScalars; toy golden needs Number|Identifier products.
-      atom: {
-        kind: 'custom',
-        fn: 'atom',
-        reason: 'Pratt atom products are Number|Identifier nodes; keep+leafValue only yields scalars',
-      },
+      // Delegate to Atom choice so Number|Identifier nodes need no runtime custom.
+      atom: { kind: 'rule', name: 'Atom' },
       group: { kind: 'inline' },
       prefix: {
         kind: 'node',
@@ -159,18 +155,42 @@ export const toyShape: ShapeSpec = {
         ],
       },
     },
+    // Three node arms (FIRST-overlap + true backtrack). Multi-alt single custom arm
+    // remains proven via shape-parity's taggedCustomShape emit (altPath witness).
     Tagged: {
       kind: 'choice',
       arms: [
         {
-          name: 'Tagged',
-          altIndices: [0, 1, 2],
+          name: 'ColonTag',
+          altIndices: [0],
           shape: {
-            kind: 'custom',
-            fn: 'Tagged',
-            reason:
-              'Three alts share FIRST "tag" and yield ColonTag|EqualsTag|BareTag; ' +
-              'one arm must try alts sequentially and hand altPath to custom',
+            kind: 'node',
+            type: 'ColonTag',
+            fields: [
+              { name: 'name', bind: { at: 0 }, typeHint: 'string' },
+              { name: 'value', bind: { at: 1 }, typeHint: 'ExprShape' },
+            ],
+          },
+        },
+        {
+          name: 'EqualsTag',
+          altIndices: [1],
+          shape: {
+            kind: 'node',
+            type: 'EqualsTag',
+            fields: [
+              { name: 'name', bind: { at: 0 }, typeHint: 'string' },
+              { name: 'value', bind: { at: 1 }, typeHint: 'ExprShape' },
+            ],
+          },
+        },
+        {
+          name: 'BareTag',
+          altIndices: [2],
+          shape: {
+            kind: 'node',
+            type: 'BareTag',
+            fields: [{ name: 'name', bind: { at: 0 }, typeHint: 'string' }],
           },
         },
       ],
@@ -230,7 +250,7 @@ export type ToyAstCustoms = Record<string, (ctx: ToyAstCustomCtx) => unknown>;
 const I = (name: string) => ({ type: 'Identifier', name });
 const N = (value: number) => ({ type: 'Number', value });
 
-/** Default customs so golden rows work without per-call overrides. */
+/** Optional customs for override/witness tests (default toy shape needs none). */
 export const toyCustoms: ToyAstCustoms = {
   atom: (ctx) => {
     const t = ctx.kids[0];
@@ -247,6 +267,30 @@ export const toyCustoms: ToyAstCustoms = {
   },
 };
 
+/** Multi-alt single custom arm — used by shape-parity altPath witness emit. */
+export const toyTaggedCustomShape: ShapeSpec = {
+  ...toyShape,
+  rules: {
+    ...toyShape.rules,
+    Tagged: {
+      kind: 'choice',
+      arms: [
+        {
+          name: 'Tagged',
+          altIndices: [0, 1, 2],
+          shape: {
+            kind: 'custom',
+            fn: 'Tagged',
+            reason:
+              'Three alts share FIRST "tag" and yield ColonTag|EqualsTag|BareTag; ' +
+              'one arm must try alts sequentially and hand altPath to custom',
+          },
+        },
+      ],
+    },
+  },
+};
+
 /** Proto2 golden (10), including custom-ctx row. */
 export const toyGolden: { src: string; expect: unknown; customs?: ToyAstCustoms }[] = [
   { src: 'bang !x;', expect: { type: 'Program', body: [{ type: 'BangOne', arg: I('x') }] } },
@@ -282,7 +326,7 @@ export const toyGolden: { src: string; expect: unknown; customs?: ToyAstCustoms
   },
 ];
 
-/** Seeded corpus builder identical to proto2-harness (seed 0x5a2_2026 → 520). */
+/** Seeded corpus: SH2-0 base + SH2-0b multi-stmt / nested-group / adv-112 (seed → 800). */
 export function buildToyCorpus(seed = 0x5a2_2026): { src: string; source: string }[] {
   function rng32(s: number) {
     return () => {
@@ -305,11 +349,24 @@ export function buildToyCorpus(seed = 0x5a2_2026): { src: string; source: string
     if (r < .56) return `${expr(depth + 1)}(${expr(depth + 1)},${rng() < .25 ? '' : expr(depth + 1)})`;
     return `${expr(depth + 1)}${pick(['+', '-', '*', '/'])}${expr(depth + 1)}`;
   }
+  /** Nested grouping for choice-arm Expr tails. */
+  function groupedExpr(depth = 1): string {
+    let e = atom();
+    for (let i = 0; i < depth; i++) e = `(${e})`;
+    return e;
+  }
   function validItem(): string {
     const r = rng();
     if (r < .18) return `bang ${rng() < .5 ? '!' : '!!'}${atom()}`;
     if (r < .40) {
-      const tail = pick([`:${expr()}`, `=${expr()}`, '']);
+      const nest = 1 + Math.floor(rng() * 4);
+      const tail = pick([
+        `:${expr()}`,
+        `=${expr()}`,
+        `:${groupedExpr(nest)}`,
+        `=${groupedExpr(nest)}`,
+        '',
+      ]);
       return `tag ${pick(ids)}${tail}`;
     }
     if (r < .56) return `guard ${pick(ids)}${rng() < .5 ? `:${pick(ids)}` : ''}`;
@@ -320,6 +377,11 @@ export function buildToyCorpus(seed = 0x5a2_2026): { src: string; source: string
     }
     return expr();
   }
+  /** Always 2–5 statements (SH2-0b multi-stmt coverage). */
+  function multiProgram(): string {
+    const n = 2 + Math.floor(rng() * 4);
+    return Array.from({ length: n }, () => validItem() + ';').join(rng() < .3 ? '\n' : ' ');
+  }
   function validProgram(): string {
     const n = Math.floor(rng() * 5);
     return Array.from({ length: n }, () => validItem() + ';').join(rng() < .3 ? '\n' : ' ');
@@ -331,14 +393,34 @@ export function buildToyCorpus(seed = 0x5a2_2026): { src: string; source: string
       'tag x', 'bang !!x', 'unknown unknown;', 'guard ;', 'args(1 2);',
     ]);
   }
+  // Planner adversarial 112 cases (depth groups + fragment cross-product).
+  const advCases: string[] = [];
+  for (let d = 1; d <= 40; d += 3) {
+    advCases.push('tag x' + '='.repeat(1) + '('.repeat(d) + '1' + ')'.repeat(d) + ';');
+  }
+  const frag = [
+    'tag x:1;', 'tag y=2;', 'tag z;', 'bang!x;', 'bang x;', 'f(1,2,)( );', 'a:(-b);',
+    'tag x:1', 'tag x=;', 'tag :1;', 'bang !;', 'f(,);', 'f(1,,2);', 'tag x:1;tag y=2;tag z;',
+  ];
+  for (const a of frag) for (const b of frag.slice(0, 7)) advCases.push(a + b);
+
   const anchors = [
     '', 'bang !x;', 'bang !!x;', 'bang !!!x;', 'tag x:1;', 'tag x=1;', 'tag x;',
     'tag x:;', 'tag x=;', 'guard bad;', 'guard good;', 'guard good:a;',
     'args();', 'args(1,);', 'args(1,2);', 'f();', 'f(1,);', 'f(1)(2);',
     '1+2*3;', '-x(1);', 'tag t:f(1,2); bang !!7;',
+    // SH2-0b: choice-arm nested groups + multi-stmt
+    'tag x=(1);', 'tag x=((1));', 'tag y=(((2)));', 'tag z:(3);', 'tag z:((a));',
+    'tag x:1;tag y=2;', 'bang!x;tag z;', 'tag x=(1);tag y=2;tag z;',
+    'tag a=(1);tag b=((2));tag c:(((3)));bang!x;guard ok;',
   ].map((src) => ({ src, source: 'boundary' }));
-  const corpus = [...anchors];
-  while (corpus.length < 360) corpus.push({ src: validProgram(), source: 'random-valid' });
-  while (corpus.length < 520) corpus.push({ src: invalidProgram(), source: 'random-invalid' });
+
+  const corpus = [
+    ...anchors,
+    ...advCases.map((src) => ({ src, source: 'adv-112' as const })),
+  ];
+  while (corpus.length < 560) corpus.push({ src: multiProgram(), source: 'multi-stmt' });
+  while (corpus.length < 680) corpus.push({ src: validProgram(), source: 'random-valid' });
+  while (corpus.length < 800) corpus.push({ src: invalidProgram(), source: 'random-invalid' });
   return corpus;
 }

test/shape-parity.ts

@@ -1,4 +1,4 @@
-// Gate: SH2-0 shape CST↔parseAst parity — calc+toy corpus ≥800, toy golden 10/10,
+// Gate: SH2-0b shape CST↔parseAst parity — calc+toy corpus ≥1200, toy golden 10/10,
 // coverage table, typescript+SH0 fail-fast (no home-path permanent dependency).
 import { mkdirSync, writeFileSync } from 'node:fs';
 import { pathToFileURL } from 'node:url';
@@ -9,7 +9,7 @@ import { calcShape } from '../src/shape-calc.ts';
 import type { ShapeSpec } from '../src/shape-schema.ts';
 import calcGrammar from './fixtures/calc.ts';
 import toyGrammar, {
-  toyShape, toyCustoms, toyGolden, buildToyCorpus, type ToyAstCustoms,
+  toyShape, toyCustoms, toyTaggedCustomShape, toyGolden, buildToyCorpus, type ToyAstCustoms,
 } from './fixtures/shape-toy.ts';
 import typescriptGrammar from '../typescript.ts';
 
@@ -83,11 +83,15 @@ function buildCalcCorpus(): { src: string; source: string }[] {
     'let x = 1;', '1 + 2;', '1 + 2 * 3;', '-a;', '(1);',
     'let a = 1; let b = 2; a + b;', '1 - 2 - 3;', '-(a * b);',
     'foo; bar; baz;', '2 / 3;', '--x;',
+    // SH2-0b: choice-arm group + multi-stmt (star Stmt)
+    'let x = (1+2);', 'let y = ((3));', 'let a = (1); let b = (2+3); a + b;',
+    'let x = 1; let y = 2; let z = 3; x + y + z;',
+    '(1+2); (3);', 'let x = (1+2); 3 + 4;',
     'let ;', '1+', '(1', 'let x =', ';;;', 'x = 1;',
   ].map((src) => ({ src, source: 'boundary' }));
   const out = [...anchors];
-  while (out.length < 220) out.push({ src: prog(), source: 'random-valid' });
-  while (out.length < 320) {
+  while (out.length < 320) out.push({ src: prog(), source: 'random-valid' });
+  while (out.length < 450) {
     out.push({
       src: pick(['let ;', '1+', '(1', 'let x =', 'x = 1;', '* 2;', 'let let = 1;']),
       source: 'random-invalid',
@@ -188,16 +192,18 @@ async function main(): Promise<void> {
 
   let goldenOk = 0;
   for (const g of toyGolden) {
-    const got = toyMod.parseAst(g.src, { customs: toyCustoms });
+    const got = toyMod.parseAst(g.src);
     if (deepEq(got, g.expect)) goldenOk++;
     else {
       console.error(`  golden fail ${JSON.stringify(g.src)}`);
       console.error(`    got  ${JSON.stringify(got)}`);
       console.error(`    want ${JSON.stringify(g.expect)}`);
     }
   }
+  // Multi-alt custom arm + altPath witness (separate emit; default toy is node arms).
+  const taggedCustomMod = await emitLoad('toy-tagged-custom', toyGrammar, toyTaggedCustomShape);
   let customSeen: unknown = null;
-  const customGot = toyMod.parseAst('tag ctx=9;', {
+  const customGot = taggedCustomMod.parseAst('tag ctx=9;', {
     customs: {
       ...toyCustoms,
       Tagged: (ctx) => {
@@ -224,10 +230,10 @@ async function main(): Promise<void> {
   check(goldenOk === 10, `toy golden ${goldenOk}/10`);
 
   const toyCorpus = buildToyCorpus(0x5a2_2026);
-  check(toyCorpus.length === 520, `toy corpus exact 520 (got ${toyCorpus.length})`);
+  check(toyCorpus.length === 800, `toy corpus exact 800 (got ${toyCorpus.length})`);
   const calcCorpus = buildCalcCorpus();
   const totalN = toyCorpus.length + calcCorpus.length;
-  check(totalN >= 800, `corpus total ≥800 (got ${totalN})`);
+  check(totalN >= 1200, `corpus total ≥1200 (got ${totalN})`);
 
   function parity(
     label: string,
@@ -259,7 +265,7 @@ async function main(): Promise<void> {
     return { diverge, cstAcc, astAcc };
   }
 
-  const toyP = parity('toy', toyMod, toyCorpus, toyCustoms);
+  const toyP = parity('toy', toyMod, toyCorpus);
   const calcP = parity('calc', calcMod, calcCorpus);
 
   const calcSpot = stripSpans(calcMod.parseAst('let x = 1; 2 + 3;'));
@@ -273,6 +279,20 @@ async function main(): Promise<void> {
     }),
     'calc spot golden',
   );
+  const calcGroup = stripSpans(calcMod.parseAst('let x = (1+2); let y = ((3));'));
+  check(
+    deepEq(calcGroup, {
+      type: 'Program',
+      body: [
+        {
+          type: 'LetStatement', id: 'x',
+          init: { type: 'BinaryExpression', left: 1, operator: '+', right: 2 },
+        },
+        { type: 'LetStatement', id: 'y', init: 3 },
+      ],
+    }),
+    'calc group-in-let + multi-stmt',
+  );
 
   let tsErr = '';
   try {