|
| 1 | +# Shape mapping — declarative AST on top of the CST |
| 2 | + |
| 3 | +A **shape** maps a grammar's concrete syntax tree (CST) into a consumer AST |
| 4 | +(types + builders). You declare the mapping once as a `ShapeSpec`; |
| 5 | +[`emitTs(grammar, { shape })`](../src/target-ts.ts) appends typed AST constructors |
| 6 | +and a specialized `parseAst` entry. Without a shape, `emitTs(grammar)` is |
| 7 | +byte-identical to `tsTarget.emitParser(...)`. |
| 8 | + |
| 9 | +Source of truth for types: [`src/shape-schema.ts`](../src/shape-schema.ts). |
| 10 | +Target-neutral contracts (custom context, fold markers, transaction rules): |
| 11 | +[`src/shape-machine.ts`](../src/shape-machine.ts). |
| 12 | + |
| 13 | +--- |
| 14 | + |
| 15 | +## ShapeSpec |
| 16 | + |
| 17 | +```ts |
| 18 | +type ShapeSpec = { |
| 19 | + grammar: string; // must match the grammar's name |
| 20 | + spans: 'required' | 'optional' | 'none'; // off/end injection on nodes |
| 21 | + unmapped: 'default' | 'error'; // keep vs fail on missing rules |
| 22 | + leaves: Record<string, TokenLeafPolicy>; // per token-class policy |
| 23 | + rules: Record<string, RuleShape>; // per IR / CST rule product |
| 24 | +}; |
| 25 | +``` |
| 26 | + |
| 27 | +| Field | Meaning | |
| 28 | +|---|---| |
| 29 | +| `grammar` | Labels the spec; validated against the grammar you emit. | |
| 30 | +| `spans` | Inject unified `off`/`end` source-range fields on node products (`required` always, `optional` as `off?`/`end?`, `none` omit). | |
| 31 | +| `unmapped` | `'default'` keeps unmapped rules as positional CST-shaped nodes; `'error'` demands full coverage (calc uses this). | |
| 32 | +| `leaves` | Token-class policies: `{ action: 'drop' \| 'keep' \| 'leafValue', fn? }`. `leafValue` coercions: `identity`, `number`, `bigint`, `string`, `boolean`, `ident` (or a custom name). | |
| 33 | +| `rules` | Map rule name → product shape. Exact IR names override inherited `cstName` mappings. | |
| 34 | + |
| 35 | +Adjudications (spans, unmapped, heterogeneous alts, rule keying) live in |
| 36 | +`ADJUDICATIONS` in [`shape-schema.ts`](../src/shape-schema.ts). |
| 37 | + |
| 38 | +### Field binds (`FieldDecl`) |
| 39 | + |
| 40 | +Node fields bind from packed kids / Pratt locals: |
| 41 | + |
| 42 | +| Bind | Effect | |
| 43 | +|---|---| |
| 44 | +| `{ at: n }` | Kid at index `n`. | |
| 45 | +| `{ label }` | Named kid (when labeled). | |
| 46 | +| `{ from: 'list'; of: n \| 'rest' }` | List / rest slot. | |
| 47 | +| `{ from: 'opt'; at: n }` | Optional hole (null when absent). | |
| 48 | +| `'opText'` | Pratt connector text (**bindOp**) — prefix / binary / postfix only. | |
| 49 | + |
| 50 | +--- |
| 51 | + |
| 52 | +## Primitives |
| 53 | + |
| 54 | +### Core six |
| 55 | + |
| 56 | +| Kind | Role | |
| 57 | +|---|---| |
| 58 | +| `drop` | Consume the rule; product is absent (`null`). | |
| 59 | +| `inline` | Splice kids into the parent (no wrapper). Pratt `group` often uses this for `(Expr)`. | |
| 60 | +| `node` | Named product `{ type, fields, exact? }` with field binds. | |
| 61 | +| `list` | Array product over a repeated element rule (`field` + optional `elemHint`). | |
| 62 | +| `leafValue` | Coerce a token lexeme (`fn: 'number' \| 'ident' \| …`). Usually via `leaves`, not RD rules. | |
| 63 | +| `custom` | Handwritten builder by name: `{ kind: 'custom', fn, reason, result?, folds? }`. | |
| 64 | + |
| 65 | +### Declared additions (`ADDED_PRIMITIVES`) |
| 66 | + |
| 67 | +| Name | Why | |
| 68 | +|---|---| |
| 69 | +| `choice` | Heterogeneous RD alternatives — each arm maps a disjoint `altIndices` set to a product. | |
| 70 | +| `pratt` | Pratt rules have distinct atom / group / prefix / binary / postfix / … slots. | |
| 71 | +| `keep` | Sugar for the default positional node (`{ type, children, headText, … }`). | |
| 72 | +| `bindOp` | Not a `kind` — the `'opText'` field bind so dropped operators still expose source text. | |
| 73 | +| `parentFold` | Parent `custom.folds: ParentFold[]` accumulates child `__shapePartial` markers into a field. | |
| 74 | + |
| 75 | +### Pratt slots (brief) |
| 76 | + |
| 77 | +```ts |
| 78 | +type PrattShape = { |
| 79 | + kind: 'pratt'; |
| 80 | + atom?, group?, nudSeq?, nudCapped?, |
| 81 | + prefix?, binary?, postfix?, led?, postfixTok?, |
| 82 | + template?: CustomShape | KeepShape; // see below |
| 83 | +}; |
| 84 | +``` |
| 85 | + |
| 86 | +Unmapped IR brackets default to **keep**. Omitted slots follow the same keep/default |
| 87 | +rules as adjudication. |
| 88 | + |
| 89 | +**`template`** — template-literal product (`$templateHead`… or a no-subst Template |
| 90 | +leaf). Only `custom` | `keep`. Omitted → legacy `$template` keep with portable |
| 91 | +`interpRule` holes (CST-parity accept set). Declared → kids are head, expr, |
| 92 | +optional middle/expr pairs, then tail (or `[leaf]` for no-subst), finished by this |
| 93 | +slot; hole *accept* still uses portable `interpRule`, hole *AST* uses the |
| 94 | +enclosing Pratt. |
| 95 | + |
| 96 | +--- |
| 97 | + |
| 98 | +## Custom context (`AstCustomCtx`) |
| 99 | + |
| 100 | +Defined in [`src/shape-machine.ts`](../src/shape-machine.ts). Populated **only after** |
| 101 | +the recognizer has successfully finished the current rule / Pratt event: |
| 102 | + |
| 103 | +```ts |
| 104 | +type AstCustomCtx = { |
| 105 | + src: string; // full source |
| 106 | + kids: readonly unknown[]; // packed kids (lists as arrays; absent opts as null) |
| 107 | + off: number; // consumed range start |
| 108 | + end: number; // consumed range end |
| 109 | + altPath: readonly number[]; // selected RD/Pratt arm, then nested inline alts (outermost-first) |
| 110 | + opText?: string; // Pratt LED connector text |
| 111 | + left?: unknown; // Pratt LED left value |
| 112 | + state?: unknown; // present only when parent declares folds (fold counters) |
| 113 | +}; |
| 114 | +type AstCustom = (ctx: AstCustomCtx) => unknown; |
| 115 | +``` |
| 116 | + |
| 117 | +`altPath` in the machine is a richer `AltPath` (RD / inline-alt / pratt-nud / |
| 118 | +pratt-led). The emitted TS `parseAst` surface exposes flattened numeric indices |
| 119 | +on the ctx object above — enough for arm selection in builders. |
| 120 | + |
| 121 | +Transaction rules for speculative append channels (same module): |
| 122 | +`SHAPE_TRANSACTION_CONTRACT` — append-only restore by length; overwrites via undo |
| 123 | +log or commit-on-success; Pratt locals commit on success; control flags restore |
| 124 | +with the checkpoint. |
| 125 | + |
| 126 | +--- |
| 127 | + |
| 128 | +## Fold protocol |
| 129 | + |
| 130 | +A child `custom` that produces a partial returns: |
| 131 | + |
| 132 | +```ts |
| 133 | +{ __shapePartial: tag, mode: 'start' | 'append', value } |
| 134 | +``` |
| 135 | + |
| 136 | +The parent declares how to fold those markers: |
| 137 | + |
| 138 | +```ts |
| 139 | +type ParentFold = { tag: string; into: string }; |
| 140 | +// on CustomShape: |
| 141 | +folds?: ParentFold[]; |
| 142 | +``` |
| 143 | + |
| 144 | +At parent finish, `_shapeFoldKids` runs **before** the parent's custom callback: |
| 145 | + |
| 146 | +1. Matching `start` opens an output item (`value` pushed as-is). |
| 147 | +2. Matching `append` pushes `value` into that item's `into` array field. |
| 148 | +3. Non-partial kids pass through (lists recurse). |
| 149 | +4. Fold counters land on `ctx.state` when `folds` is non-empty. |
| 150 | + |
| 151 | +Example pattern (switch cases — see `estreeSwitchCase` / |
| 152 | +`Stmt.folds` in the TypeScript shape fixture): case arms `start` a |
| 153 | +`SwitchCase`; following statement arms `append` into `consequent`. |
| 154 | + |
| 155 | +No grammar or rule name is embedded in the mechanism — only the declared `tag`. |
| 156 | + |
| 157 | +--- |
| 158 | + |
| 159 | +## API |
| 160 | + |
| 161 | +### Emit |
| 162 | + |
| 163 | +```ts |
| 164 | +import { emitTs } from './src/target-ts.ts'; |
| 165 | +import type { ShapeSpec } from './src/shape-schema.ts'; |
| 166 | + |
| 167 | +const src = emitTs(grammar, { shape: myShape }); |
| 168 | +// write src → module that exports tokenize, parse, parseAst, AstRoot, … |
| 169 | +``` |
| 170 | + |
| 171 | +- No `shape` → same bytes as `emitParser(grammar, tsTarget)` (byte-identity gate). |
| 172 | +- With `shape` → base CST parser **plus** AST type decls, `parseAst*`, helpers, |
| 173 | + and `shapeCoverage`. |
| 174 | + |
| 175 | +Validation (`validateShape` / `validateShapeOrThrow`) runs inside `emitTs` before |
| 176 | +codegen. |
| 177 | + |
| 178 | +### Parse |
| 179 | + |
| 180 | +```ts |
| 181 | +import type { AstCustoms } from './emitted-parser.ts'; // generated |
| 182 | + |
| 183 | +const ast = parseAst(src, { customs?: AstCustoms }); |
| 184 | +// AstRoot | null (full consume; pos === toks.length) |
| 185 | +``` |
| 186 | + |
| 187 | +Register every `custom.fn` name used by the spec. Missing customs throw at the |
| 188 | +call site (`shape: custom X not provided`). Specs that need no customs (e.g. calc) |
| 189 | +call `parseAst(src)` with an empty map. |
| 190 | + |
| 191 | +--- |
| 192 | + |
| 193 | +## Walkthrough: calc |
| 194 | + |
| 195 | +Full declarative coverage lives in [`src/shape-calc.ts`](../src/shape-calc.ts) |
| 196 | +(`unmapped: 'error'`). Highlights: |
| 197 | + |
| 198 | +```ts |
| 199 | +export const calcShape: ShapeSpec = { |
| 200 | + grammar: 'calc', |
| 201 | + spans: 'optional', |
| 202 | + unmapped: 'error', |
| 203 | + leaves: { |
| 204 | + $punct: { action: 'drop' }, |
| 205 | + $keyword: { action: 'drop' }, |
| 206 | + Number: { action: 'leafValue', fn: 'number' }, |
| 207 | + Ident: { action: 'leafValue', fn: 'ident' }, |
| 208 | + }, |
| 209 | + rules: { |
| 210 | + Expr: { |
| 211 | + kind: 'pratt', |
| 212 | + atom: { kind: 'keep' }, |
| 213 | + group: { kind: 'inline' }, |
| 214 | + prefix: { |
| 215 | + kind: 'node', type: 'UnaryExpression', |
| 216 | + fields: [ |
| 217 | + { name: 'operator', bind: 'opText', typeHint: 'string' }, |
| 218 | + { name: 'argument', bind: { at: 0 }, typeHint: 'Expression' }, |
| 219 | + ], |
| 220 | + }, |
| 221 | + binary: { |
| 222 | + kind: 'node', type: 'BinaryExpression', |
| 223 | + fields: [ |
| 224 | + { name: 'left', bind: { at: 0 }, typeHint: 'Expression' }, |
| 225 | + { name: 'operator', bind: 'opText', typeHint: 'string' }, |
| 226 | + { name: 'right', bind: { at: 1 }, typeHint: 'Expression' }, |
| 227 | + ], |
| 228 | + }, |
| 229 | + }, |
| 230 | + Stmt: { |
| 231 | + kind: 'choice', |
| 232 | + arms: [ |
| 233 | + { name: 'LetStatement', altIndices: [0], shape: { kind: 'node', type: 'LetStatement', /* id, init */ } }, |
| 234 | + { name: 'ExpressionStatement', altIndices: [1], shape: { kind: 'node', type: 'ExpressionStatement', /* expression */ } }, |
| 235 | + ], |
| 236 | + }, |
| 237 | + Program: { |
| 238 | + kind: 'node', type: 'Program', |
| 239 | + fields: [{ name: 'body', bind: { from: 'list', of: 0 }, typeHint: 'Statement' }], |
| 240 | + }, |
| 241 | + }, |
| 242 | +}; |
| 243 | +``` |
| 244 | + |
| 245 | +Emit → load → parse (mirrors [`test/shape-codegen.ts`](../test/shape-codegen.ts)): |
| 246 | + |
| 247 | +```ts |
| 248 | +import calcGrammar from './test/fixtures/calc.ts'; |
| 249 | +import { calcShape } from './src/shape-calc.ts'; |
| 250 | +import { emitTs } from './src/target-ts.ts'; |
| 251 | + |
| 252 | +const code = emitTs(calcGrammar, { shape: calcShape }); |
| 253 | +// write + dynamic import → { parseAst } |
| 254 | + |
| 255 | +parseAst('let x = 1;'); |
| 256 | +// { type: 'Program', body: [{ type: 'LetStatement', id: 'x', init: 1 }], off?, end? } |
| 257 | + |
| 258 | +parseAst('1 + 2 * 3;'); |
| 259 | +// Program → ExpressionStatement → BinaryExpression(+ , BinaryExpression(*, …)) |
| 260 | +``` |
| 261 | + |
| 262 | +No `customs` map — every product is declarative (`node` / `choice` / `pratt` / |
| 263 | +`leafValue` / `inline` / `keep`). |
| 264 | + |
| 265 | +--- |
| 266 | + |
| 267 | +## Gates |
| 268 | + |
| 269 | +| Gate | Question | |
| 270 | +|---|---| |
| 271 | +| [`test/shape-codegen.ts`](../test/shape-codegen.ts) | `validateShape` on calc; no-shape byte identity vs `emitParser`; golden `parseAst` ASTs for calc (handwritten expects); perf smoke. | |
| 272 | +| [`test/shape-parity.ts`](../test/shape-parity.ts) | CST↔`parseAst` accept/reject parity (calc + toy corpora); custom/`parentFold` spots; TypeScript shape coverage + ESTree customs. | |
| 273 | + |
| 274 | +Both run under `npm run check` (see [`docs/TESTING.md`](TESTING.md)). |
0 commit comments