Shared Backend Refactoring

Author: nx10

ARCHITECTURE.md

@@ -71,13 +71,13 @@ The IR is the skeleton of the command line; the bindings define the typed interf
 
 ## Styx 1 vs Styx 2
 
-|                     | Styx 1 (Python)                                                                          | Styx 2 (TypeScript)                                                                      |
-| ------------------- | ---------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------- |
-| **IR**              | Dataclass hierarchy (`Param[T]` with body types)                                         | Algebraic expr tree with `kind` discriminant                                             |
-| **Optimization**    | Minimal (string merging)                                                                 | Pass-based pipeline (flatten, simplify, canonicalize)                                    |
-| **Type resolution** | Direct mapping in frontend; each backend re-derives types via language provider protocol | Solver produces a universal `BoundType` tree; backends just translate it                 |
+|                     | Styx 1 (Python)                                                                          | Styx 2 (TypeScript)                                                                        |
+| ------------------- | ---------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------ |
+| **IR**              | Dataclass hierarchy (`Param[T]` with body types)                                         | Algebraic expr tree with `kind` discriminant                                               |
+| **Optimization**    | Minimal (string merging)                                                                 | Pass-based pipeline (flatten, simplify, canonicalize)                                      |
+| **Type resolution** | Direct mapping in frontend; each backend re-derives types via language provider protocol | Solver produces a universal `BoundType` tree; backends just translate it                   |
 | **Backends**        | Python mature, TS/R partial; each implements a complex `LanguageProvider` protocol       | TypeScript + JSON Schema complete; Python/R stubs; simpler since solver does heavy lifting |
-| **Output files**    | First-class: path templates with param refs, suffix stripping, fallbacks                 | Not yet modeled to the same degree                                                       |
+| **Output files**    | First-class: path templates with param refs, suffix stripping, fallbacks                 | Not yet modeled to the same degree                                                         |
 
 Key Styx 1 features to eventually match:
 

packages/core/src/backend/collect-field-info.ts

@@ -0,0 +1,50 @@
+import type { BoundType } from "../bindings/index.js";
+import type { CodegenContext } from "../manifest/index.js";
+import { findDoc } from "./find-doc.js";
+import { findStructNode } from "./find-struct-node.js";
+import { resolveFieldBinding } from "./resolve-field-binding.js";
+
+/**
+ * Metadata extracted for each field of a struct type.
+ *
+ * `doc` is the field's description, recovered from wrapper nodes via `findDoc`.
+ * `defaultValue` is pulled from the wrapper or binding node's metadata.
+ */
+export interface FieldInfo {
+  doc?: string;
+  defaultValue?: string | number | boolean;
+}
+
+/**
+ * Collect field metadata (doc, defaultValue) for each field of a struct type.
+ *
+ * Walks the IR tree to find the sequence node containing the struct's fields,
+ * then resolves each child to its field binding. Metadata is recovered from both
+ * the wrapper node (where the parser hoists doc) and the binding node (where the
+ * solver places the binding after sequence collapse).
+ */
+export function collectFieldInfo(
+  ctx: CodegenContext,
+  structType: Extract<BoundType, { kind: "struct" }>,
+): Map<string, FieldInfo> {
+  const info = new Map<string, FieldInfo>();
+
+  const structNode = findStructNode(ctx.expr, ctx, structType);
+  if (!structNode) return info;
+
+  for (const child of structNode.attrs.nodes) {
+    const match = resolveFieldBinding(child, ctx, structType);
+    if (!match) continue;
+    const { binding, wrapperNode } = match;
+    const fieldInfo: FieldInfo = {};
+    const fieldType = structType.fields[binding.name]!;
+    // Check wrapper node first (doc may be hoisted there), then binding node
+    const doc = findDoc(wrapperNode, fieldType) ?? findDoc(binding.node, fieldType);
+    if (doc) fieldInfo.doc = doc;
+    const defaultValue = wrapperNode.meta?.defaultValue ?? binding.node.meta?.defaultValue;
+    if (defaultValue !== undefined) fieldInfo.defaultValue = defaultValue;
+    info.set(binding.name, fieldInfo);
+  }
+
+  return info;
+}

packages/core/src/backend/collect-named-types.ts

@@ -0,0 +1,87 @@
+import type { BoundType } from "../bindings/index.js";
+import { Scope } from "./scope.js";
+import { structKey, unionKey } from "./type-keys.js";
+
+/** A named compound type (struct or union) discovered during type collection. */
+export interface NamedType {
+  name: string;
+  type: BoundType;
+}
+
+/**
+ * Recursively collect all struct/union types and assign unique names.
+ *
+ * Walks the BoundType tree depth-first, using structural keys (`structKey`,
+ * `unionKey`) to deduplicate types that appear multiple times in the tree.
+ * Each unique struct/union gets a name generated by applying `nameTransform`
+ * to a hint string (derived from the root name or field/variant names).
+ *
+ * @param rootType - The root BoundType to walk.
+ * @param rootName - The hint for the root type's name.
+ * @param scope - Symbol scope for collision avoidance.
+ * @param nameTransform - Converts a hint string to a type name (e.g. pascalCase, snake_case).
+ * @returns `namedTypes` maps structural keys to assigned names, `typeDecls` lists declarations in order.
+ */
+export function collectNamedTypes(
+  rootType: BoundType,
+  rootName: string,
+  scope: Scope,
+  nameTransform: (hint: string) => string,
+): { namedTypes: Map<string, string>; typeDecls: NamedType[] } {
+  const namedTypes = new Map<string, string>();
+  const typeDecls: NamedType[] = [];
+
+  function visit(type: BoundType, hint: string): void {
+    switch (type.kind) {
+      case "struct": {
+        const key = structKey(type);
+        if (!namedTypes.has(key)) {
+          const name = scope.add(nameTransform(hint));
+          namedTypes.set(key, name);
+          typeDecls.push({ name, type });
+          for (const [fieldName, fieldType] of Object.entries(type.fields)) {
+            visit(fieldType, fieldName);
+          }
+        }
+        break;
+      }
+      case "union": {
+        const key = unionKey(type);
+        if (!namedTypes.has(key)) {
+          const name = scope.add(nameTransform(hint));
+          namedTypes.set(key, name);
+          typeDecls.push({ name, type });
+          for (const v of type.variants) {
+            visit(v.type, v.name ?? hint);
+          }
+        }
+        break;
+      }
+      case "optional":
+        visit(type.inner, hint);
+        break;
+      case "list":
+        visit(type.item, hint);
+        break;
+      default:
+        break;
+    }
+  }
+
+  visit(rootType, rootName);
+  return { namedTypes, typeDecls };
+}
+
+/**
+ * Create a type name resolver from a namedTypes map.
+ * Returns a function that maps a BoundType to its assigned name (if any).
+ */
+export function resolveTypeName(
+  namedTypes: Map<string, string>,
+): (type: BoundType) => string | undefined {
+  return (type) => {
+    if (type.kind === "struct") return namedTypes.get(structKey(type));
+    if (type.kind === "union") return namedTypes.get(unionKey(type));
+    return undefined;
+  };
+}

packages/core/src/backend/find-doc.ts

@@ -0,0 +1,38 @@
+import type { BoundType } from "../bindings/index.js";
+import type { Expr } from "../ir/index.js";
+
+/**
+ * Find a description from an IR node, traversing through wrapper nodes.
+ *
+ * The parser's `wrapNode` hoists doc metadata to the outermost wrapper node,
+ * but the solver's simplify pass can collapse sequences, burying descriptions
+ * deeper in the tree.
+ *
+ * This traversal is type-aware: it only enters sequences when the corresponding
+ * BoundType is not a struct. Struct sequences have their own field collection
+ * call, so entering them would steal nested struct children's descriptions.
+ *
+ * @param node - The IR node to search for a description.
+ * @param fieldType - The BoundType of the field, used to determine traversal boundaries.
+ */
+export function findDoc(node: Expr, fieldType: BoundType): string | undefined {
+  if (node.meta?.doc?.description) return node.meta.doc.description;
+  switch (node.kind) {
+    case "optional":
+      return findDoc(node.attrs.node, fieldType.kind === "optional" ? fieldType.inner : fieldType);
+    case "repeat":
+      return findDoc(node.attrs.node, fieldType.kind === "list" ? fieldType.item : fieldType);
+    case "sequence": {
+      // Only traverse into sequences that were collapsed (non-struct field types).
+      // Struct sequences have their own collectFieldInfo call for their children.
+      if (fieldType.kind === "struct") return undefined;
+      for (const child of node.attrs.nodes) {
+        const doc = findDoc(child, fieldType);
+        if (doc) return doc;
+      }
+      return undefined;
+    }
+    default:
+      return undefined;
+  }
+}

packages/core/src/backend/find-struct-node.ts

@@ -0,0 +1,66 @@
+import type { BoundType } from "../bindings/index.js";
+import type { Expr } from "../ir/index.js";
+import type { CodegenContext } from "../manifest/index.js";
+import { resolveFieldBinding } from "./resolve-field-binding.js";
+
+/**
+ * Find the sequence node whose child bindings match a struct type's fields.
+ *
+ * Traverses through optional, repeat, and alternative wrappers to find the
+ * sequence that directly contains the struct's field bindings. This is necessary
+ * because the solver may collapse `seq(lit("--flag"), terminal)` into the terminal,
+ * burying bindings deeper in the tree.
+ *
+ * Uses a two-phase check for sequences:
+ * 1. Direct binding check (`ctx.resolve`) - matches when bindings are on immediate children
+ * 2. Recursive binding check (`resolveFieldBinding`) - matches when solver collapsed
+ *    a seq(lit, terminal) and the binding is buried deeper
+ *
+ * Phase 1 is tried first to avoid falsely matching an outer sequence when an inner
+ * sequence is the actual struct owner (e.g. `seq(lit("--flag"), seq(field1, field2))`).
+ */
+export function findStructNode(
+  node: Expr,
+  ctx: CodegenContext,
+  structType: Extract<BoundType, { kind: "struct" }>,
+): Extract<Expr, { kind: "sequence" }> | undefined {
+  switch (node.kind) {
+    case "sequence": {
+      // Phase 1: Check if any direct child has a binding matching a struct field
+      for (const child of node.attrs.nodes) {
+        const binding = ctx.resolve(child);
+        if (
+          binding &&
+          binding.name in structType.fields &&
+          binding.type === structType.fields[binding.name]
+        ) {
+          return node;
+        }
+      }
+      // Recurse into child nodes first (prefer deeper matches)
+      for (const child of node.attrs.nodes) {
+        const result = findStructNode(child, ctx, structType);
+        if (result) return result;
+      }
+      // Phase 2: Check via resolveFieldBinding for collapsed sequences
+      // where bindings are buried inside collapsed seq(lit, terminal)
+      for (const child of node.attrs.nodes) {
+        if (resolveFieldBinding(child, ctx, structType)) return node;
+      }
+      return undefined;
+    }
+    case "optional":
+      return findStructNode(node.attrs.node, ctx, structType);
+    case "repeat":
+      return findStructNode(node.attrs.node, ctx, structType);
+    case "alternative": {
+      for (const alt of node.attrs.alts) {
+        const result = findStructNode(alt, ctx, structType);
+        if (result) return result;
+      }
+      return undefined;
+    }
+    default:
+      return undefined;
+  }
+}

packages/core/src/backend/index.ts

@@ -1,7 +1,15 @@
 export type { Backend, EmitError, EmitResult, EmitWarning, TypeMap } from "./backend.js";
 export { CodeBuilder } from "./code-builder.js";
+export type { FieldInfo } from "./collect-field-info.js";
+export { collectFieldInfo } from "./collect-field-info.js";
+export type { NamedType } from "./collect-named-types.js";
+export { collectNamedTypes, resolveTypeName } from "./collect-named-types.js";
+export { findDoc } from "./find-doc.js";
+export { findStructNode } from "./find-struct-node.js";
+export { resolveFieldBinding } from "./resolve-field-binding.js";
 export { Scope } from "./scope.js";
 export type { JsonSchema } from "./schema/index.js";
 export { generateSchema, JsonSchemaBackend } from "./schema/index.js";
 export { camelCase, pascalCase, screamingSnakeCase, snakeCase } from "./string-case.js";
+export { structKey, typeKey, unionKey } from "./type-keys.js";
 export { generateTypeScript, TypeScriptBackend } from "./typescript/index.js";

packages/core/src/backend/resolve-field-binding.ts

@@ -0,0 +1,41 @@
+import type { Binding, BoundType } from "../bindings/index.js";
+import type { Expr } from "../ir/index.js";
+import type { CodegenContext } from "../manifest/index.js";
+
+/**
+ * Resolve a struct child node to its field binding, handling collapsed sequences.
+ *
+ * When the solver's simplify pass collapses `seq(lit("--flag"), terminal)` into
+ * just the terminal, the binding ends up on the inner node while metadata (doc,
+ * defaultValue) may remain on the outermost wrapper. This function recursively
+ * descends through collapsed sequences to find the binding, tracking the outermost
+ * node for metadata recovery.
+ *
+ * Uses type identity (`===`) to verify the binding matches the struct's field type,
+ * preventing cross-nesting name collisions where an inner struct has a field with
+ * the same name as the outer struct.
+ */
+export function resolveFieldBinding(
+  node: Expr,
+  ctx: CodegenContext,
+  structType: Extract<BoundType, { kind: "struct" }>,
+  outermost?: Expr,
+): { binding: Binding; wrapperNode: Expr } | undefined {
+  const wrapper = outermost ?? node;
+  const binding = ctx.resolve(node);
+  if (
+    binding &&
+    binding.name in structType.fields &&
+    binding.type === structType.fields[binding.name]
+  ) {
+    return { binding, wrapperNode: wrapper };
+  }
+  // Recurse into collapsed sequences to find the binding deeper
+  if (node.kind === "sequence") {
+    for (const inner of node.attrs.nodes) {
+      const result = resolveFieldBinding(inner, ctx, structType, wrapper);
+      if (result) return result;
+    }
+  }
+  return undefined;
+}