compiler: detect excessively-deep specialization (#599)

Author: pdubroy

packages/compiler/src/Compiler.ts

@@ -1328,6 +1328,13 @@ export class Compiler {
     const patternsByRule = new Map<string, Map<string, Expr[]>>();
     const refCounts = new Map();
 
+    // Track the recursive specialization depth per base rule name.
+    // If specializing a rule leads to specializing the *same* rule again
+    // beyond this depth, the parameters are expanding without bound
+    // (e.g. `grow<e> = e | grow<(e | "x")>`).
+    const MAX_SPECIALIZATION_DEPTH = 32;
+    const specializationDepth = new Map<string, number>();
+
     const specialize = (exp: Expr): Expr =>
       ir.rewrite(exp, {
         Apply: app => {
@@ -1339,6 +1346,17 @@ export class Compiler {
           // If not yet seen, recursively visit the body of the specialized
           // rule. Note that this also applies to non-parameterized rules!
           if (!newRules.has(specializedName)) {
+            const prevDepth = specializationDepth.get(ruleName) ?? 0;
+            if (children.length > 0) {
+              if (prevDepth >= MAX_SPECIALIZATION_DEPTH) {
+                throw new Error(
+                  `Excessively deep specialization of rule '${ruleName}' (>${MAX_SPECIALIZATION_DEPTH} levels). ` +
+                    'This usually means its parameters grow on each recursive call, ' +
+                    'producing an infinite number of specialized rules.'
+                );
+              }
+              specializationDepth.set(ruleName, prevDepth + 1);
+            }
             newRules.set(specializedName, {} as RuleInfo); // Prevent infinite recursion.
 
             // Visit the body with the parameter substituted, to ensure we
@@ -1347,6 +1365,9 @@ export class Compiler {
               ir.substituteParams(ruleInfo.body, children as Exclude<Expr, ir.Param>[])
             );
 
+            // Restore the depth after the recursive visit.
+            specializationDepth.set(ruleName, prevDepth);
+
             // If there are any args, replace the body with an application of
             // the generalized rule.
             if (children.length > 0) {

packages/compiler/test/test-wasm.js

@@ -1939,3 +1939,24 @@ test('chunkedBindings: false', async t => {
     wasmGrammar.match('hello;').use(r => t.true(r.succeeded()));
   }
 });
+
+// When parameters grow at each recursive step — e.g., grow<(e | "x")> where
+// e keeps expanding — each specialization produces a new unique name, so the
+// placeholder cycle detection never fires. The specializer should detect this
+// and throw a clear error rather than blowing the stack / running out of memory.
+test('parameterized rules: growing parameters should not blow the stack', t => {
+  t.throws(
+    () => {
+      const compiler = new Compiler(
+        ohm.grammar(`
+        G {
+          start = grow<"a">
+          grow<e> = e | grow<(e | "x")>
+        }
+      `)
+      );
+      compiler.compile();
+    },
+    {message: /Excessively deep specialization/}
+  );
+});