Parser: Expose Prism Nodes in Herb Syntax Tree

Author: marcoroth

.rubocop.yml

@@ -58,7 +58,9 @@ Metrics/CyclomaticComplexity:
     - lib/herb/ast/nodes.rb
     - lib/herb/cli.rb
     - lib/herb/engine.rb
+    - lib/herb/prism_inspect.rb
     - lib/herb/engine/**/*.rb
+    - templates/template.rb
 
 Metrics/MethodLength:
   Max: 20
@@ -68,6 +70,7 @@ Metrics/MethodLength:
     - lib/herb/project.rb
     - lib/herb/engine.rb
     - lib/herb/engine/**/*.rb
+    - lib/herb/prism_inspect.rb
     - templates/template.rb
     - test/fork_helper.rb
     - test/snapshot_utils.rb
@@ -83,6 +86,7 @@ Metrics/AbcSize:
     - lib/herb/engine.rb
     - lib/herb/engine/**/*.rb
     - lib/herb/token.rb
+    - lib/herb/prism_inspect.rb
     - templates/template.rb
     - test/fork_helper.rb
     - test/snapshot_utils.rb
@@ -98,6 +102,7 @@ Metrics/ClassLength:
     - lib/herb/engine/**/*.rb
     - lib/herb/project.rb
     - lib/herb/visitor.rb
+    - lib/herb/ast/nodes.rb
     - test/**/*_test.rb
 
 Metrics/ModuleLength:
@@ -121,14 +126,17 @@ Metrics/ParameterLists:
   Exclude:
     - lib/herb/ast/node.rb
     - lib/herb/ast/nodes.rb
-    - lib/herb/errors.rb
     - lib/herb/engine/validators/security_validator.rb
+    - lib/herb/errors.rb
+    - lib/herb/parser_options.rb
+    - lib/herb/prism_inspect.rb
 
 Metrics/PerceivedComplexity:
   Exclude:
     - lib/herb/ast/nodes.rb
     - lib/herb/cli.rb
     - lib/herb/project.rb
+    - lib/herb/prism_inspect.rb
     - lib/herb/engine.rb
     - lib/herb/engine/**/*.rb
     - templates/template.rb

config.yml

@@ -369,6 +369,12 @@ nodes:
           type: array
           kind: Node
 
+        - name: prism_context
+          type: prism_context
+
+        - name: prism_node
+          type: prism_node
+
     - name: LiteralNode
       fields:
         - name: content
@@ -643,6 +649,9 @@ nodes:
         - name: valid
           type: boolean
 
+        - name: prism_node
+          type: prism_node
+
     - name: ERBEndNode
       fields:
         - name: tag_opening
@@ -686,6 +695,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -717,6 +729,9 @@ nodes:
         # - name: opener
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: body
           type: array
           kind: Node
@@ -765,6 +780,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: conditions
           type: array
           kind: ERBWhenNode
@@ -795,6 +813,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: conditions
           type: array
           kind: ERBInNode
@@ -821,6 +842,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -843,6 +867,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -868,6 +895,9 @@ nodes:
         # - name: collection
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -928,6 +958,9 @@ nodes:
         - name: tag_closing
           type: token
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -965,6 +998,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node

ext/herb/extconf.rb

@@ -27,7 +27,6 @@
 $CFLAGS << " -DPRISM_EXCLUDE_PRETTYPRINT"
 $CFLAGS << " -DPRISM_EXCLUDE_JSON"
 $CFLAGS << " -DPRISM_EXCLUDE_PACK"
-$CFLAGS << " -DPRISM_EXCLUDE_SERIALIZATION"
 
 herb_src_files = Dir.glob("#{$srcdir}/../../src/**/*.c").map { |file| file.delete_prefix("../../../../ext/herb/") }.sort
 

ext/herb/extension.c

@@ -136,6 +136,18 @@ static VALUE Herb_parse(int argc, VALUE* argv, VALUE self) {
     }
     if (!NIL_P(action_view_helpers) && RTEST(action_view_helpers)) { parser_options.action_view_helpers = true; }
 
+    VALUE prism_nodes = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_nodes"));
+    if (NIL_P(prism_nodes)) { prism_nodes = rb_hash_lookup(options, ID2SYM(rb_intern("prism_nodes"))); }
+    if (!NIL_P(prism_nodes) && RTEST(prism_nodes)) { parser_options.prism_nodes = true; }
+
+    VALUE prism_nodes_deep = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_nodes_deep"));
+    if (NIL_P(prism_nodes_deep)) { prism_nodes_deep = rb_hash_lookup(options, ID2SYM(rb_intern("prism_nodes_deep"))); }
+    if (!NIL_P(prism_nodes_deep) && RTEST(prism_nodes_deep)) { parser_options.prism_nodes_deep = true; }
+
+    VALUE prism_program = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_program"));
+    if (NIL_P(prism_program)) { prism_program = rb_hash_lookup(options, ID2SYM(rb_intern("prism_program"))); }
+    if (!NIL_P(prism_program) && RTEST(prism_program)) { parser_options.prism_program = true; }
+
     VALUE arena_stats = rb_hash_lookup(options, rb_utf8_str_new_cstr("arena_stats"));
     if (NIL_P(arena_stats)) { arena_stats = rb_hash_lookup(options, ID2SYM(rb_intern("arena_stats"))); }
     if (!NIL_P(arena_stats) && RTEST(arena_stats)) { print_arena_stats = true; }

java/Makefile

@@ -30,7 +30,7 @@ PRISM_BUILD = $(PRISM_PATH)/build
 
 JAVAC = $(JAVA_HOME)/bin/javac
 JAVA_CMD = $(JAVA_HOME)/bin/java
-CFLAGS = -std=c99 -Wall -Wextra -fPIC -O2 -DHERB_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_JSON -DPRISM_EXCLUDE_PACK -DPRISM_EXCLUDE_SERIALIZATION
+CFLAGS = -std=c99 -Wall -Wextra -fPIC -O2 -DHERB_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_JSON -DPRISM_EXCLUDE_PACK
 INCLUDES = -I. -I$(SRC_DIR)/include -I$(PRISM_INCLUDE) $(JNI_INCLUDES)
 LDFLAGS = -shared
 LIBS = $(PRISM_BUILD)/libprism.a

java/herb_jni.c

@@ -68,6 +68,30 @@ Java_org_herb_Herb_parse(JNIEnv* env, jclass clazz, jstring source, jobject opti
       jboolean actionViewHelpers = (*env)->CallBooleanMethod(env, options, getActionViewHelpers);
       parser_options.action_view_helpers = (actionViewHelpers == JNI_TRUE);
     }
+
+    jmethodID getPrismNodes =
+        (*env)->GetMethodID(env, optionsClass, "isPrismNodes", "()Z");
+
+    if (getPrismNodes != NULL) {
+      jboolean prismNodes = (*env)->CallBooleanMethod(env, options, getPrismNodes);
+      parser_options.prism_nodes = (prismNodes == JNI_TRUE);
+    }
+
+    jmethodID getPrismNodesDeep =
+        (*env)->GetMethodID(env, optionsClass, "isPrismNodesDeep", "()Z");
+
+    if (getPrismNodesDeep != NULL) {
+      jboolean prismNodesDeep = (*env)->CallBooleanMethod(env, options, getPrismNodesDeep);
+      parser_options.prism_nodes_deep = (prismNodesDeep == JNI_TRUE);
+    }
+
+    jmethodID getPrismProgram =
+        (*env)->GetMethodID(env, optionsClass, "isPrismProgram", "()Z");
+
+    if (getPrismProgram != NULL) {
+      jboolean prismProgram = (*env)->CallBooleanMethod(env, options, getPrismProgram);
+      parser_options.prism_program = (prismProgram == JNI_TRUE);
+    }
   }
 
   hb_allocator_T allocator;

java/org/herb/HerbTest.java

@@ -102,6 +102,55 @@ void testParserOptionsTrackWhitespace() {
     assertFalse(inspectWithout.contains("WhitespaceNode"));
   }
 
+  @Test
+  void testParserOptionsPrismNodes() {
+    String source = "<%= user.name %>";
+
+    ParseResult withPrism = Herb.parse(source, ParserOptions.create().prismNodes(true));
+    String inspected = withPrism.inspect();
+
+    assertTrue(inspected.contains("prism_node_serialized:"));
+    assertTrue(inspected.contains("CallNode"));
+  }
+
+  @Test
+  void testParserOptionsPrismNodesString() {
+    String source = "<%= \"String\" %>";
+
+    ParseResult withPrism = Herb.parse(source, ParserOptions.create().prismNodes(true));
+    String inspected = withPrism.inspect();
+
+    assertTrue(inspected.contains("prism_node_serialized:"));
+    assertTrue(inspected.contains("StringNode"));
+    assertTrue(inspected.contains("String"));
+  }
+
+  @Test
+  void testParserOptionsPrismProgram() {
+    String source = "<%= \"hello\" %>";
+
+    ParseResult withPrism = Herb.parse(source, ParserOptions.create().prismProgram(true));
+    String inspected = withPrism.inspect();
+
+    assertTrue(inspected.contains("prism_node_serialized:"));
+    assertTrue(inspected.contains("ProgramNode"));
+  }
+
+  @Test
+  void testParserOptionsPrismNodesDeep() {
+    String source = "<% if true %><%= \"yes\" %><% end %>";
+
+    ParseResult withDeep = Herb.parse(source, ParserOptions.create().prismNodes(true).prismNodesDeep(true));
+    ParseResult withoutDeep = Herb.parse(source, ParserOptions.create().prismNodes(true).prismNodesDeep(false));
+
+    String inspectDeep = withDeep.inspect();
+    String inspectShallow = withoutDeep.inspect();
+
+    assertTrue(inspectDeep.contains("prism_node_serialized:"));
+    assertTrue(inspectShallow.contains("prism_node_serialized:"));
+    assertTrue(inspectDeep.contains("IfNode"));
+  }
+
   @Test
   void testParserOptionsAnalyze() {
     String source = "<% if true %><div></div><% end %>";

java/org/herb/ParseResult.java

@@ -50,7 +50,7 @@ public String inspect() {
     StringBuilder builder = new StringBuilder();
 
     if (value != null) {
-      builder.append(value.inspect());
+      builder.append(value.inspect(source));
     }
 
     if (hasErrors()) {

java/org/herb/ParserOptions.java

@@ -5,6 +5,9 @@ public class ParserOptions {
   private boolean analyze = true;
   private boolean strict = true;
   private boolean actionViewHelpers = false;
+  private boolean prismNodes = false;
+  private boolean prismNodesDeep = false;
+  private boolean prismProgram = false;
 
   public ParserOptions() {}
 
@@ -44,6 +47,33 @@ public boolean isActionViewHelpers() {
     return actionViewHelpers;
   }
 
+  public ParserOptions prismNodes(boolean value) {
+    this.prismNodes = value;
+    return this;
+  }
+
+  public boolean isPrismNodes() {
+    return prismNodes;
+  }
+
+  public ParserOptions prismNodesDeep(boolean value) {
+    this.prismNodesDeep = value;
+    return this;
+  }
+
+  public boolean isPrismNodesDeep() {
+    return prismNodesDeep;
+  }
+
+  public ParserOptions prismProgram(boolean value) {
+    this.prismProgram = value;
+    return this;
+  }
+
+  public boolean isPrismProgram() {
+    return prismProgram;
+  }
+
   public static ParserOptions create() {
     return new ParserOptions();
   }

java/org/herb/ast/BaseNode.java

@@ -30,6 +30,11 @@ public java.util.List<Node> getErrors() {
     return errors;
   }
 
+  @Override
+  public String inspect(String source) {
+    return inspect();
+  }
+
   @Override
   public <R, C> R accept(NodeVisitor<R, C> visitor, C context) {
     return visitor.visitNode(this, context);
@@ -79,6 +84,13 @@ protected String inspectErrors(String prefix) {
    * Helper to format an array of nodes for tree inspection.
    */
   protected String inspectArray(java.util.List<Node> array, String prefix) {
+    return inspectArray(array, prefix, null);
+  }
+
+  /**
+   * Helper to format an array of nodes for tree inspection with source for Prism nodes.
+   */
+  protected String inspectArray(java.util.List<Node> array, String prefix, String source) {
     if (array == null) return "∅\n";
     if (array.isEmpty()) return "[]\n";
 
@@ -92,7 +104,7 @@ protected String inspectArray(java.util.List<Node> array, String prefix) {
       String nextPrefix = isLast ? "    " : "│   ";
 
       if (item != null) {
-        String tree = item.inspect();
+        String tree = source != null ? item.inspect(source) : item.inspect();
 
         if (tree.endsWith("\n")) {
           tree = tree.substring(0, tree.length() - 1);
@@ -119,8 +131,15 @@ protected String inspectArray(java.util.List<Node> array, String prefix) {
    * Helper to format a single node for tree inspection.
    */
   protected String inspectNode(Node node, String prefix) {
+    return inspectNode(node, prefix, null);
+  }
+
+  /**
+   * Helper to format a single node for tree inspection with source for Prism nodes.
+   */
+  protected String inspectNode(Node node, String prefix, String source) {
     if (node == null) return "∅\n";
-    String tree = node.inspect();
+    String tree = source != null ? node.inspect(source) : node.inspect();
 
     if (tree.endsWith("\n")) {
       tree = tree.substring(0, tree.length() - 1);