Parser: Expose Prism Nodes in Herb Syntax Tree (#1350)

This pull request updates the parser to be able expose the Prism Nodes
as part of the Herb Syntax Tree. We now expose three new parser options
that control the way the parser exposes the Prism Nodes it sees:
`prism_nodes`, `prism_nodes_deep`, and `prism_program`.

* `prism_nodes` exposes the Prism nodes for each individual `ERB*` node
on the node itself
* `prism_nodes_deep` does the same thing as `prism_nodes`, but it also
includes all nodes of all children inside of itself. For `<%= tag.div do
%><%= user.name %><% end %>` the `tag.div` ERB block node will also have
the `user.name` nodes as part of the `prism_node` field on itself.
* `prism_program` extracts the Ruby from the whole file and passes that
to Prism. It then exposes the whole template on the `DocumentNode` on
the `prism_node` field.

For example for a template like: 
```html+erb
<h1><%= @post.title %></h1>
```

The parser will now produces the following using `prism_nodes: true`:
```js
@ DocumentNode (location: (1:0)-(1:27))
└── children: (1 item)
    └── @ HTMLElementNode (location: (1:0)-(1:27))
        ├── open_tag: 
        │   └── @ HTMLOpenTagNode (location: (1:0)-(1:4))
        │       ├── tag_opening: "<" (location: (1:0)-(1:1))
        │       ├── tag_name: "h1" (location: (1:1)-(1:3))
        │       ├── tag_closing: ">" (location: (1:3)-(1:4))
        │       ├── children: []
        │       └── is_void: false
        │       
        ├── tag_name: "h1" (location: (1:1)-(1:3))
        ├── body: (1 item)
        │   └── @ ERBContentNode (location: (1:4)-(1:22))
        │       ├── tag_opening: "<%=" (location: (1:4)-(1:7))
        │       ├── content: " @post.title " (location: (1:7)-(1:20))
        │       ├── tag_closing: "%>" (location: (1:20)-(1:22))
        │       ├── parsed: true
        │       ├── valid: true
        │       └── prism_node: 
        │           └── @ CallNode (location: (1:8)-(1:19))
        │               ├── receiver:
        │               │   └── @ InstanceVariableReadNode (location: (1:8)-(1:13))
        │               │       └── name: "@post"
        │               ├── callOperatorLoc: (location: (1:13)-(1:14))
        │               ├── name: "title"
        │               ├── messageLoc: (location: (1:14)-(1:19))
        │               ├── openingLoc: ∅
        │               ├── arguments_: ∅
        │               ├── closingLoc: ∅
        │               ├── equalLoc: ∅
        │               └── block: ∅
        │   
        ├── close_tag: 
        │   └── @ HTMLCloseTagNode (location: (1:22)-(1:27))
        │       ├── tag_opening: "</" (location: (1:22)-(1:24))
        │       ├── tag_name: "h1" (location: (1:24)-(1:26))
        │       ├── children: []
        │       └── tag_closing: ">" (location: (1:26)-(1:27))
        │       
        ├── is_void: false
        └── element_source: "HTML"
```

Resolves #1036
Resolves #1121

Author: marcoroth

.rubocop.yml

@@ -58,7 +58,9 @@ Metrics/CyclomaticComplexity:
     - lib/herb/ast/nodes.rb
     - lib/herb/cli.rb
     - lib/herb/engine.rb
+    - lib/herb/prism_inspect.rb
     - lib/herb/engine/**/*.rb
+    - templates/template.rb
 
 Metrics/MethodLength:
   Max: 20
@@ -68,6 +70,7 @@ Metrics/MethodLength:
     - lib/herb/project.rb
     - lib/herb/engine.rb
     - lib/herb/engine/**/*.rb
+    - lib/herb/prism_inspect.rb
     - templates/template.rb
     - test/fork_helper.rb
     - test/snapshot_utils.rb
@@ -83,6 +86,7 @@ Metrics/AbcSize:
     - lib/herb/engine.rb
     - lib/herb/engine/**/*.rb
     - lib/herb/token.rb
+    - lib/herb/prism_inspect.rb
     - templates/template.rb
     - test/fork_helper.rb
     - test/snapshot_utils.rb
@@ -98,6 +102,7 @@ Metrics/ClassLength:
     - lib/herb/engine/**/*.rb
     - lib/herb/project.rb
     - lib/herb/visitor.rb
+    - lib/herb/ast/nodes.rb
     - test/**/*_test.rb
 
 Metrics/ModuleLength:
@@ -121,14 +126,17 @@ Metrics/ParameterLists:
   Exclude:
     - lib/herb/ast/node.rb
     - lib/herb/ast/nodes.rb
-    - lib/herb/errors.rb
     - lib/herb/engine/validators/security_validator.rb
+    - lib/herb/errors.rb
+    - lib/herb/parser_options.rb
+    - lib/herb/prism_inspect.rb
 
 Metrics/PerceivedComplexity:
   Exclude:
     - lib/herb/ast/nodes.rb
     - lib/herb/cli.rb
     - lib/herb/project.rb
+    - lib/herb/prism_inspect.rb
     - lib/herb/engine.rb
     - lib/herb/engine/**/*.rb
     - templates/template.rb

config.yml

@@ -369,6 +369,12 @@ nodes:
           type: array
           kind: Node
 
+        - name: prism_context
+          type: prism_context
+
+        - name: prism_node
+          type: prism_node
+
     - name: LiteralNode
       fields:
         - name: content
@@ -643,6 +649,9 @@ nodes:
         - name: valid
           type: boolean
 
+        - name: prism_node
+          type: prism_node
+
     - name: ERBEndNode
       fields:
         - name: tag_opening
@@ -686,6 +695,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -717,6 +729,9 @@ nodes:
         # - name: opener
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: body
           type: array
           kind: Node
@@ -765,6 +780,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: conditions
           type: array
           kind: ERBWhenNode
@@ -795,6 +813,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: conditions
           type: array
           kind: ERBInNode
@@ -821,6 +842,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -843,6 +867,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -868,6 +895,9 @@ nodes:
         # - name: collection
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -928,6 +958,9 @@ nodes:
         - name: tag_closing
           type: token
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node
@@ -965,6 +998,9 @@ nodes:
         # - name: predicate
         #   type: prism_node
 
+        - name: prism_node
+          type: prism_node
+
         - name: statements
           type: array
           kind: Node

ext/herb/extconf.rb

@@ -27,7 +27,6 @@
 $CFLAGS << " -DPRISM_EXCLUDE_PRETTYPRINT"
 $CFLAGS << " -DPRISM_EXCLUDE_JSON"
 $CFLAGS << " -DPRISM_EXCLUDE_PACK"
-$CFLAGS << " -DPRISM_EXCLUDE_SERIALIZATION"
 
 herb_src_files = Dir.glob("#{$srcdir}/../../src/**/*.c").map { |file| file.delete_prefix("../../../../ext/herb/") }.sort
 

ext/herb/extension.c

@@ -136,6 +136,18 @@ static VALUE Herb_parse(int argc, VALUE* argv, VALUE self) {
     }
     if (!NIL_P(action_view_helpers) && RTEST(action_view_helpers)) { parser_options.action_view_helpers = true; }
 
+    VALUE prism_nodes = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_nodes"));
+    if (NIL_P(prism_nodes)) { prism_nodes = rb_hash_lookup(options, ID2SYM(rb_intern("prism_nodes"))); }
+    if (!NIL_P(prism_nodes) && RTEST(prism_nodes)) { parser_options.prism_nodes = true; }
+
+    VALUE prism_nodes_deep = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_nodes_deep"));
+    if (NIL_P(prism_nodes_deep)) { prism_nodes_deep = rb_hash_lookup(options, ID2SYM(rb_intern("prism_nodes_deep"))); }
+    if (!NIL_P(prism_nodes_deep) && RTEST(prism_nodes_deep)) { parser_options.prism_nodes_deep = true; }
+
+    VALUE prism_program = rb_hash_lookup(options, rb_utf8_str_new_cstr("prism_program"));
+    if (NIL_P(prism_program)) { prism_program = rb_hash_lookup(options, ID2SYM(rb_intern("prism_program"))); }
+    if (!NIL_P(prism_program) && RTEST(prism_program)) { parser_options.prism_program = true; }
+
     VALUE arena_stats = rb_hash_lookup(options, rb_utf8_str_new_cstr("arena_stats"));
     if (NIL_P(arena_stats)) { arena_stats = rb_hash_lookup(options, ID2SYM(rb_intern("arena_stats"))); }
     if (!NIL_P(arena_stats) && RTEST(arena_stats)) { print_arena_stats = true; }

ext/herb/extension_helpers.c

@@ -89,6 +89,9 @@ VALUE create_parse_result(AST_DOCUMENT_NODE_T* root, VALUE source, const parser_
   rb_hash_aset(kwargs, ID2SYM(rb_intern("track_whitespace")), options->track_whitespace ? Qtrue : Qfalse);
   rb_hash_aset(kwargs, ID2SYM(rb_intern("analyze")), options->analyze ? Qtrue : Qfalse);
   rb_hash_aset(kwargs, ID2SYM(rb_intern("action_view_helpers")), options->action_view_helpers ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_nodes")), options->prism_nodes ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_nodes_deep")), options->prism_nodes_deep ? Qtrue : Qfalse);
+  rb_hash_aset(kwargs, ID2SYM(rb_intern("prism_program")), options->prism_program ? Qtrue : Qfalse);
 
   VALUE parser_options_args[1] = { kwargs };
   VALUE parser_options = rb_class_new_instance_kw(1, parser_options_args, cParserOptions, RB_PASS_KEYWORDS);

java/Makefile

@@ -30,7 +30,7 @@ PRISM_BUILD = $(PRISM_PATH)/build
 
 JAVAC = $(JAVA_HOME)/bin/javac
 JAVA_CMD = $(JAVA_HOME)/bin/java
-CFLAGS = -std=c99 -Wall -Wextra -fPIC -O2 -DHERB_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_JSON -DPRISM_EXCLUDE_PACK -DPRISM_EXCLUDE_SERIALIZATION
+CFLAGS = -std=c99 -Wall -Wextra -fPIC -O2 -DHERB_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_PRETTYPRINT -DPRISM_EXCLUDE_JSON -DPRISM_EXCLUDE_PACK
 INCLUDES = -I. -I$(SRC_DIR)/include -I$(PRISM_INCLUDE) $(JNI_INCLUDES)
 LDFLAGS = -shared
 LIBS = $(PRISM_BUILD)/libprism.a

java/herb_jni.c

@@ -68,6 +68,30 @@ Java_org_herb_Herb_parse(JNIEnv* env, jclass clazz, jstring source, jobject opti
       jboolean actionViewHelpers = (*env)->CallBooleanMethod(env, options, getActionViewHelpers);
       parser_options.action_view_helpers = (actionViewHelpers == JNI_TRUE);
     }
+
+    jmethodID getPrismNodes =
+        (*env)->GetMethodID(env, optionsClass, "isPrismNodes", "()Z");
+
+    if (getPrismNodes != NULL) {
+      jboolean prismNodes = (*env)->CallBooleanMethod(env, options, getPrismNodes);
+      parser_options.prism_nodes = (prismNodes == JNI_TRUE);
+    }
+
+    jmethodID getPrismNodesDeep =
+        (*env)->GetMethodID(env, optionsClass, "isPrismNodesDeep", "()Z");
+
+    if (getPrismNodesDeep != NULL) {
+      jboolean prismNodesDeep = (*env)->CallBooleanMethod(env, options, getPrismNodesDeep);
+      parser_options.prism_nodes_deep = (prismNodesDeep == JNI_TRUE);
+    }
+
+    jmethodID getPrismProgram =
+        (*env)->GetMethodID(env, optionsClass, "isPrismProgram", "()Z");
+
+    if (getPrismProgram != NULL) {
+      jboolean prismProgram = (*env)->CallBooleanMethod(env, options, getPrismProgram);
+      parser_options.prism_program = (prismProgram == JNI_TRUE);
+    }
   }
 
   hb_allocator_T allocator;

java/org/herb/HerbTest.java

@@ -102,6 +102,51 @@ void testParserOptionsTrackWhitespace() {
     assertFalse(inspectWithout.contains("WhitespaceNode"));
   }
 
+  @Test
+  void testParserOptionsPrismNodes() {
+    String source = "<%= user.name %>";
+
+    ParseResult withPrism = Herb.parse(source, ParserOptions.create().prismNodes(true));
+    String inspected = withPrism.inspect();
+
+    assertTrue(inspected.contains("prism_node:"));
+  }
+
+  @Test
+  void testParserOptionsPrismNodesString() {
+    String source = "<%= \"String\" %>";
+
+    ParseResult withPrism = Herb.parse(source, ParserOptions.create().prismNodes(true));
+    String inspected = withPrism.inspect();
+
+    assertTrue(inspected.contains("prism_node:"));
+    assertTrue(inspected.contains("String"));
+  }
+
+  @Test
+  void testParserOptionsPrismProgram() {
+    String source = "<%= \"hello\" %>";
+
+    ParseResult withPrism = Herb.parse(source, ParserOptions.create().prismProgram(true));
+    String inspected = withPrism.inspect();
+
+    assertTrue(inspected.contains("prism_node:"));
+  }
+
+  @Test
+  void testParserOptionsPrismNodesDeep() {
+    String source = "<% if true %><%= \"yes\" %><% end %>";
+
+    ParseResult withDeep = Herb.parse(source, ParserOptions.create().prismNodes(true).prismNodesDeep(true));
+    ParseResult withoutDeep = Herb.parse(source, ParserOptions.create().prismNodes(true).prismNodesDeep(false));
+
+    String inspectDeep = withDeep.inspect();
+    String inspectShallow = withoutDeep.inspect();
+
+    assertTrue(inspectDeep.contains("prism_node:"));
+    assertTrue(inspectShallow.contains("prism_node:"));
+  }
+
   @Test
   void testParserOptionsAnalyze() {
     String source = "<% if true %><div></div><% end %>";

java/org/herb/ParseResult.java

@@ -50,7 +50,7 @@ public String inspect() {
     StringBuilder builder = new StringBuilder();
 
     if (value != null) {
-      builder.append(value.inspect());
+      builder.append(value.inspect(source));
     }
 
     if (hasErrors()) {

java/org/herb/ParserOptions.java

@@ -5,6 +5,9 @@ public class ParserOptions {
   private boolean analyze = true;
   private boolean strict = true;
   private boolean actionViewHelpers = false;
+  private boolean prismNodes = false;
+  private boolean prismNodesDeep = false;
+  private boolean prismProgram = false;
 
   public ParserOptions() {}
 
@@ -44,6 +47,33 @@ public boolean isActionViewHelpers() {
     return actionViewHelpers;
   }
 
+  public ParserOptions prismNodes(boolean value) {
+    this.prismNodes = value;
+    return this;
+  }
+
+  public boolean isPrismNodes() {
+    return prismNodes;
+  }
+
+  public ParserOptions prismNodesDeep(boolean value) {
+    this.prismNodesDeep = value;
+    return this;
+  }
+
+  public boolean isPrismNodesDeep() {
+    return prismNodesDeep;
+  }
+
+  public ParserOptions prismProgram(boolean value) {
+    this.prismProgram = value;
+    return this;
+  }
+
+  public boolean isPrismProgram() {
+    return prismProgram;
+  }
+
   public static ParserOptions create() {
     return new ParserOptions();
   }