Support UTF-8 characters in ERB blocks when pretty printing AST (#93)

Closes #92

Author: marcoroth

lib/herb/token.rb

@@ -25,7 +25,7 @@ def to_json(*args)
     end
 
     def tree_inspect
-      %("#{value}" #{location.tree_inspect})
+      %("#{value.force_encoding("utf-8")}" #{location.tree_inspect})
     end
 
     def value_inspect

test/parser/utf8_test.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require_relative "../test_helper"
+
+module Parser
+  class UTF8Test < Minitest::Spec
+    include SnapshotUtils
+
+    test "opening guillemet" do
+      assert_parsed_snapshot(<<~ERB)
+        <%= link_to '«', url %>
+      ERB
+    end
+
+    test "closing guillemet" do
+      assert_parsed_snapshot(<<~ERB)
+        <%= link_to '»', url %>
+      ERB
+    end
+
+    test "single opening guillemet" do
+      assert_parsed_snapshot(<<~ERB)
+        <%= link_to '‹', url %>
+      ERB
+    end
+
+    test "single closing guillemet" do
+      assert_parsed_snapshot(<<~ERB)
+        <%= link_to '›', url %>
+      ERB
+    end
+  end
+end

test/snapshots/parser/utf8_test/test_0001_opening_guillemet_856d313d5fa2c08f87ea3b9a79a870b0.txt

@@ -0,0 +1,11 @@
+@ DocumentNode (location: (1:0)-(2:0))
+└── children: (2 items)
+    ├── @ ERBContentNode (location: (1:0)-(1:24))
+    │   ├── tag_opening: "<%=" (location: (1:0)-(1:3))
+    │   ├── content: " link_to '«', url " (location: (1:3)-(1:22))
+    │   ├── tag_closing: "%>" (location: (1:22)-(1:24))
+    │   ├── parsed: true
+    │   └── valid: true
+    │
+    └── @ HTMLTextNode (location: (1:24)-(2:0))
+        └── content: "\n"

test/snapshots/parser/utf8_test/test_0002_closing_guillemet_46999175f7a57d90bf4ac8417289630d.txt

@@ -0,0 +1,11 @@
+@ DocumentNode (location: (1:0)-(2:0))
+└── children: (2 items)
+    ├── @ ERBContentNode (location: (1:0)-(1:24))
+    │   ├── tag_opening: "<%=" (location: (1:0)-(1:3))
+    │   ├── content: " link_to '»', url " (location: (1:3)-(1:22))
+    │   ├── tag_closing: "%>" (location: (1:22)-(1:24))
+    │   ├── parsed: true
+    │   └── valid: true
+    │
+    └── @ HTMLTextNode (location: (1:24)-(2:0))
+        └── content: "\n"

test/snapshots/parser/utf8_test/test_0003_single_opening_guillemet_39365d03f7c1b51be8ba359dcdda448d.txt

@@ -0,0 +1,11 @@
+@ DocumentNode (location: (1:0)-(2:0))
+└── children: (2 items)
+    ├── @ ERBContentNode (location: (1:0)-(1:25))
+    │   ├── tag_opening: "<%=" (location: (1:0)-(1:3))
+    │   ├── content: " link_to '‹', url " (location: (1:3)-(1:23))
+    │   ├── tag_closing: "%>" (location: (1:23)-(1:25))
+    │   ├── parsed: true
+    │   └── valid: true
+    │
+    └── @ HTMLTextNode (location: (1:25)-(2:0))
+        └── content: "\n"

test/snapshots/parser/utf8_test/test_0004_single_closing_guillemet_9259b210ee4ee3e7703e1a42face4fcf.txt

@@ -0,0 +1,11 @@
+@ DocumentNode (location: (1:0)-(2:0))
+└── children: (2 items)
+    ├── @ ERBContentNode (location: (1:0)-(1:25))
+    │   ├── tag_opening: "<%=" (location: (1:0)-(1:3))
+    │   ├── content: " link_to '›', url " (location: (1:3)-(1:23))
+    │   ├── tag_closing: "%>" (location: (1:23)-(1:25))
+    │   ├── parsed: true
+    │   └── valid: true
+    │
+    └── @ HTMLTextNode (location: (1:25)-(2:0))
+        └── content: "\n"