Improve lexing of HTML comments

Author: marcoroth

ext/erbx/test/lexer/tags_test.rb

@@ -525,5 +525,50 @@ class TagsTest < Minitest::Spec
 
       assert_equal expected, result.array.items.map(&:type)
     end
+
+    test "HTML comment followed by html tag" do
+      result = ERBX.lex(%(<!--Hello World--><h1>Hello</h1>))
+
+      expected = %w[
+        TOKEN_HTML_COMMENT_START
+        TOKEN_HTML_COMMENT_CONTENT
+        TOKEN_HTML_COMMENT_END
+        TOKEN_HTML_TAG_START
+        TOKEN_HTML_TAG_NAME
+        TOKEN_HTML_TAG_END
+        TOKEN_TEXT_CONTENT
+        TOKEN_HTML_CLOSE_TAG_START
+        TOKEN_HTML_TAG_NAME
+        TOKEN_HTML_TAG_END
+        TOKEN_EOF
+      ]
+
+      assert_equal expected, result.array.items.map(&:type)
+    end
+
+    xtest "HTML comment followed by html tag with nested comment" do
+      result = ERBX.lex(%(
+        <!--Hello World-->
+        <h1><!-- Hello World --></h1>
+      ))
+
+      expected = %w[
+        TOKEN_HTML_COMMENT_START
+        TOKEN_HTML_COMMENT_CONTENT
+        TOKEN_HTML_COMMENT_END
+        TOKEN_HTML_TAG_START
+        TOKEN_HTML_TAG_NAME
+        TOKEN_HTML_TAG_END
+        TOKEN_HTML_COMMENT_START
+        TOKEN_HTML_COMMENT_CONTENT
+        TOKEN_HTML_COMMENT_END
+        TOKEN_HTML_CLOSE_TAG_START
+        TOKEN_HTML_TAG_NAME
+        TOKEN_HTML_TAG_END
+        TOKEN_EOF
+      ]
+
+      assert_equal expected, result.array.items.map(&:type)
+    end
   end
 end

src/lexer.c

@@ -434,6 +434,7 @@ token_T* lexer_handle_html_comment_close_state(lexer_T* lexer) {
     lexer_advance(lexer);
     lexer_advance(lexer);
     lexer_advance(lexer);
+    lexer->state = STATE_DATA;
     return token_init("-->", TOKEN_HTML_COMMENT_END, lexer);
   }