parse comments

ensure fragments works

Author: sinnbeck

src/Ast/NodeType.php

@@ -8,4 +8,5 @@ enum NodeType
     case ELEMENT;
     case TEXT;
     case RAW;
+    case COMMENT;
 }

src/Ast/Parser.php

@@ -44,32 +44,34 @@ protected function parseNode(): ?Node
         }
         if ($token->type === TokenType::DOCTYPE) {
             $this->advance();
-
             return new Node(NodeType::DOCTYPE, '', [], [], $token->value);
         }
         if ($token->type === TokenType::TEXT) {
             $this->advance();
-
             return new Node(NodeType::TEXT, '', [], [], $token->value);
         }
+        // NEW: Parse a comment token and create a COMMENT node.
+        if ($token->type === TokenType::COMMENT) {
+            $this->advance();
+            return new Node(NodeType::COMMENT, '', [], [], $token->value);
+        }
         if ($token->type === TokenType::RAW) {
             $this->advance();
-
             return new Node(NodeType::RAW, '', [], [], $token->value);
         }
         if ($token->type === TokenType::TAG_OPEN) {
             return $this->parseElement();
         }
         if ($token->type === TokenType::TAG_CLOSE) {
             $this->consumeClosingTag();
-
             return null;
         }
         $this->advance();
 
         return null;
     }
 
+
     protected function parseElement(): Node
     {
         // Consume the open tag token (e.g. from '<script').

src/Lexer/Lexer.php

@@ -102,14 +102,22 @@ protected function consumeDoctype(): void
     protected function consumeComment(): void
     {
         $this->consume(4); // consumes '<!--'
+        $start = $this->position;
         $end = strpos($this->input, '-->', $this->position);
         if ($end === false) {
+            // If there is no closing tag, capture the rest of the input as the comment.
+            $commentText = substr($this->input, $start);
             $this->position = $this->length;
         } else {
-            $this->position = $end + 3;
+            // Extract the comment text (everything up to but not including '-->')
+            $commentText = substr($this->input, $start, $end - $start);
+            $this->position = $end + 3; // consume the closing '-->'
         }
+        // Add a token for the comment.
+        $this->addToken(TokenType::COMMENT, $commentText);
     }
 
+
     /**
      * Consume text until a '<' is encountered.
      * Only produces a token if the text contains at least one non-whitespace character.

src/Lexer/TokenType.php

@@ -13,4 +13,5 @@ enum TokenType
     case RAW;
     case ATTR_NAME;
     case ATTR_VALUE;
+    case COMMENT;
 }

src/Printer.php

@@ -39,7 +39,7 @@ public function render(array $nodes, int $level = 0): string
             $html .= $this->renderNode($node, $level);
         }
 
-        return $html;
+        return rtrim($html); //ensure there is no linebreak on last element
     }
 
     protected function renderNode(Node $node, int $level): string
@@ -56,6 +56,10 @@ protected function renderNode(Node $node, int $level): string
             return $this->renderRaw($node->content, $level);
         }
 
+        if ($node->type === NodeType::COMMENT) {
+            return $this->renderComment($node->content, $indent);
+        }
+
         // For plain text nodes.
         if ($node->type === NodeType::TEXT) {
             return $indent . $node->content . $this->newline;
@@ -133,4 +137,9 @@ protected function renderRaw(string $raw, int $level): string
 
         return implode($this->newline, $newLines) . $this->newline;
     }
+
+    private function renderComment(string $content, string $indent)
+    {
+        return $indent . '<!--' . $content . '-->' . $this->newline;
+    }
 }

tests/Feature/LexerTest.php

@@ -5,24 +5,24 @@
 
 it('can read an html file and output a tokens array', function () {
     $html = getFixture('basic.html');
-    $lexer  = new Lexer($html);
+    $lexer = new Lexer($html);
     expect($lexer->lex())->toHaveCount(69);
 });
 
 it('has the same output no matter format of the input', function () {
     $html = getFixture('basic.html');
-    $lexer  = Lexer::fromString($html);
+    $lexer = Lexer::fromString($html);
     $tokens1 = $lexer->lex();
 
     $html = getFixture('basic-scrambled.html');
-    $lexer  = Lexer::fromString($html);
+    $lexer = Lexer::fromString($html);
     $tokens2 = $lexer->lex();
     expect($tokens1)->toEqual($tokens2);
 });
 
 it('can read get open and closing div tags', function () {
     $html = getFixture('basic.html');
-    $lexer  = new Lexer($html);
+    $lexer = new Lexer($html);
     $tokens = $lexer->lex();
 
     //Opening div tag
@@ -42,7 +42,7 @@
 
 it('can read get open and closing script tags', function () {
     $html = getFixture('complex.html');
-    $lexer  = new Lexer($html);
+    $lexer = new Lexer($html);
     $tokens = $lexer->lex();
 
     //Opening script tag
@@ -58,4 +58,18 @@
     expect($tokens[62])
         ->toHaveKey('value', 'script')
         ->toHaveKey('type', TokenType::TAG_CLOSE);
+});
+
+it('can handle comments', function () {
+    $lexer = Lexer::fromString(getFixture('comments.html'));
+
+    $tokens = $lexer->lex();
+
+    expect($tokens[7])
+        ->toHaveKey('type', TokenType::COMMENT)
+        ->toHaveKey('value', ' title tag ');
+
+    expect($tokens[13])
+        ->toHaveKey('type', TokenType::COMMENT)
+        ->toHaveKey('value', ' Body ');
 });

tests/Feature/PrintTest.php

@@ -37,6 +37,30 @@
 
     $nodes = $ast->parse();
 
+    $printer = new Printer();
+    expect($printer->render($nodes))->toEqual($html);
+});
+
+it('can print fragments', function () {
+    $html = getFixture('fragment.html');
+    $lexer = new Lexer($html);
+
+    $ast = new Parser($lexer->lex());
+
+    $nodes = $ast->parse();
+
+    $printer = new Printer();
+    expect($printer->render($nodes))->toEqual($html);
+});
+
+it('can print comments', function () {
+    $html = getFixture('comments.html');
+    $lexer = new Lexer($html);
+
+    $ast = new Parser($lexer->lex());
+
+    $nodes = $ast->parse();
+
     $printer = new Printer();
     expect($printer->render($nodes))->toEqual($html);
 });

tests/fixtures/basic-scrambled.html

@@ -22,4 +22,4 @@
         Smaller text
 
         </div>
-</body></html>
+</body></html>

tests/fixtures/basic.html

@@ -19,4 +19,4 @@
             Smaller text
         </div>
     </body>
-</html>
+</html>

tests/fixtures/broken.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+    <head>
+        <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
+        <title>
+            Test page
+        </title>
+    </head>
+    <body style="width:1024px;" class="h-full">
+        <div class="wrapper" style="font-size: 10px">
+            Smaller text
+    </body>
+</html>