Parse t-strings into TNodes/TAttributes representation using TemplateParser (#82)

* Parse t-strings into intermediary TNodes/TAttributes representation using TemplateParser.

* Move lru cache from processor into parser with a few tests.

Author: ianjosephwilson

tdom/escaping.py

@@ -0,0 +1,90 @@
+import re
+from string.templatelib import Interpolation
+
+from markupsafe import Markup
+
+from .utils import format_interpolation as base_format_interpolation
+
+
+def _format_safe(value: object, format_spec: str) -> str:
+    """Use Markup() to mark a value as safe HTML."""
+    assert format_spec == "safe"
+    return Markup(value)
+
+
+def _format_unsafe(value: object, format_spec: str) -> str:
+    """Convert a value to a plain string, forcing it to be treated as unsafe."""
+    assert format_spec == "unsafe"
+    return str(value)
+
+
+CUSTOM_FORMATTERS = (("safe", _format_safe), ("unsafe", _format_unsafe))
+
+
+def format_interpolation(interpolation: Interpolation) -> object:
+    return base_format_interpolation(
+        interpolation,
+        formatters=CUSTOM_FORMATTERS,
+    )
+
+
+def escape_html_comment(text):
+    """Escape text injected into an HTML comment."""
+    GT = ">"
+    LT = "<"
+
+    if not text:
+        return text
+    # - text must not start with the string ">"
+    if text[0] == ">":
+        text = GT + text[1:]
+
+    # - nor start with the string "->"
+    if text[:2] == "->":
+        text = "-" + GT + text[2:]
+
+    # - nor contain the strings "<!--", "-->", or "--!>"
+    if (index := text.find("<!--")) and index != -1:
+        text = text[:index] + LT + text[index + 1]
+    if (index := text.find("-->")) and index != -1:
+        text = text[: index + 2] + GT + text[index + 3]
+    if (index := text.find("--!>")) and index != -1:
+        text = text[: index + 3] + GT + text[index + 4]
+
+    # - nor end with the string "<!-".
+    if text[-3:] == "<!-":
+        text = text[:-3] + LT + "!-"
+
+    return text
+
+
+def escape_html_style(text):
+    LT = "&lt;"
+    close_str = "</style>"
+    close_str_re = re.compile(close_str, re.I | re.A)
+    replace_str = LT + close_str[1:]
+    return re.sub(close_str_re, replace_str, text)
+
+
+def escape_html_script(text):
+    """
+    https://html.spec.whatwg.org/multipage/scripting.html#restrictions-for-contents-of-script-elements
+
+    (from link) The easiest and safest way to avoid the rather strange restrictions
+    described in this section is to always escape an ASCII case-insensitive
+    match for:
+    - "<!--" as "\x3c!--"
+    - "<script" as "\x3cscript"
+    - "</script" as "\x3c/script"`
+
+    This does not make a script *run*; it just tries to prevent accidentally injecting
+    *another* SCRIPT tag into a SCRIPT tag being rendered.
+    """
+    match_to_replace = (
+        (re.compile("<!--", re.I | re.A), "\x3c!--"),
+        (re.compile("<script", re.I | re.A), "\x3cscript"),
+        (re.compile("</script", re.I | re.A), "\x3c/script"),
+    )
+    for match_re, replace_text in match_to_replace:
+        text = re.sub(match_re, replace_text, text)
+    return text

tdom/nodes.py

@@ -1,7 +1,15 @@
 from dataclasses import dataclass, field
+from string.templatelib import Template
 
 from markupsafe import escape
 
+from .escaping import (
+    escape_html_comment,
+    escape_html_script,
+    escape_html_style,
+)
+
+
 # See https://developer.mozilla.org/en-US/docs/Glossary/Void_element
 VOID_ELEMENTS = frozenset(
     [
@@ -31,8 +39,125 @@
 # FUTURE: make nodes frozen (and have the parser work with mutable builders)
 
 
+def to_template_repr(template):
+    """
+    Convert a template to a comparable representation.
+
+    This is mostly for testing because Templates/Interpolations are not comparable.
+    """
+    parts = []
+    for index, s in enumerate(template.strings):
+        parts.append(s)
+        if index < len(template.strings) - 1:
+            ip = template.interpolations[index]
+            parts.append((ip.value, ip.expression, ip.conversion, ip.format_spec))
+    return tuple(parts)
+
+
+@dataclass
+class TNodeBase:
+    def __str__(self) -> str:
+        raise NotImplementedError("Cannot serialize dynamic nodes.")
+
+    def __html__(self) -> str:
+        raise NotImplementedError("Cannot serialize dynamic nodes.")
+
+
+type TAttribute = (
+    StaticAttribute | SpreadAttribute | TemplatedAttribute | InterpolatedAttribute
+)
+
+
+@dataclass
+class StaticAttribute:
+    name: str
+    value: str | None = None
+
+
+@dataclass
+class SpreadAttribute:
+    interpolation_index: int
+
+
+@dataclass
+class TemplatedAttribute:
+    name: str
+    value_t: Template
+
+    def to_comparable(self):
+        return (self.name, to_template_repr(self.value_t))
+
+    def __eq__(self, other: object):
+        return (
+            isinstance(other, TemplatedAttribute)
+            and self.to_comparable() == other.to_comparable()
+        )
+
+
+@dataclass
+class InterpolatedAttribute:
+    name: str
+    interpolation_index: int
+
+
+type TNode = TElement | TComponent | TFragment | TText | TComment | TDocumentType
+
+
+@dataclass
+class TDocumentType(TNodeBase):
+    text: str
+
+
+@dataclass
+class TElement(TNodeBase):
+    tag: str
+    attrs: tuple[TAttribute, ...] = field(default_factory=tuple)
+    children: tuple[TNode, ...] = field(default_factory=tuple)
+
+
+@dataclass
+class TFragment(TNodeBase):
+    children: tuple[TNode, ...] = field(default_factory=tuple)
+
+
+@dataclass
+class TComponent(TNodeBase):
+    starttag_interpolation_index: int
+    endtag_interpolation_index: int
+    starttag_string_index: (
+        int  # string index where the starttag > or startendtag /> occurs.
+    )
+    endtag_string_index: (
+        int  # string index where the endtag > or startendtag /> occurs.
+    )
+    attrs: tuple[TAttribute, ...] = field(default_factory=tuple)
+    children: tuple[TNode, ...] = field(default_factory=tuple)
+
+
+@dataclass
+class TText(TNodeBase):
+    text_t: Template
+
+    def __eq__(self, other: object) -> bool:
+        # This is primarily of use for testing purposes. We only consider
+        # two Text nodes equal if their string representations match.
+        return isinstance(other, TText) and to_template_repr(
+            self.text_t
+        ) == to_template_repr(other.text_t)
+
+
+@dataclass
+class TComment(TNodeBase):
+    text_t: Template
+
+    def __eq__(self, other: object) -> bool:
+        return isinstance(other, TComment) and to_template_repr(
+            self.text_t
+        ) == to_template_repr(other.text_t)
+
+
 @dataclass(slots=True)
-class Node:
+class Node(TNodeBase):
     def __html__(self) -> str:
         """Return the HTML representation of the node."""
         # By default, just return the string representation
@@ -66,7 +191,7 @@ class Comment(Node):
     text: str
 
     def __str__(self) -> str:
-        return f"<!--{self.text}-->"
+        return f"<!--{escape_html_comment(self.text)}-->"
 
 
 @dataclass(slots=True)
@@ -100,6 +225,28 @@ def is_void(self) -> bool:
     def is_content(self) -> bool:
         return self.tag in CONTENT_ELEMENTS
 
+    def _children_to_str(self):
+        if not self.children:
+            return ""
+        if self.tag in ("script", "style"):
+            chunks = []
+            for child in self.children:
+                if isinstance(child, Text):
+                    chunks.append(child.text)
+                else:
+                    raise ValueError(
+                        "Cannot serialize non-text content inside a script tag."
+                    )
+            raw_children_str = "".join(chunks)
+            if self.tag == "script":
+                return escape_html_script(raw_children_str)
+            elif self.tag == "style":
+                return escape_html_style(raw_children_str)
+            else:
+                raise ValueError("Unsupported tag for single-level bulk escaping.")
+        else:
+            return "".join(str(child) for child in self.children)
+
     def __str__(self) -> str:
         # We use markupsafe's escape to handle HTML escaping of attribute values
         # which means it's possible to mark them as safe if needed.
@@ -111,5 +258,5 @@ def __str__(self) -> str:
             return f"<{self.tag}{attrs_str} />"
         if not self.children:
             return f"<{self.tag}{attrs_str}></{self.tag}>"
-        children_str = "".join(str(child) for child in self.children)
+        children_str = self._children_to_str()
         return f"<{self.tag}{attrs_str}>{children_str}</{self.tag}>"