fix: use SAFE_PROTOCOLS instead of HARMFUL_PROTOCOLS

Author: lepture

src/mistune/renderers/html.py

@@ -1,4 +1,4 @@
-from typing import Any, ClassVar, Dict, Optional, Tuple, Literal
+from typing import Any, ClassVar, Dict, Iterable, Optional, Tuple, Union, Literal
 from urllib.parse import unquote
 from ..core import BaseRenderer, BlockState
 from ..util import escape as escape_text
@@ -9,20 +9,17 @@ class HTMLRenderer(BaseRenderer):
     """A renderer for converting Markdown to HTML."""
 
     _escape: bool
+    _allow_harmful_protocols: Optional[Union[bool, Iterable[str]]]
     NAME: ClassVar[Literal["html"]] = "html"
-    HARMFUL_PROTOCOLS: ClassVar[Tuple[str, ...]] = (
-        "javascript:",
-        "vbscript:",
-        "file:",
-        "data:",
-        "feed:",
-        "jar:",
-        "livescript:",
-        "mocha:",
-        "ms-its:",
-        "mk:",
-        "res:",
-        "view-source:",
+    SAFE_PROTOCOLS: ClassVar[Tuple[str, ...]] = (
+        "http:",
+        "https:",
+        "mailto:",
+        "tel:",
+        "ftp:",
+        "ftps:",
+        "irc:",
+        "ircs:",
     )
     GOOD_DATA_PROTOCOLS: ClassVar[Tuple[str, ...]] = (
         "data:image/gif;",
@@ -31,7 +28,11 @@ class HTMLRenderer(BaseRenderer):
         "data:image/webp;",
     )
 
-    def __init__(self, escape: bool = True, allow_harmful_protocols: Optional[bool] = None) -> None:
+    def __init__(
+        self,
+        escape: bool = True,
+        allow_harmful_protocols: Optional[Union[bool, Iterable[str]]] = None,
+    ) -> None:
         super(HTMLRenderer, self).__init__()
         self._allow_harmful_protocols = allow_harmful_protocols
         self._escape = escape
@@ -59,16 +60,17 @@ def safe_url(self, url: str) -> str:
         """Ensure the given URL is safe. This method is used for rendering
         links, images, and etc.
         """
-        if self._allow_harmful_protocols is True:
+        allow_harmful_protocols = self._allow_harmful_protocols
+        if allow_harmful_protocols is True:
             return escape_text(url)
 
-        _url = _unquote_url(url).lower()
-        if self._allow_harmful_protocols and _url.startswith(tuple(self._allow_harmful_protocols)):
+        _url = _unquote_url(url).lower().lstrip()
+        if allow_harmful_protocols and _url.startswith(tuple(allow_harmful_protocols)):
             return escape_text(url)
 
-        if _url.startswith(self.HARMFUL_PROTOCOLS) and not _url.startswith(self.GOOD_DATA_PROTOCOLS):
-            return "#harmful-link"
-        return escape_text(url)
+        if _is_safe_url(_url, self.SAFE_PROTOCOLS, self.GOOD_DATA_PROTOCOLS):
+            return escape_text(url)
+        return "#harmful-link"
 
     def text(self, text: str) -> str:
         if self._escape:
@@ -169,3 +171,13 @@ def _unquote_url(url: str) -> str:
             break
         url = decoded
     return url
+
+
+def _is_safe_url(url: str, safe_protocols: Tuple[str, ...], good_data_protocols: Tuple[str, ...]) -> bool:
+    if url.startswith(safe_protocols):
+        return True
+    if url.startswith(good_data_protocols):
+        return True
+    if url.startswith(("/", "#", "?")):
+        return True
+    return ":" not in url.split("/", 1)[0]

tests/test_commonmark.py

@@ -3,12 +3,12 @@
 
 
 class TestCommonMark(BaseTestCase):
-    @classmethod
-    def ignore_case(cls, n):
-        return False
+    markdown = mistune.create_markdown(
+        renderer=mistune.HTMLRenderer(escape=False, allow_harmful_protocols=True),
+    )
 
     def assert_case(self, n, text, html):
-        result = mistune.html(text)
+        result = self.markdown(text)
         self.assertEqual(normalize_html(result), normalize_html(html))
 
 

tests/test_security_urls.py

@@ -1,20 +1,76 @@
 from unittest import TestCase
 
-from mistune import create_markdown
+from mistune import create_markdown, html
 
 
 class TestSafeUrlSecurity(TestCase):
+    def test_known_harmful_url_schemes_are_blocked(self):
+        for url in [
+            "javascript:alert(1)",
+            "vbscript:msgbox(1)",
+            "file:///etc/passwd",
+            "data:text/html;base64,PHNjcmlwdD4=",
+            "feed:javascript:alert(1)",
+            "livescript:alert(1)",
+            "mocha:alert(1)",
+            "view-source:javascript:alert(1)",
+            "jar:javascript:alert(1)",
+            "ms-its:javascript:alert(1)",
+            "mk:@MSITStore:javascript:alert(1)",
+            "res:javascript:",
+        ]:
+            rendered = html(f"[h]({url})")
+            self.assertIn('href="#harmful-link"', rendered, url)
+
+    def test_unknown_url_scheme_is_blocked(self):
+        rendered = html("[h](x-javascript:alert(1))")
+
+        self.assertIn('href="#harmful-link"', rendered)
+
+    def test_reference_and_autolink_harmful_url_schemes_are_blocked(self):
+        cases = [
+            "[h][r]\n\n[r]: feed:javascript:alert(1)",
+            "<feed:javascript:alert(1)>",
+            "[h][r]\n\n[r]: x-javascript:alert(1)",
+            "<x-javascript:alert(1)>",
+        ]
+        for text in cases:
+            rendered = create_markdown()(text)
+            self.assertIn('href="#harmful-link"', rendered, text)
+
     def test_percent_encoded_harmful_url_scheme_is_blocked(self):
         for text in [
             "[h](javascript%3Aalert(1))",
             "[h](javascript%253Aalert(1))",
             "[h][r]\n\n[r]: javascript%3Aalert(1)",
             "![h](data%3Atext/html;base64,PHNjcmlwdD4=)",
             "[h](view-source:javascript:alert(1))",
+            "[h](%20javascript%3Aalert(1))",
         ]:
-            html = create_markdown()(text)
-            self.assertIn("#harmful-link", html, text)
+            rendered = create_markdown()(text)
+            self.assertIn("#harmful-link", rendered, text)
 
     def test_safe_percent_encoded_data_image_is_allowed(self):
-        html = create_markdown()("![h](data%3Aimage/png;base64,AAAA)")
-        self.assertIn('src="data%3Aimage/png;base64,AAAA"', html)
+        rendered = create_markdown()("![h](data%3Aimage/png;base64,AAAA)")
+        self.assertIn('src="data%3Aimage/png;base64,AAAA"', rendered)
+
+    def test_safe_url_schemes_are_allowed(self):
+        for url in [
+            "http://example.com",
+            "https://example.com",
+            "mailto:user@example.com",
+            "tel:+123456789",
+            "ftp://example.com/file",
+            "ftps://example.com/file",
+            "irc://example.com/channel",
+            "ircs://example.com/channel",
+            "//example.com/path",
+            "/path",
+            "./path:with-colon",
+            "../path:with-colon",
+            "path/with:colon",
+            "#fragment",
+            "?query",
+        ]:
+            rendered = html(f"[h]({url})")
+            self.assertIn(f'href="{url}"', rendered, url)