Add new errors argument to decode_content()

Author: User

tests/test_decode_html.py

@@ -1,59 +1,74 @@
 # coding: utf-8
+import pytest
+
 from unicodec import decode_content
 
 
-def test_basic_usage():
-    # type: () -> None
+def test_basic_usage():  # type: () -> None
     assert decode_content(b"asdf") == "asdf"
 
 
-def test_input_bytess():
-    # type: () -> None
+def test_input_bytess():  # type: () -> None
     # fmt: off
     assert decode_content(u"крокодил".encode("utf-8")) == u"крокодил"
     # fmt: on
 
 
-def test_input_str():
-    # type: () -> None
+def test_input_str():  # type: () -> None
     # fmt: off
     assert decode_content(u"крокодил") == u"крокодил"
     # fmt: on
 
 
-def test_arg_decode_entities_default():
-    # type: () -> None
+def test_arg_decode_entities_default():  # type: () -> None
     # fmt: off
     assert decode_content("©") == u"©"
     # fmt: on
 
 
-def test_arg_decode_entities_false():
-    # type: () -> None
+def test_arg_decode_entities_false():  # type: () -> None
     assert decode_content("©", decode_entities=False) == "©"
 
 
-def test_arg_remove_null_bytes_default():
-    # type: () -> None
+def test_arg_remove_null_bytes_default():  # type: () -> None
     assert decode_content("as\x00df") == "asdf"
 
 
-def test_arg_remove_null_bytes_false():
-    # type: () -> None
+def test_arg_remove_null_bytes_false():  # type: () -> None
     assert decode_content("as\x00df", remove_null_bytes=False) == "as\x00df"
 
 
-def test_arg_encoding_default():
-    # type: () -> None
+def test_arg_encoding_default():  # type: () -> None
     # fmt: off
     assert decode_content(u"крокодил".encode("utf-8")) == u"крокодил"
     # fmt: on
 
 
-def test_arg_encoding_explicit():
-    # type: () -> None
+def test_arg_encoding_explicit():  # type: () -> None
     # fmt: off
     assert decode_content(u"крокодил".encode("cp1251"), encoding="cp1251") == (
         u"крокодил"
     )
     # fmt: on
+
+
+def test_errors_strict_default():  # type: () -> None
+    with pytest.raises(UnicodeDecodeError):
+        decode_content(b"\x80", encoding="utf-8")
+
+
+def test_errors_strict():  # type: () -> None
+    with pytest.raises(UnicodeDecodeError):
+        decode_content(b"\x80", encoding="utf-8", errors="strict")
+
+
+def test_errors_replace():  # type: () -> None
+    # fmt: off
+    assert decode_content(b"\x80", encoding="utf-8", errors="replace") == u"�"
+    # fmt: on
+
+
+def test_errors_ignore():  # type: () -> None
+    # fmt: off
+    assert decode_content(b"\x80", encoding="utf-8", errors="ignore") == u""
+    # fmt: on

unicodec/main.py

@@ -49,14 +49,15 @@ def decode_content(  # pylint: disable=R0917
     encoding=None,  # type: None | str
     content_type_header=None,  # type: None | str
     markup="html",  # type: Literal["html", "xml"]
+    errors="strict",  # type: Literal["strict", "ignore", "replace"]
 ):
     # type: (...) -> str
     if isinstance(data, bytes):
         if encoding is None:
             encoding = detect_content_encoding(
                 data, content_type_header=content_type_header, markup=markup
             )
-        data = data.decode(encoding)
+        data = data.decode(encoding, errors=errors)
     # Remove BOM, it might be at the start of decoded unicode text
     if data.startswith(BOM_UNICODE):
         data = data[len(BOM_UNICODE) :]