fix: fix html encoding to support foreign characters (#452)

MthwRobinson · web-flow · commit b855fd269fa5 · 2023-04-05T20:18:54.000Z
* fix: fix html encoding to support foreign characters

* version and changelog
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,4 +1,4 @@
-## 0.5.11-dev0
+## 0.5.11
 
 ### Enhancements
 
@@ -7,6 +7,7 @@
 ### Fixes
 
 * Guard against null style attribute in docx document elements
+* Update HTML encoding to better support foreign language characters
 
 ## 0.5.10
 
diff --git a/test_unstructured/partition/test_html_partition.py b/test_unstructured/partition/test_html_partition.py
@@ -149,3 +149,9 @@ def test_user_without_file_write_permission_can_partition_html(tmp_path, monkeyp
     # partition html should still work
     elements = partition_html(filename=read_only_file.resolve())
     assert len(elements) > 0
+
+
+def test_partition_html_processes_chinese_chracters():
+    html_text = "<html><div><p>每日新闻</p></div></html>"
+    elements = partition_html(text=html_text)
+    assert elements[0].text == "每日新闻"
diff --git a/unstructured/__version__.py b/unstructured/__version__.py
@@ -1 +1 @@
-__version__ = "0.5.11-dev0"  # pragma: no cover
+__version__ = "0.5.11"  # pragma: no cover
diff --git a/unstructured/documents/xml.py b/unstructured/documents/xml.py
@@ -55,7 +55,14 @@ def pages(self) -> List[Page]:
     def _read_xml(self, content):
         """Reads in an XML file and converts it to an lxml element tree object."""
         if self.document_tree is None:
-            document_tree = etree.fromstring(content.encode(), self.parser)
+            try:
+                document_tree = etree.fromstring(content, self.parser)
+            # NOTE(robinson) - The following ValueError occurs with unicode strings. In that
+            # case, we call back to encoding the string and passing in bytes.
+            #     ValueError: Unicode strings with encoding declaration are not supported.
+            #     Please use  bytes input or XML fragments without declaration.
+            except ValueError:
+                document_tree = etree.fromstring(content.encode(), self.parser)
 
             if self.stylesheet:
                 if isinstance(self.parser, etree.HTMLParser):

Original file line number	Diff line number	Diff line change
`@@ -1 +1 @@`
`1`		`-__version__ = "0.5.11-dev0" # pragma: no cover`
	`1`	`+__version__ = "0.5.11" # pragma: no cover`