fix: track narrative text and figure captions in HTML documents (#309)

* fix for missing narrative text in partition_html

* fixes so existing tests pass

* tests for figure caption and narrative text

* bump version; changelog

Author: MthwRobinson

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.4.17-dev1
+## 0.5.0
 
 ### Enhancements
 
@@ -12,6 +12,8 @@ instantiating a class or running a function
 ### Fixes
 
 * Fix `process_document` file cleaning on failure
+* Fixes an error introduced in the metadata tracking commit that caused `NarrativeText`
+  and `FigureCaption` elements to be represented as `Text` in HTML documents.
 
 ## 0.4.16
 

test_unstructured/partition/test_common.py

@@ -34,6 +34,19 @@ def test_normalize_layout_element_dict_caption():
     )
 
 
+def test_normalize_layout_element_dict_figure_caption():
+    layout_element = {
+        "type": "FigureCaption",
+        "coordinates": [[1, 2], [3, 4], [5, 6], [7, 8]],
+        "text": "Some lovely text",
+    }
+    element = common.normalize_layout_element(layout_element)
+    assert element == FigureCaption(
+        text="Some lovely text",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+    )
+
+
 def test_normalize_layout_element_dict_misc():
     layout_element = {
         "type": "Misc",
@@ -57,6 +70,19 @@ def test_normalize_layout_element_layout_element():
     )
 
 
+def test_normalize_layout_element_layout_element_narrative_text():
+    layout_element = LayoutElement(
+        type="NarrativeText",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+        text="Some lovely text",
+    )
+    element = common.normalize_layout_element(layout_element)
+    assert element == NarrativeText(
+        text="Some lovely text",
+        coordinates=[[1, 2], [3, 4], [5, 6], [7, 8]],
+    )
+
+
 def test_normalize_layout_element_checked_box():
     layout_element = LayoutElement(
         type="Checked",

unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.4.17-dev1"  # pragma: no cover
+__version__ = "0.5.0"  # pragma: no cover

unstructured/documents/elements.py

@@ -201,6 +201,8 @@ def __init__(
 TYPE_TO_TEXT_ELEMENT_MAP: Dict[str, Any] = {
     "UncategorizedText": Text,
     "FigureCaption": FigureCaption,
+    "Figure": FigureCaption,
+    "Text": NarrativeText,
     "NarrativeText": NarrativeText,
     "ListItem": ListItem,
     "BulletedText": ListItem,

unstructured/partition/common.py

@@ -2,15 +2,13 @@
 from typing import List, Optional, Union
 
 from unstructured.documents.elements import (
+    TYPE_TO_TEXT_ELEMENT_MAP,
     CheckBox,
     Element,
     ElementMetadata,
-    FigureCaption,
     ListItem,
-    NarrativeText,
     PageBreak,
     Text,
-    Title,
 )
 from unstructured.nlp.patterns import ENUMERATED_BULLETS_RE, UNICODE_BULLETS_RE
 
@@ -31,20 +29,15 @@ def normalize_layout_element(layout_element) -> Union[Element, List[Element]]:
     coordinates = layout_dict.get("coordinates")
     element_type = layout_dict.get("type")
 
-    if element_type == "Title":
-        return Title(text=text, coordinates=coordinates)
-    elif element_type == "Text":
-        return NarrativeText(text=text, coordinates=coordinates)
-    elif element_type == "Figure":
-        return FigureCaption(text=text, coordinates=coordinates)
-    elif element_type == "List":
+    if element_type == "List":
         return layout_list_to_list_items(text, coordinates)
+    elif element_type in TYPE_TO_TEXT_ELEMENT_MAP:
+        _element_class = TYPE_TO_TEXT_ELEMENT_MAP[element_type]
+        return _element_class(text=text, coordinates=coordinates)
     elif element_type == "Checked":
         return CheckBox(checked=True, coordinates=coordinates)
     elif element_type == "Unchecked":
         return CheckBox(checked=False, coordinates=coordinates)
-    elif element_type == "PageBreak":
-        return PageBreak()
     else:
         return Text(text=text, coordinates=coordinates)