fix: update detect_filetype for JSONs with text/plain MIME type (#520)

* check to see if text file is a json

* add json check into filetype detection

* added test for updated file detection logic

* bytes/strings handling

* changlog and version bump

Author: MthwRobinson

Diff for: CHANGELOG.md

@@ -1,4 +1,4 @@
-## 0.6.2-dev2
+## 0.6.2-dev3
 
 ### Enhancements
 
@@ -12,6 +12,7 @@
 ### Fixes
 
 * Fix how `exceeds_cap_ratio` handles empty (returns `True` instead of `False`)
+* Updates `detect_filetype` to properly detect JSONs when the MIME type is `text/plain`.
 
 ## 0.6.1
 

Diff for: example-docs/spring-weather.html.json

@@ -0,0 +1,226 @@
+[
+  {
+    "element_id": "41f6e17bf5e9a407fcca74e902f802a0",
+    "text": "News Around NOAA",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "aa589c25dc22dcc8a75baba1244e6c8f",
+    "text": "National Program",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "62c26d2e16774d2334bd804c7bb6a711",
+    "text": "Are You Weather-Ready for the Spring?",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "32709cd3bec72640bbbe32f58e6e23f6",
+    "text": "Weather.gov >",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "2661da76db570876b075083aaeeaee55",
+    "text": "News Around NOAA > Are You Weather-Ready for the Spring?",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "fab6c4df083f0fb6f324fff65b652c86",
+    "text": "Weather Safety                                                                                                        Air Quality                                                                            Beach Hazards                                                                            Cold                                                                            Cold Water                                                                            Drought                                                                            Floods                                                                            Fog                                                                            Heat                                                                             Hurricanes                                                                             Lightning Safety                                                                            Rip Currents                                                                            Safe Boating                                                                            Space Weather                                                                            Sun (Ultraviolet Radiation)                                                                             Thunderstorms & Tornadoes                                                                            Tornado                                                                            Tsunami                                                                            Wildfire                                                                            Wind                                                                            Winter",
+    "type": "ListItem",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "45c26cf3457e6d18985a435e2c0fcc65",
+    "text": "Safety Campaigns                                                                                                        Seasonal Safety Campaigns                                                                            #SafePlaceSelfie                                                                            Deaf & Hard of Hearing                                                                            Intellectual Disabilities                                                                            Spanish-language Content                                                                            The Great Outdoors",
+    "type": "ListItem",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "77f5acc603de9a165ed87a5c3fbaf14a",
+    "text": "Ambassador                                                                                                        About WRN Ambassadors                                                                            Become an Ambassador                                                                            Ambassadors of Excellence                                                                            People of WRN                                                                             FAQS                                                                            Tell Your Success Story                                                                             Success Stories                                                                            Tri-fold                                                                            Aviation                                                                             Current Ambassadors                                                                            Brochure                                                                            En Español",
+    "type": "ListItem",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "8f19bcaabbd1bafa5e9826ac69766c8b",
+    "text": "Education                                                                                                        NWS Education Home                                                                            Be A Force Of Nature                                                                            WRN Kids Flyer                                                                            Wireless Emergency Alerts                                                                            NOAA Weather Radio                                                                            Mobile Weather                                                                            Brochures                                                                            Hourly Weather Forecast                                                                            Citizen Science                                                                            Intellectual Disabilities",
+    "type": "ListItem",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "1245f9cf9e019713391e4ee3bac54a63",
+    "text": "Collaboration                                                                                                        Get Involved                                                                             Social Media                                                                            WRN Ambassadors ​                                                                            Enterprise Resources                                                                            StormReady                                                                            TsunamiReady                                                                            NWSChat (core partners only)                                                                            InteractiveNWS (iNWS) (core partners only)​                                                                            SKYWARN",
+    "type": "ListItem",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "23dfa7f98424dbf86e00b3d500096dfa",
+    "text": "News & Events                                                                                                        Latest News                                                                            Calendar                                                                            Meetings & Workshops                                                                            NWS Aware Newsletter",
+    "type": "ListItem",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "93202df2ec7081b28b47901b5c287a5a",
+    "text": "International",
+    "type": "ListItem",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "e53d6a9c615bdf1a8d7b98a67cade488",
+    "text": "About                                                                                                        Contact Us                                                                             What is WRN?                                                                             WRN FAQ                                                                            WRN Brochure                                                                            Hazard Simplification                                                                            IDSS Brochure                                                                            Roadmap                                                                            Strategic Plan                                                                            WRN International                                                                            Social Science",
+    "type": "ListItem",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "6cbcf8c11f8c0781bd9ecc7f67169ff0",
+    "text": "The spring season is all about change – a rebirth both literally and figuratively. Even though the spring season doesn’t officially (astronomically, that is) begin until March 20 this year, climatologically, it starts March 1.",
+    "type": "NarrativeText",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "7184168da442c6ef28553b274bf2be8f",
+    "text": "As cold winter nights are replaced by the warmth of longer daylight hours, the National Weather Service invites you to do two important things that may save your life or the life of a loved one.",
+    "type": "NarrativeText",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "f3be9748ecd68b20d706548129baa22d",
+    "text": "First, take steps to better prepare for the seasonal hazards weather can throw at you.\nThis could include a spring cleaning of your storm shelter or ensuring your emergency kit is fully stocked. Take a look at our infographics and social media posts to help you become “weather-ready.”",
+    "type": "NarrativeText",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "126c3cd201fb259cfeabc6bffc0b5473",
+    "text": "Second, encourage others to become Weather-Ready as well. Share the message by taking advantage of our vast array of weather safety content – everything posted on our Spring Safety website is freely available, and we encourage sharing on social media networks. Also remember those who are most vulnerable, like an elderly family member or neighbor who might have limited mobility or is isolated. Reach out to those who are at higher risk of being impacted by extreme weather, and help them get prepared. This simple act of caring could become heroic.",
+    "type": "NarrativeText",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "c1944fb037f3e1cb14969bc59a7dd9c2",
+    "text": "This spring, the campaign is focused on heat dangers. Heat illness and death can occur even in spring’s moderately warm weather. The majority of all heat-related deaths occur outside of heat waves and roughly a third of child hot car deaths occur outside of the summer months. Learn more by viewing the infographics that are now available.",
+    "type": "NarrativeText",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "fa1b939ef6159d95260bc095f58ebbc2",
+    "text": "Stay safe this spring, and every season, by being informed, prepared, and Weather-Ready.",
+    "type": "NarrativeText",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "47d5d0d27a35a36d7467dfc8b6e089b3",
+    "text": "US Dept of Commerce\n                        National Oceanic and Atmospheric Administration\n                        National Weather Service\n                        News Around NOAA1325 East West HighwaySilver Spring, MD 20910Comments? Questions? Please Contact Us.",
+    "type": "NarrativeText",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "129c678fce59acee7ac6a6fdb67b6310",
+    "text": "Disclaimer",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "3c96caaebd949e39d25b3ccf4133c5d8",
+    "text": "Information Quality",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "b79cac926e0b2e347e72cc91d5174037",
+    "text": "Help",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "4c4e436f9a453c776dbf011f98d932d6",
+    "text": "Glossary",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "506ff394621596dd88138642eddfc1e4",
+    "text": "Privacy Policy",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "c70ae8c30a61c450d2c5148d1b6a0447",
+    "text": "Freedom of Information Act (FOIA)",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "5d8c71abc527284cd463aa58f3f48098",
+    "text": "About Us",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  },
+  {
+    "element_id": "a8a00c355d2fa1461d532a1088274f32",
+    "text": "Career Opportunities",
+    "type": "Title",
+    "metadata": {
+      "page_number": 1
+    }
+  }
+]

Diff for: test_unstructured/file_utils/test_filetype.py

@@ -31,6 +31,7 @@
         ("unsupported/fake-excel.xlsx", FileType.XLSX),
         ("fake-power-point.pptx", FileType.PPTX),
         ("winter-sports.epub", FileType.EPUB),
+        ("spring-weather.html.json", FileType.JSON),
     ],
 )
 def test_detect_filetype_from_filename(file, expected):
@@ -53,6 +54,7 @@ def test_detect_filetype_from_filename(file, expected):
         ("fake-power-point.pptx", FileType.PPTX),
         ("winter-sports.epub", FileType.EPUB),
         ("fake-doc.rtf", FileType.RTF),
+        ("spring-weather.html.json", FileType.JSON),
     ],
 )
 def test_detect_filetype_from_filename_with_extension(monkeypatch, file, expected):
@@ -257,10 +259,15 @@ def test_detect_filetype_detects_png(monkeypatch):
     assert detect_filetype(filename="made_up.png") == FileType.PNG
 
 
-def test_detect_filetype_detects_unknown_text_types_as_txt(monkeypatch):
+def test_detect_filetype_detects_unknown_text_types_as_txt(monkeypatch, tmpdir):
     monkeypatch.setattr(magic, "from_file", lambda *args, **kwargs: "text/new-type")
     monkeypatch.setattr(os.path, "isfile", lambda *args, **kwargs: True)
-    assert detect_filetype(filename="made_up.png") == FileType.TXT
+
+    filename = os.path.join(tmpdir.dirname, "made_up.png")
+    with open(filename, "w") as f:
+        f.write("here is a fake file!")
+
+    assert detect_filetype(filename=filename) == FileType.TXT
 
 
 def test_detect_filetype_raises_with_both_specified():

Diff for: test_unstructured/partition/test_auto.py

@@ -447,3 +447,17 @@ def test_auto_partition_warns_if_header_set_and_not_url(caplog):
     filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "fake-email.eml")
     partition(filename=filename, headers={"Accept": "application/pdf"})
     assert caplog.records[0].levelname == "WARNING"
+
+
+def test_auto_partition_works_with_unstructured_jsons():
+    filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "spring-weather.html.json")
+    elements = partition(filename=filename)
+    assert elements[0].text == "News Around NOAA"
+
+
+def test_auto_partition_works_with_unstructured_jsons_from_file():
+    filename = os.path.join(EXAMPLE_DOCS_DIRECTORY, "spring-weather.html.json")
+
+    with open(filename, "rb") as f:
+        elements = partition(file=f)
+    assert elements[0].text == "News Around NOAA"

Diff for: unstructured/__version__.py

@@ -1 +1 @@
-__version__ = "0.6.2-dev2"  # pragma: no cover
+__version__ = "0.6.2-dev3"  # pragma: no cover

Diff for: unstructured/file_utils/filetype.py

@@ -1,8 +1,10 @@
 import os
+import re
 import zipfile
 from enum import Enum
 from typing import IO, Optional
 
+from unstructured.nlp.patterns import LIST_OF_DICTS_PATTERN
 from unstructured.partition.common import exactly_one
 
 try:
@@ -169,7 +171,8 @@ def detect_filetype(
     file_filename: Optional[str] = None,
 ) -> Optional[FileType]:
     """Use libmagic to determine a file's type. Helps determine which partition brick
-    to use for a given file. A return value of None indicates a non-supported file type."""
+    to use for a given file. A return value of None indicates a non-supported file type.
+    """
     exactly_one(filename=filename, file=file)
 
     if content_type:
@@ -239,18 +242,6 @@ def detect_filetype(
     elif mime_type.endswith("rtf"):
         return FileType.RTF
 
-    elif mime_type in TXT_MIME_TYPES:
-        if extension and extension == ".eml":
-            return FileType.EML
-        elif extension and extension == ".md":
-            return FileType.MD
-        elif extension and extension == ".rtf":
-            return FileType.RTF
-
-        if file and not extension and _check_eml_from_buffer(file=file) is True:
-            return FileType.EML
-        return FileType.TXT
-
     elif mime_type.endswith("xml"):
         if extension and extension == ".html":
             return FileType.HTML
@@ -260,7 +251,19 @@ def detect_filetype(
     elif mime_type == "text/html":
         return FileType.HTML
 
-    elif mime_type.startswith("text"):
+    elif mime_type in TXT_MIME_TYPES or mime_type.startswith("text"):
+        if extension and extension == ".eml":
+            return FileType.EML
+        elif extension and extension == ".md":
+            return FileType.MD
+        elif extension and extension == ".rtf":
+            return FileType.RTF
+
+        if _is_text_file_a_json(file=file, filename=filename):
+            return FileType.JSON
+
+        if file and not extension and _check_eml_from_buffer(file=file) is True:
+            return FileType.EML
         return FileType.TXT
 
     elif mime_type in XLSX_MIME_TYPES:
@@ -317,10 +320,35 @@ def _detect_filetype_from_octet_stream(file: IO) -> FileType:
         elif all(f in archive_filenames for f in EXPECTED_PPTX_FILES):
             return FileType.PPTX
 
-    logger.warning("Could not detect the filetype from application/octet-stream MIME type.")
+    logger.warning(
+        "Could not detect the filetype from application/octet-stream MIME type.",
+    )
     return FileType.UNK
 
 
+def _is_text_file_a_json(
+    filename: Optional[str] = None,
+    content_type: Optional[str] = None,
+    file: Optional[IO] = None,
+):
+    """Detects if a file that has a text/plain MIME type is a JSON file."""
+    exactly_one(filename=filename, file=file)
+
+    if file is not None:
+        file.seek(0)
+        file_content = file.read(4096)
+        if isinstance(file_content, str):
+            file_text = file_content
+        else:
+            file_text = file_content.decode()
+        file.seek(0)
+    elif filename is not None:
+        with open(filename) as f:
+            file_text = f.read()
+
+    return re.match(LIST_OF_DICTS_PATTERN, file_text) is not None
+
+
 def _check_eml_from_buffer(file: IO) -> bool:
     """Checks if a text/plain file is actually a .eml file. Uses a regex pattern to see if the
     start of the file matches the typical pattern for a .eml file."""