fix case where extension is not available

Author: rbiseck3

test_unstructured/file_utils/test_filetype.py

@@ -154,10 +154,10 @@ def test_it_identifies_NDJSON_for_file_like_object_with_no_name_but_NDJSON_conte
     assert detect_filetype(file=file, content_type=FileType.NDJSON.mime_type) == FileType.NDJSON
 
 
-# TODO: ideally this test should pass, currently fails
-# def test_it_identifies_NDJSON_for_file_with_ndjson_extension_but_JSON_content_type():
-#     file_path = example_doc_path("simple.ndjson")
-#     assert detect_filetype(file_path, content_type=FileType.JSON.mime_type) == FileType.NDJSON
+def test_it_identifies_NDJSON_for_file_with_ndjson_extension_but_JSON_content_type():
+    file_path = example_doc_path("simple.ndjson")
+    assert detect_filetype(file_path, content_type=FileType.JSON.mime_type) == FileType.NDJSON
+
 
 # ================================================================================================
 # STRATEGY #3 - GUESS MIME-TYPE WITH LIBMAGIC/FILETYPE LIBRARY
@@ -396,6 +396,27 @@ def test_it_detects_HTML_from_guessed_mime_type_ending_with_xml_and_html_extensi
     assert file_type is FileType.HTML
 
 
+@pytest.mark.parametrize(
+    ("expected_value", "file_name"),
+    [(FileType.NDJSON, "simple.ndjson"), (FileType.JSON, "spring-weather.html.json")],
+)
+def test_it_detects_correct_json_type_without_extension(expected_value: FileType, file_name: str):
+    with open(example_doc_path(file_name), "rb") as f:
+        file = io.BytesIO(f.read())
+
+    filetype = detect_filetype(file=file)
+    assert filetype == expected_value
+
+
+@pytest.mark.parametrize(
+    ("expected_value", "file_name"),
+    [(FileType.NDJSON, "simple.ndjson"), (FileType.JSON, "spring-weather.html.json")],
+)
+def test_it_detects_correct_json_type_with_extension(expected_value: FileType, file_name: str):
+    filetype = detect_filetype(file_path=example_doc_path(file_name))
+    assert filetype == expected_value
+
+
 @pytest.mark.parametrize(
     ("mime_type", "file_name"),
     [

unstructured/file_utils/filetype.py

@@ -179,7 +179,11 @@ def _file_type(self) -> FileType:
         if file_type := self._file_type_from_file_extension:
             return file_type
 
-        # -- strategy 5: give up and report FileType.UNK --
+        # -- strategy 5: edge case where JSON/NDJSON content without file extension --
+        if file_type := self._disambiguate_json_file_type:
+            return file_type
+
+        # -- strategy 6: give up and report FileType.UNK --
         return FileType.UNK
 
     # == STRATEGIES ============================================================
@@ -210,6 +214,20 @@ def _file_type_from_content_type(self) -> FileType | None:
         # -- otherwise we trust the passed `content_type` as long as `FileType` recognizes it --
         return FileType.from_mime_type(self._ctx.content_type)
 
+    @property
+    def _disambiguate_json_file_type(self) -> FileType | None:
+        """Disambiguate JSON/NDJSON file-type based on file contents.
+
+        This method is used when the content-type is `application/json` and the file is not empty.
+        """
+        if self._ctx.content_type != "application/json":
+            return None
+        if is_json_processable(file_text=self._ctx.text_head):
+            return FileType.JSON
+        if is_ndjson_processable(file_text=self._ctx.text_head):
+            return FileType.NDJSON
+        return None
+
     @property
     def _file_type_from_guessed_mime_type(self) -> FileType | None:
         """FileType based on auto-detection of MIME-type by libmagic.

unstructured/file_utils/model.py

@@ -82,7 +82,8 @@ def from_mime_type(cls, mime_type: str | None) -> FileType | None:
         Returns `None` when `mime_type` is `None` or does not map to the canonical MIME-type of a
         `FileType` member or one of its alias MIME-types.
         """
-        if mime_type is None:
+        if mime_type is None or mime_type == "application/json":
+            # application/json is ambiguous as it may point ot JSON and NDJSON file types
             return None
         # -- not super efficient but plenty fast enough for once-or-twice-per-file use and avoids
         # -- limitations on defining a class variable on an Enum.