feat: detect language for PDFs (#4051)

The `@apply_metadata` decorator already contains logic to detect the
language of the element text (on either a document or element level).
Update pdfs, and later images, to use this decorator to get accurate
element language results outputted.

Test
```
from unstructured.partition.auto import partition

def test_partition_pdf():
    pdf_path = "example-docs/language-docs/fr_olap.pdf"
    elements = partition(pdf_path)  # optionally set `detect_language_per_element=True)`
    print(f"Number of elements partitioned: {len(elements)}")

    # Check if elements are returned
    assert len(elements) > 0, "No elements were partitioned from the PDF."

    # check language outputted for each element
    for element in elements:
        print(element)
        print(element.metadata.languages)
        print("-------------------------------")

test_partition_pdf()
```

---------

Co-authored-by: ryannikolaidis <[email protected]>
Co-authored-by: shreyanid <[email protected]>

Author: shreyanid

CHANGELOG.md

@@ -1,3 +1,12 @@
+## 0.18.7
+
+### Enhancements
+
+### Features
+- **Add language detection for PDFs** Add document and element level language detection to PDFs.
+
+### Fixes
+
 ## 0.18.6
 
 ### Enhancements

example-docs/language-docs/fr_olap.pdf

@@ -569,6 +569,49 @@ def test_partition_pdf_strategies_keep_languages_metadata(strategy):
     assert elements[0].metadata.languages == ["kor"]
 
 
+@pytest.mark.parametrize(
+    "strategy",
+    [
+        PartitionStrategy.FAST,
+        PartitionStrategy.HI_RES,
+        PartitionStrategy.OCR_ONLY,
+    ],
+)
+def test_partition_pdf_detects_document_language(strategy):
+    filename = example_doc_path("language-docs/fr_olap.pdf")
+    elements = pdf.partition_pdf(
+        filename=filename,
+        url=None,
+        strategy=strategy,
+    )
+
+    assert len(elements) > 0
+    assert elements[0].metadata.languages == ["fra"]
+    assert elements[-1].metadata.languages == ["fra"]
+
+
+@pytest.mark.parametrize(
+    "strategy",
+    [
+        PartitionStrategy.FAST,
+        PartitionStrategy.HI_RES,
+        PartitionStrategy.OCR_ONLY,
+    ],
+)
+def test_partition_pdf_detects_language_per_element(strategy):
+    filename = example_doc_path("language-docs/fr_olap.pdf")
+    elements = pdf.partition_pdf(
+        filename=filename,
+        url=None,
+        strategy=strategy,
+        detect_language_per_element=True,
+    )
+
+    assert len(elements) > 0
+    assert elements[0].metadata.languages == ["fra"]
+    assert elements[-1].metadata.languages == ["eng"]
+
+
 @pytest.mark.parametrize(
     "ocr_mode",
     [

test_unstructured/partition/pdf_image/test_pdf.py

@@ -561,6 +561,7 @@ def test_auto_partition_pdf_with_fast_strategy(request: FixtureRequest):
         strategy=PartitionStrategy.FAST,
         languages=None,
         metadata_filename=None,
+        detect_language_per_element=False,
         infer_table_structure=False,
         extract_images_in_pdf=False,
         extract_image_block_types=None,
@@ -1301,6 +1302,27 @@ def test_auto_partition_passes_user_provided_languages_arg_to_PDF():
     assert all(e.metadata.languages == ["eng"] for e in elements)
 
 
+@pytest.mark.parametrize(
+    "strategy",
+    [
+        PartitionStrategy.FAST,
+        PartitionStrategy.HI_RES,
+        PartitionStrategy.OCR_ONLY,
+    ],
+)
+def test_auto_partition_detects_pdf_language_per_element(strategy):
+    filename = example_doc_path("language-docs/fr_olap.pdf")
+    elements = partition(
+        filename=filename,
+        strategy=strategy,
+        detect_language_per_element=True,
+    )
+
+    assert len(elements) > 0
+    assert elements[0].metadata.languages == ["fra"]
+    assert elements[-1].metadata.languages == ["eng"]
+
+
 def test_auto_partition_languages_argument_default_to_None_when_omitted():
     elements = partition(example_doc_path("handbook-1p.docx"), detect_language_per_element=True)
     # -- PageBreak and any other element with no text is assigned `None` --
@@ -1309,17 +1331,17 @@ def test_auto_partition_languages_argument_default_to_None_when_omitted():
 
 def test_auto_partition_default_does_not_overwrite_other_defaults():
     """`partition()` ["eng"] default does not overwrite ["auto"] default in other partitioners."""
-    # the default for `languages` is ["auto"] in partiton_text
+    # the default for `languages` is ["auto"] in partition_text
     from unstructured.partition.text import partition_text
 
     # Use a document that is primarily in a language other than English
     file_path = example_doc_path("language-docs/UDHR_first_article_all.txt")
     text_elements = partition_text(file_path)
-    assert text_elements[0].metadata.languages != ["eng"]
+    assert text_elements[13].metadata.languages != ["eng"]
 
     auto_elements = partition(file_path)
-    assert auto_elements[0].metadata.languages != ["eng"]
-    assert auto_elements[0].metadata.languages == text_elements[0].metadata.languages
+    assert auto_elements[13].metadata.languages != ["eng"]
+    assert auto_elements[13].metadata.languages == text_elements[13].metadata.languages
 
 
 # ================================================================================================

test_unstructured/partition/test_auto.py

@@ -5,10 +5,7 @@
     "text": "My First Heading",
     "metadata": {
       "languages": [
-        "por",
-        "cat",
-        "eng",
-        "vie"
+        "eng"
       ],
       "filetype": "text/html",
       "data_source": {
@@ -29,10 +26,7 @@
     "text": "My first paragraph.",
     "metadata": {
       "languages": [
-        "por",
-        "cat",
-        "eng",
-        "vie"
+        "eng"
       ],
       "filetype": "text/html",
       "data_source": {
@@ -53,10 +47,7 @@
     "text": "Some CP1252-specific characters:",
     "metadata": {
       "languages": [
-        "por",
-        "cat",
-        "eng",
-        "vie"
+        "eng"
       ],
       "filetype": "text/html",
       "data_source": {
@@ -77,9 +68,6 @@
     "text": "    ¡\t¢\t£\t¤\t¥\t¦\t§\t¨\t©\tª\t«\t¬\tSHY\t®\t¯\n°\t±\t²\t³\t´\tµ\t¶\t·\t¸\t¹\tº\t»\t¼\t½\t¾\t¿\nÀ\tÁ\tÂ\tÃ\tÄ\tÅ\tÆ\tÇ\tÈ\tÉ\tÊ\tË\tÌ\tÍ\tÎ\tÏ\nÐ\tÑ\tÒ\tÓ\tÔ\tÕ\tÖ\t×\tØ\tÙ\tÚ\tÛ\tÜ\tÝ\tÞ\tß\nà\tá\tâ\tã\tä\tå\tæ\tç\tè\té\tê\të\tì\tí\tî\tï\nð\tñ\tò\tó\tô\tõ\tö\t÷\tø\tù\tú\tû\tü\tý\tþ\tÿ",
     "metadata": {
       "languages": [
-        "por",
-        "cat",
-        "eng",
         "vie"
       ],
       "filetype": "text/html",