feat: track text source (#4112)

The purpose of this PR is to use the newly created `is_extracted`
parameter in `TextRegion` (and the corresponding vector version
`is_extracted_array` in `TextRegions`), flagging elements that were
extracted directly from PDFs as such.

This also involved:
- New tests
- A version update to bring in the new `unstructured-inference`
- An ingest fixtures update
- An optimization from Codeflash that's not directly related

One important thing to review is that all avenues by which an element is
extracted and ends up in the output of a partition are covered... fast,
hi_res, etc.

---------

Co-authored-by: ryannikolaidis <[email protected]>
Co-authored-by: codeflash-ai[bot] <148906541+codeflash-ai[bot]@users.noreply.github.com>
Co-authored-by: luke-kucing <[email protected]>
Co-authored-by: Claude <[email protected]>
Co-authored-by: qued <[email protected]>

Author: 5 people

CHANGELOG.md

@@ -1,7 +1,17 @@
+## 0.18.19-dev0
+
+### Enhancement
+- Flag extracted elements as such in the metadata for downstream use
+
+### Features
+
+### Fixes
+
 ## 0.18.18
 
 ### Fixes
 - **Prevent path traversal in email MSG attachment filenames** Fixed a security vulnerability (GHSA-gm8q-m8mv-jj5m) where malicious attachment filenames containing path traversal sequences could write files outside the intended directory. The fix normalizes both Unix and Windows path separators before sanitizing filenames, preventing cross-platform path traversal attacks in `partition_msg` functions
+
 ## 0.18.17
 
 ### Enhancement

requirements/extra-pdf-image.in

@@ -12,5 +12,5 @@ google-cloud-vision
 effdet
 # Do not move to constraints.in, otherwise unstructured-inference will not be upgraded
 # when unstructured library is.
-unstructured-inference>=1.0.5
+unstructured-inference>=1.1.1
 unstructured.pytesseract>=0.3.12

requirements/extra-pdf-image.txt

@@ -283,7 +283,7 @@ typing-extensions==4.15.0
     #   torch
 tzdata==2025.2
     # via pandas
-unstructured-inference==1.0.5
+unstructured-inference==1.1.1
     # via -r ./extra-pdf-image.in
 unstructured-pytesseract==0.3.15
     # via -r ./extra-pdf-image.in

requirements/extra-xlsx.txt

@@ -15,9 +15,9 @@ cryptography==46.0.3
 et-xmlfile==2.0.0
     # via openpyxl
 msoffcrypto-tool==5.4.2
-    # via -r ./extra-xlsx.in
+    # via -r extra-xlsx.in
 networkx==3.4.2
-    # via -r ./extra-xlsx.in
+    # via -r extra-xlsx.in
 numpy==2.2.6
     # via
     #   -c /Users/luke/git/unstructured/requirements/base.txt
@@ -27,9 +27,9 @@ olefile==0.47
     #   -c /Users/luke/git/unstructured/requirements/base.txt
     #   msoffcrypto-tool
 openpyxl==3.1.5
-    # via -r ./extra-xlsx.in
+    # via -r extra-xlsx.in
 pandas==2.3.3
-    # via -r ./extra-xlsx.in
+    # via -r extra-xlsx.in
 pycparser==2.23
     # via
     #   -c /Users/luke/git/unstructured/requirements/base.txt
@@ -51,4 +51,4 @@ typing-extensions==4.15.0
 tzdata==2025.2
     # via pandas
 xlrd==2.0.2
-    # via -r ./extra-xlsx.in
+    # via -r extra-xlsx.in

test_unstructured/partition/common/test_common.py

@@ -4,6 +4,7 @@
 import numpy as np
 import pytest
 from PIL import Image
+from unstructured_inference.constants import IsExtracted
 from unstructured_inference.inference import layout
 from unstructured_inference.inference.elements import TextRegion
 from unstructured_inference.inference.layoutelement import LayoutElement
@@ -445,3 +446,23 @@ def test_ocr_data_to_elements():
             points=layout_el.bbox.coordinates,
             system=coordinate_system,
         )
+
+
+def test_normalize_layout_element_layout_element_text_source_metadata():
+    layout_element = LayoutElement.from_coords(
+        type="NarrativeText",
+        x1=1,
+        y1=2,
+        x2=3,
+        y2=4,
+        text="Some lovely text",
+        is_extracted=IsExtracted.TRUE,
+    )
+    coordinate_system = PixelSpace(width=10, height=20)
+    element = common.normalize_layout_element(
+        layout_element,
+        coordinate_system=coordinate_system,
+    )
+    assert hasattr(element, "metadata")
+    assert hasattr(element.metadata, "is_extracted")
+    assert element.metadata.is_extracted == "true"

test_unstructured/partition/pdf_image/test_merge_elements.py

@@ -0,0 +1,48 @@
+from PIL import Image
+from unstructured_inference.constants import IsExtracted
+from unstructured_inference.inference.elements import Rectangle
+from unstructured_inference.inference.layout import DocumentLayout, PageLayout
+from unstructured_inference.inference.layoutelement import LayoutElement, LayoutElements
+
+from unstructured.partition.pdf_image.pdfminer_processing import (
+    merge_inferred_with_extracted_layout,
+)
+
+
+def test_text_source_preserved_during_merge():
+    """Test that text_source property is preserved when elements are merged."""
+
+    # Create two simple LayoutElements with different text_source values
+    inferred_element = LayoutElement(
+        bbox=Rectangle(0, 0, 100, 50), text=None, is_extracted=IsExtracted.FALSE
+    )
+
+    extracted_element = LayoutElement(
+        bbox=Rectangle(0, 0, 100, 50), text="Extracted text", is_extracted=IsExtracted.TRUE
+    )
+
+    # Create LayoutElements arrays
+    inferred_layout_elements = LayoutElements.from_list([inferred_element])
+    extracted_layout_elements = LayoutElements.from_list([extracted_element])
+
+    # Create a PageLayout for the inferred layout
+    image = Image.new("RGB", (200, 200))
+    inferred_page = PageLayout(number=1, image=image)
+    inferred_page.elements_array = inferred_layout_elements
+
+    # Create DocumentLayout from the PageLayout
+    inferred_document_layout = DocumentLayout(pages=[inferred_page])
+
+    # Merge them
+    merged_layout = merge_inferred_with_extracted_layout(
+        inferred_document_layout=inferred_document_layout,
+        extracted_layout=[extracted_layout_elements],
+        hi_res_model_name="test_model",
+    )
+
+    # Verify text_source is preserved
+    # Check the merged page's elements_array
+    merged_page = merged_layout.pages[0]
+    assert "Extracted text" in merged_page.elements_array.texts
+    assert hasattr(merged_page.elements_array, "is_extracted_array")
+    assert IsExtracted.TRUE in merged_page.elements_array.is_extracted_array

test_unstructured/partition/pdf_image/test_pdfminer_processing.py

@@ -4,6 +4,7 @@
 import pytest
 from pdfminer.layout import LAParams
 from PIL import Image
+from unstructured_inference.constants import IsExtracted
 from unstructured_inference.constants import Source as InferenceSource
 from unstructured_inference.inference.elements import (
     EmbeddedTextRegion,
@@ -249,6 +250,11 @@ def test_process_file_with_pdfminer():
     assert links[0][0]["url"] == "https://layout-parser.github.io"
 
 
+def test_process_file_with_pdfminer_is_extracted_array():
+    layout, _ = process_file_with_pdfminer(example_doc_path("pdf/layout-parser-paper-fast.pdf"))
+    assert all(is_extracted is IsExtracted.TRUE for is_extracted in layout[0].is_extracted_array)
+
+
 @patch("unstructured.partition.pdf_image.pdfminer_utils.LAParams", return_value=LAParams())
 def test_laprams_are_passed_from_partition_to_pdfminer(pdfminer_mock):
     partition(

test_unstructured_ingest/expected-structured-output/azure/Core-Skills-for-Biomedical-Data-Scientists-2-pages.pdf.json

@@ -4,6 +4,7 @@
     "element_id": "1e41f20785644cdea2f017cfb67bb359",
     "text": "Core Skills for Biomedical Data Scientists",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -26,6 +27,7 @@
     "element_id": "c915a2a57c901810a698491ca2393669",
     "text": "Maryam Zaringhalam, PhD, AAAS Science & Technology Policy Fellow",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -48,6 +50,7 @@
     "element_id": "b24c3f8d268b2f834a00966d8faef975",
     "text": "Lisa Federer, MLIS, Data Science Training Coordinator",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -70,6 +73,7 @@
     "element_id": "fcff333f886b39cee0a7084a9ff9204d",
     "text": "Michael F. Huerta, PhD, Associate Director of NLM for Program Development and NLM Coordinator of Data Science and Open Science Initiatives",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -92,6 +96,7 @@
     "element_id": "1b86fad341db35208d75a543bcf819ae",
     "text": "Executive Summary",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -114,6 +119,7 @@
     "element_id": "fee71d4f7ef7a5f253a44f6df648d12a",
     "text": "This report provides recommendations for a minimal set of core skills for biomedical data scientists based on analysis that draws on opinions of data scientists, curricula for existing biomedical data science programs, and requirements for biomedical data science jobs. Suggested high-level core skills include:",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -136,6 +142,7 @@
     "element_id": "caa3c2eba90fedb7c8923ae8cd8de961",
     "text": "1. General biomedical subject matter knowledge: biomedical data scientists should have a general working knowledge of the principles of biology, bioinformatics, and basic clinical science;",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -158,6 +165,7 @@
     "element_id": "a4622e6575ee04b0c4d74c0c6b3b2452",
     "text": "2. Programming language expertise: biomedical data scientists should be fluent in at least one programming language (typically R and/or Python);",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -180,6 +188,7 @@
     "element_id": "206899164b194bb9c379531b35eae01b",
     "text": "3. Predictive analytics, modeling, and machine learning: while a range of statistical methods may be useful, predictive analytics, modeling, and machine learning emerged as especially important skills in biomedical data science;",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -202,6 +211,7 @@
     "element_id": "36eb8f3c3778fbb71dc056571e71175d",
     "text": "4. Team science and scientific communication: “soft” skills, like the ability to work well on teams and communicate effectively in both verbal and written venues, may be as important as the more technical skills typically associated with data science.",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -224,6 +234,7 @@
     "element_id": "afe37b1ec10a6d08294ff0fb6df79996",
     "text": "5. Responsible data stewardship: a successful data scientist must be able to implement best practices for data management and stewardship, as well as conduct research in an ethical manner that maintains data security and privacy.",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -246,6 +257,7 @@
     "element_id": "b29f66200f2cc9ff2b49f3d07fd8022b",
     "text": "The report further details specific skills and expertise relevant to biomedical data scientists.",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -268,6 +280,7 @@
     "element_id": "bab05a183c34df666bfc920f04d17637",
     "text": "Motivation",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -290,6 +303,7 @@
     "element_id": "f250e86931949c66fe99d742fd9be29c",
     "text": "Training a biomedical data science (BDS) workforce is a central theme in NLM’s Strategic Plan for the coming decade. That commitment is echoed in the NIH-wide Big Data to Knowledge (BD2K) initiative, which invested $61 million between FY2014 and FY2017 in training programs for the development and use of biomedical big data science methods and tools. In line with",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -312,6 +326,7 @@
     "element_id": "9aa82368657b60536f152fd413aec316",
     "text": "Core Skills for Biomedical Data Scientists",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -334,6 +349,7 @@
     "element_id": "4f2dbe3656a9ebc60c7e3426ad3cb3e3",
     "text": "_____________________________________________________________________________________________",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -356,6 +372,7 @@
     "element_id": "cd359ae8c49885ead47318021438eead",
     "text": "this commitment, a recent report to the NLM Director recommended working across NIH to identify and develop core skills required of a biomedical data scientist to consistency across the cohort of NIH-trained data scientists. This report provides a set of recommended core skills based on analysis of current BD2K-funded training programs, biomedical data science job ads, and practicing members of the current data science workforce.",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -378,6 +395,7 @@
     "element_id": "bf8321a34edb7103ec4209f3e4a8a8da",
     "text": "Methodology",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -400,6 +418,7 @@
     "element_id": "1e1d3d1a5c1397fc588393568d829bc8",
     "text": "The Workforce Excellence team took a three-pronged approach to identifying core skills required of a biomedical data scientist (BDS), drawing from:",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -422,6 +441,7 @@
     "element_id": "45d7ff56632d66a2ab2d4dd2716d4d2e",
     "text": "a) Responses to a 2017 Kaggle1 survey2 of over 16,000 self-identified data scientists working across many industries. Analysis of the Kaggle survey responses from the current data science workforce provided insights into the current generation of data scientists, including how they were trained and what programming and analysis skills they use.",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -444,6 +464,7 @@
     "element_id": "bf452aac5123fcedda30dd6ed179f41c",
     "text": "b) Data science skills taught in BD2K-funded training programs. A qualitative content analysis was applied to the descriptions of required courses offered under the 12 BD2K-funded training programs. Each course was coded using qualitative data analysis software, with each skill that was present in the description counted once. The coding schema of data science-related skills was inductively developed and was organized into four major categories: (1) statistics and math skills; (2) computer science; (3) subject knowledge; (4) general skills, like communication and teamwork. The coding schema is detailed in Appendix A.",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -466,6 +487,7 @@
     "element_id": "ca176cbef532792b1f11830ff7520587",
     "text": "c) Desired skills identified from data science-related job ads. 59 job ads from government (8.5%), academia (42.4%), industry (33.9%), and the nonprofit sector (15.3%) were sampled from websites like Glassdoor, Linkedin, and Ziprecruiter. The content analysis methodology and coding schema utilized in analyzing the training programs were applied to the job descriptions. Because many job ads mentioned the same skill more than once, each occurrence of the skill was coded, therefore weighting important skills that were mentioned multiple times in a single ad.",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -488,6 +510,7 @@
     "element_id": "11b170fedd889c3b895bbd28acd811ca",
     "text": "Analysis of the above data provided insights into the current state of biomedical data science training, as well as a view into data science-related skills likely to be needed to prepare the BDS workforce to succeed in the future. Together, these analyses informed recommendations for core skills necessary for a competitive biomedical data scientist.",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -510,6 +533,7 @@
     "element_id": "2665aadf75bca259f1f5b4c91a53a301",
     "text": "1 Kaggle is an online community for data scientists, serving as a platform for collaboration, competition, and learning: http://kaggle.com",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -532,6 +556,7 @@
     "element_id": "8bbfe1c3e6bca9a33226d20d69b2297a",
     "text": "2 In August 2017, Kaggle conducted an industry-wide survey to gain a clearer picture of the state of data science and machine learning. A standard set of questions were asked of all respondents, with more specific questions related to work for employed data scientists and questions related to learning for data scientists in training. Methodology and results: https://www.kaggle.com/kaggle/kaggle-survey-2017",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"
@@ -554,6 +579,7 @@
     "element_id": "dd4a661e1a3c898a5cf6328ba56b924d",
     "text": "2",
     "metadata": {
+      "is_extracted": "true",
       "filetype": "application/pdf",
       "languages": [
         "eng"