feat: add display name to connectors (#532)

# Add `display_name` attribute for `FileData` in connectors that miss it
Display name might be helpful when debugging failed processing.
This PR:
- adds this attribute to the connectors that miss it
- updates tests to check for that field
- udpates fixtures/expected processed data files (=fixtures update) -
__MOST OF THE CHANGES IN THIS PR__

Author: pawel-kmiecik

.github/workflows/ingest-test-fixtures-update-pr.yml

@@ -16,10 +16,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: ./.github/actions/base-cache
+      - uses: ./.github/actions/setup-environment
         with:
           python-version: ${{ env.PYTHON_VERSION }}
-          check-only: 'true'
 
   update-fixtures-and-pr:
     runs-on: ubuntu-latest-m
@@ -35,7 +34,7 @@ jobs:
         id: full-python-version
         run: echo version=$(python -c "import sys; print('-'.join(str(v) for v in sys.version_info))") >> $GITHUB_OUTPUT
       - name: Setup virtual environment
-        uses: ./.github/actions/base-cache
+        uses: ./.github/actions/setup-environment
         with:
           python-version: ${{ env.PYTHON_VERSION }}
       - name: Update test fixtures

CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.0.41
+
+* **Add `display_name` to FileData in 14 connectors**
+
 ## 1.0.40
 
 * **Fix extracting embedded files from Confluence pages**

test/integration/connectors/databricks/test_volumes_native.py

@@ -18,6 +18,7 @@
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
+    source_filedata_display_name_set_check,
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
@@ -114,6 +115,8 @@ async def test_volumes_native_source(tmp_path: Path):
             configs=SourceValidationConfigs(
                 test_id="databricks_volumes_native",
                 expected_num_files=1,
+                predownload_file_data_check=source_filedata_display_name_set_check,
+                postdownload_file_data_check=source_filedata_display_name_set_check,
             ),
         )
 
@@ -144,6 +147,8 @@ async def test_volumes_native_source_pat(tmp_path: Path):
             configs=SourceValidationConfigs(
                 test_id="databricks_volumes_native_pat",
                 expected_num_files=1,
+                predownload_file_data_check=source_filedata_display_name_set_check,
+                postdownload_file_data_check=source_filedata_display_name_set_check,
             ),
         )
 

test/integration/connectors/discord/test_discord.py

@@ -10,6 +10,7 @@
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
+    source_filedata_display_name_set_check,
 )
 from test.integration.utils import requires_env
 from unstructured_ingest.error import SourceConnectionError
@@ -62,6 +63,8 @@ async def test_discord_source():
                 expected_num_files=expected_num_files,
                 expected_number_indexed_file_data=expected_num_files,
                 validate_downloaded_files=True,
+                predownload_file_data_check=source_filedata_display_name_set_check,
+                postdownload_file_data_check=source_filedata_display_name_set_check,
             ),
         )
 

test/integration/connectors/elasticsearch/test_elasticsearch.py

@@ -20,6 +20,7 @@
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
+    source_filedata_display_name_set_check,
 )
 from unstructured_ingest.error import DestinationConnectionError, SourceConnectionError
 from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
@@ -203,6 +204,9 @@ async def test_elasticsearch_source(source_index: str, movies_dataframe: pd.Data
                 expected_num_files=expected_num_files,
                 expected_number_indexed_file_data=1,
                 validate_downloaded_files=True,
+                predownload_file_data_check=source_filedata_display_name_set_check,
+                postdownload_file_data_check=source_filedata_display_name_set_check,
+                exclude_fields_extend=["display_name"] # includes dynamic ids, might change
             ),
         )
 

test/integration/connectors/elasticsearch/test_opensearch.py

@@ -19,6 +19,7 @@
 from test.integration.connectors.utils.validation.source import (
     SourceValidationConfigs,
     source_connector_validation,
+    source_filedata_display_name_set_check,
 )
 from unstructured_ingest.data_types.file_data import FileData, SourceIdentifiers
 from unstructured_ingest.error import (
@@ -193,6 +194,9 @@ async def test_opensearch_source(source_index: str, movies_dataframe: pd.DataFra
                 expected_num_files=expected_num_files,
                 expected_number_indexed_file_data=1,
                 validate_downloaded_files=True,
+                predownload_file_data_check=source_filedata_display_name_set_check,
+                postdownload_file_data_check=source_filedata_display_name_set_check,
+                exclude_fields_extend=["display_name"]  # includes dynamic ids, might change
             ),
         )
 

test/integration/connectors/expected_results/astradb/file_data/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv.json

@@ -14,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1742508009.2832398",
+    "date_processed": "1749474599.0632136",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -23,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/7s/nsbls8k12qngtdvkhhkpf1n00000gn/T/pytest-of-nathan/pytest-325/test_astra_search_source0/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
-  "display_name": null
+  "local_download_path": "/tmp/pytest-of-runner/pytest-0/test_astra_search_source0/25b75f1d-a2ea-4c97-b75f-1da2eadc97f7.csv",
+  "display_name": "ingest_test_src-None-[762c0093-2277-4f3e-ac00-932277af3e0e..25b75f1d-a2ea-4c97-b75f-1da2eadc97f7]"
 }

test/integration/connectors/expected_results/astradb/file_data/43d02113-723f-5ec1-acbb-c8da8d3650dc.json

@@ -8,7 +8,7 @@
     "record_locator": null,
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1742508009.054028",
+    "date_processed": "1749474598.4987292",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -18,7 +18,7 @@
   },
   "reprocess": false,
   "local_download_path": null,
-  "display_name": null,
+  "display_name": "ingest_test_src-None-[762c0093-2277-4f3e-ac00-932277af3e0e..25b75f1d-a2ea-4c97-b75f-1da2eadc97f7]",
   "batch_items": [
     {
       "identifier": "25b75f1d-a2ea-4c97-b75f-1da2eadc97f7",

test/integration/connectors/expected_results/astradb/file_data/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv.json

@@ -14,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1742508009.284526",
+    "date_processed": "1749474599.0628886",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -23,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/7s/nsbls8k12qngtdvkhhkpf1n00000gn/T/pytest-of-nathan/pytest-325/test_astra_search_source0/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
-  "display_name": null
+  "local_download_path": "/tmp/pytest-of-runner/pytest-0/test_astra_search_source0/60297eea-73d7-4fca-a97e-ea73d7cfca62.csv",
+  "display_name": "ingest_test_src-None-[762c0093-2277-4f3e-ac00-932277af3e0e..25b75f1d-a2ea-4c97-b75f-1da2eadc97f7]"
 }

test/integration/connectors/expected_results/astradb/file_data/641d99e3-9941-4c18-9d99-e399414c183d.csv.json

@@ -14,7 +14,7 @@
     },
     "date_created": null,
     "date_modified": null,
-    "date_processed": "1742508009.284161",
+    "date_processed": "1749474599.0630517",
     "permissions_data": null,
     "filesize_bytes": null
   },
@@ -23,6 +23,6 @@
     "keyspace": null
   },
   "reprocess": false,
-  "local_download_path": "/private/var/folders/7s/nsbls8k12qngtdvkhhkpf1n00000gn/T/pytest-of-nathan/pytest-325/test_astra_search_source0/641d99e3-9941-4c18-9d99-e399414c183d.csv",
-  "display_name": null
+  "local_download_path": "/tmp/pytest-of-runner/pytest-0/test_astra_search_source0/641d99e3-9941-4c18-9d99-e399414c183d.csv",
+  "display_name": "ingest_test_src-None-[762c0093-2277-4f3e-ac00-932277af3e0e..25b75f1d-a2ea-4c97-b75f-1da2eadc97f7]"
 }