feat: add lexical search flag for Weaviate

Add enable_lexical_search flag to WeaviateUploadStagerConfig to indicate
that the collection is configured for BM25 keyword search.

Users must manually configure their Weaviate collection schema with
invertedIndexConfig.bm25 settings. See inline documentation for schema
example.

This flag is declarative - it documents user intent rather than
controlling automatic schema creation.

Author: micmarty-deepsense

REPO_UPDATES.md

@@ -0,0 +1,14 @@
+# Repository Updates
+
+## 2026-01-30 09:15:00
+
+- **fix**: Remove SFTP indexer run override (commit: 060bd52)
+- **fix**: SFTP protocol needs default (commit: ba493ee)
+
+## 2026-01-28 10:00:01
+
+- **feat**: Teradata Source and Destination Connector (commit: 577fd6b)
+- **feat**: SFTP connector improvements with better error handling (commit: 1fe1ecd, d3e8927)
+- **fix**: Fix AstraDB uploader timeouts for large files (commit: e6de207)
+- **fix**: Update Vectara token endpoint (commit: 64938d9)
+- **chore**: Remove Teradata charset setting (commit: 4fd9f71)

pyproject.toml

@@ -157,6 +157,9 @@ test = [
     "htmlBuilder",
     "office365-rest-python-client",
 ]
+dev = [
+    "pytest>=9.0.2",
+]
 
 [project.scripts]
 unstructured-ingest = "unstructured_ingest.main:main"

unstructured_ingest/processes/connectors/assets/weaviate_collection_config.json

@@ -19,14 +19,5 @@
             "tokenization": "word"
         }
     ],
-    "vectorizer": "none",
-    "invertedIndexConfig": {
-        "bm25": {
-            "b": 0.75,
-            "k1": 1.2
-        },
-        "stopwords": {
-            "preset": "en"
-        }
-    }
+    "vectorizer": "none"
 }

unstructured_ingest/processes/connectors/weaviate/weaviate.py

@@ -31,6 +31,27 @@
 
 CONNECTOR_TYPE = "weaviate"
 
+# Schema Example for BM25 Lexical Search:
+# To enable BM25 keyword search, configure your Weaviate collection with invertedIndexConfig:
+#
+# {
+#   "class": "YourCollection",
+#   "properties": [...],
+#   "vectorizer": "none",
+#   "invertedIndexConfig": {
+#     "bm25": {
+#       "b": 0.75,
+#       "k1": 1.2
+#     },
+#     "stopwords": {
+#       "preset": "en"
+#     }
+#   }
+# }
+#
+# Then set enable_lexical_search=True in WeaviateUploadStagerConfig to indicate the
+# collection is configured for hybrid search (vector + keyword/BM25).
+
 
 class WeaviateAccessConfig(AccessConfig, ABC):
     pass
@@ -59,8 +80,12 @@ def get_client(self) -> Generator["WeaviateClient", None, None]:
 class WeaviateUploadStagerConfig(UploadStagerConfig):
     enable_lexical_search: bool = Field(
         default=False,
-        description="Enable BM25 keyword search. Text fields are indexed "
-        "for lexical search via Weaviate's inverted index with BM25 scoring.",
+        description=(
+            "Indicates that the Weaviate collection is configured for BM25 keyword search. "
+            "Users must manually configure their collection schema with invertedIndexConfig.bm25 "
+            "settings. See Weaviate documentation for BM25 configuration: "
+            "https://weaviate.io/developers/weaviate/config-refs/schema#invertedindexconfig--bm25"
+        ),
     )