Add deployment fingerprint logging and enable BEIR for Vidore datasets

- Add embed model fallback detection (dim=1024 warning) to e2e.py and recall.py
- Add Milvus collection vector dimension verification after ingestion
- Enable BEIR metrics by default for all Vidore V3 datasets

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
Signed-off-by: Jacob Ioffe <jioffe@nvidia.com>

Author: jioffe502

tools/harness/src/nv_ingest_harness/cases/e2e.py

@@ -81,6 +81,11 @@ def main(config=None, log_path: str = "test_results") -> int:
 
     model_name, dense_dim = embed_info()
 
+    # Deployment fingerprint - detect silent fallback to wrong model
+    if dense_dim == 1024:
+        print("WARNING: Embedding model returned dim=1024 (nv-embedqa-e5-v5 fallback)")
+        print("WARNING: Expected dim=2048 for multimodal embed. Check embedding NIM status.")
+
     # Log configuration for transparency
     print("=== Test Configuration ===")
     print(f"Dataset: {data_dir}")
@@ -254,6 +259,24 @@ def main(config=None, log_path: str = "test_results") -> int:
     # Optional: log chunk stats and per-type breakdown
     if vdb_backend != "lancedb":
         milvus_chunks(f"http://{hostname}:19530", collection_name)
+        # Verify collection vector dimension matches expected
+        try:
+            from pymilvus import MilvusClient
+
+            mc = MilvusClient(uri=f"http://{hostname}:19530")
+            col_info = mc.describe_collection(collection_name)
+            for field in col_info.get("fields", []):
+                params = field.get("params", {})
+                if "dim" in params:
+                    actual_dim = int(params["dim"])
+                    if actual_dim != dense_dim:
+                        print(f"WARNING: Collection vector dim={actual_dim} != expected dim={dense_dim}")
+                        print("WARNING: Collection may have been created with a different embedding model")
+                    else:
+                        print(f"Collection vector dim={actual_dim} matches expected dim={dense_dim}")
+            mc.close()
+        except Exception as e:
+            print(f"Could not verify collection schema: {e}")
     text_results, table_results, chart_results = segment_results(results)
     kv_event_log("text_chunks", sum(len(x) for x in text_results), log_path)
     kv_event_log("table_chunks", sum(len(x) for x in table_results), log_path)

tools/harness/src/nv_ingest_harness/cases/recall.py

@@ -88,6 +88,11 @@ def main(config=None, log_path: str = "test_results") -> int:
     gpu_search = config.gpu_search
     model_name, dense_dim = embed_info()
 
+    # Deployment fingerprint - detect silent fallback to wrong model
+    if dense_dim == 1024:
+        print("WARNING: Embedding model returned dim=1024 (nv-embedqa-e5-v5 fallback)")
+        print("WARNING: Expected dim=2048 for multimodal embed. Check embedding NIM status.")
+
     # Recall-specific configuration with defaults
     reranker_mode = getattr(config, "reranker_mode", "none")
     recall_top_k = getattr(config, "recall_top_k", 10)
@@ -145,6 +150,27 @@ def main(config=None, log_path: str = "test_results") -> int:
     if lancedb_path:
         print(f"Using LanceDB at: {lancedb_path}")
 
+    # Verify collection schema if using Milvus
+    if vdb_backend == "milvus":
+        try:
+            from pymilvus import MilvusClient
+
+            verify_uri = f"http://{hostname}:19530"
+            mc = MilvusClient(uri=verify_uri)
+            col_info = mc.describe_collection(collection_name)
+            for field in col_info.get("fields", []):
+                params = field.get("params", {})
+                if "dim" in params:
+                    actual_dim = int(params["dim"])
+                    if actual_dim != dense_dim:
+                        print(f"WARNING: Collection vector dim={actual_dim} != embed model dim={dense_dim}")
+                        print("WARNING: Collection may have been created with a different embedding model")
+                    else:
+                        print(f"Collection vector dim={actual_dim} matches embed model dim={dense_dim}")
+            mc.close()
+        except Exception as e:
+            print(f"Could not verify collection schema: {e}")
+
     try:
         recall_results = {}
 

tools/harness/test_configs.yaml

@@ -183,6 +183,7 @@ datasets:
     extract_method: ocr
     image_elements_modality: text_image
     recall_dataset: vidore_v3_finance_en
+    enable_beir: true
 
   vidore_v3_industrial:
     path: /datasets/nv-ingest/vidore_v3_corpus/vidore_v3_industrial
@@ -195,6 +196,7 @@ datasets:
     extract_method: ocr
     image_elements_modality: text_image
     recall_dataset: vidore_v3_industrial
+    enable_beir: true
 
   vidore_v3_computer_science:
     path: /datasets/nv-ingest/vidore_v3_corpus/vidore_v3_computer_science
@@ -207,6 +209,7 @@ datasets:
     extract_method: ocr
     image_elements_modality: text_image
     recall_dataset: vidore_v3_computer_science
+    enable_beir: true
 
   vidore_v3_pharmaceuticals:
     path: /datasets/nv-ingest/vidore_v3_corpus/vidore_v3_pharmaceuticals
@@ -219,6 +222,7 @@ datasets:
     extract_method: ocr
     image_elements_modality: text_image
     recall_dataset: vidore_v3_pharmaceuticals
+    enable_beir: true
 
   vidore_v3_hr:
     path: /datasets/nv-ingest/vidore_v3_corpus/vidore_v3_hr
@@ -231,6 +235,7 @@ datasets:
     extract_method: ocr
     image_elements_modality: text_image
     recall_dataset: vidore_v3_hr
+    enable_beir: true
 
   vidore_v3_energy:
     path: /datasets/nv-ingest/vidore_v3_corpus/vidore_v3_energy
@@ -243,6 +248,7 @@ datasets:
     extract_method: ocr
     image_elements_modality: text_image
     recall_dataset: vidore_v3_energy
+    enable_beir: true
 
   vidore_v3_physics:
     path: /datasets/nv-ingest/vidore_v3_corpus/vidore_v3_physics
@@ -255,6 +261,7 @@ datasets:
     extract_method: ocr
     image_elements_modality: text_image
     recall_dataset: vidore_v3_physics
+    enable_beir: true
 
   vidore_v3_finance_fr:
     path: /datasets/nv-ingest/vidore_v3_corpus/vidore_v3_finance_fr
@@ -267,6 +274,7 @@ datasets:
     extract_method: ocr
     image_elements_modality: text_image
     recall_dataset: vidore_v3_finance_fr
+    enable_beir: true
 
 # Dataset groups for running multiple datasets together
 # Use: uv run nv-ingest-harness-run --case=e2e_recall --dataset=vidore