small fixes

Author: fabnemEPFL

docs/index_api.md

@@ -173,7 +173,7 @@ Returns the file with binary content.
 - File types supported:
 
     ```
-    .pdf, .docx, .pptx, .md, .txt, .xlsx, .xls, .csv, .mp4, .avi, .mov, .mkv, .mp3, .wav, .aac, .eml, .html
+    .pdf, .docx, .pptx, .md, .txt, .xlsx, .xls, .csv, .mp4, .avi, .mov, .mkv, .mp3, .wav, .aac, .eml, .html, .htm
     ```
     
 

examples/rag/config_api.yaml

@@ -8,7 +8,7 @@ rag:
   # Retriever Config
   retriever:
     db:
-      uri: ./proc_demo3.db
+      uri: ./proc_demo.db
       name: my_db
     hybrid_search_weight: 0.5
     k: 5

src/mmore/process/dispatcher.py

@@ -85,7 +85,7 @@ def from_dict(config: Dict) -> "DispatcherConfig":
             use_fast_processors=config.get("use_fast_processors", True),
             distributed=config.get("distributed", False),
             scheduler_file=config.get("scheduler_file"),
-            processor_config=config.get("processor"),
+            processor_config=config.get("processor_config"),
             process_batch_sizes=config.get("process_batch_sizes"),
             batch_multiplier=config.get("batch_multiplier", 1),
             extract_images=config.get("extract_images", False),
@@ -111,7 +111,7 @@ def to_dict(self) -> Dict:
             "distributed": self.distributed,
             "scheduler_file": self.scheduler_file,
             "output_path": self.output_path,
-            "processor": self.processor_config,
+            "processor_config": self.processor_config,
             "process_batch_sizes": self.process_batch_sizes,
             "batch_multiplier": self.batch_multiplier,
             "extract_images": self.extract_images,

src/mmore/process/processors/pdf_processor.py

@@ -47,7 +47,7 @@ def load_models(disable_image_extraction: bool = False):
             config=config_parser.generate_config_dict(),
         )
 
-        converter.initialize_processors(converter.default_processors)
+        converter.initialize_processors(list(converter.default_processors))
 
         return converter
 
@@ -131,7 +131,11 @@ def process_batch(
 
     def process(self, file_path: str) -> MultimodalSample:
         if self.converter is None:
-            self.converter = PDFProcessor.load_models()
+            self.converter = PDFProcessor.load_models(
+                disable_image_extraction=not self.config.custom_config.get(
+                    "extract_images", True
+                )
+            )
 
         rendered = self.converter(file_path)
         text, _, images = text_from_rendered(rendered)

src/mmore/rag/retriever.py

@@ -53,7 +53,8 @@ def from_config(cls, config: str | RetrieverConfig):
 
         if not client.has_collection(config.collection_name):
             raise ValueError(
-                "The Milvus database has not been initialized yet. Ensure the path is valid with a database that was already populated with the indexer."
+                f"The Milvus database has not been initialized yet / does not have a collection {config.collection_name}. "
+                "Ensure the path is valid with a database that was already populated with the indexer."
             )
 
         # Init models

src/mmore/run_index_api.py

@@ -99,7 +99,11 @@ async def upload_file(
                     doc.id = doc.id.replace(defDocId, fileId)
 
                 # Get indexer and index the document
-                indexer = get_indexer(COLLECTION_NAME, MILVUS_URI, MILVUS_DB)
+                try:
+                    indexer = get_indexer(COLLECTION_NAME, MILVUS_URI, MILVUS_DB)
+                except Exception as e:
+                    raise HTTPException(status_code=500, detail=str(e))
+
                 indexer.index_documents(
                     documents=documents, collection_name=COLLECTION_NAME
                 )
@@ -180,7 +184,11 @@ async def upload_files(
 
                 logging.info("Indexing the files")
 
-                indexer = get_indexer(COLLECTION_NAME, MILVUS_URI, MILVUS_DB)
+                try:
+                    indexer = get_indexer(COLLECTION_NAME, MILVUS_URI, MILVUS_DB)
+                except Exception as e:
+                    raise HTTPException(status_code=500, detail=str(e))
+
                 indexer.index_documents(
                     documents=modified_documents, collection_name=COLLECTION_NAME
                 )
@@ -240,7 +248,10 @@ async def update_file(
                     doc.id = id
 
                 # Get indexer and reindex the document
-                indexer = get_indexer(COLLECTION_NAME, MILVUS_URI, MILVUS_DB)
+                try:
+                    indexer = get_indexer(COLLECTION_NAME, MILVUS_URI, MILVUS_DB)
+                except Exception as e:
+                    raise HTTPException(status_code=500, detail=str(e))
 
                 # First delete the existing document
                 try:

src/mmore/run_process.py

@@ -65,6 +65,8 @@ def process(config_file: str):
                 ".wav",
                 ".aac",  # Audio files
                 ".eml",  # Emails
+                ".html",
+                ".htm", # HTML pages
             ],
             output_path=config.dispatcher_config.output_path,
         )

src/mmore/utils.py

@@ -3,7 +3,6 @@
 
 import yaml
 from dacite import from_dict
-from fastapi import HTTPException
 from pymilvus import MilvusClient
 
 if TYPE_CHECKING:
@@ -70,9 +69,7 @@ def create_new_indexer(collection_name: str, uri: str, db_name: str) -> "Indexer
         )
         return indexer
     except Exception as e:
-        raise HTTPException(
-            status_code=500, detail=f"Failed to create new indexer: {str(e)}"
-        )
+        raise Exception(f"Unable to create a new indexer: {str(e)}")
 
 
 def get_indexer(collection_name: str, uri: str, db_name: str) -> "Indexer":
@@ -114,9 +111,8 @@ def get_indexer(collection_name: str, uri: str, db_name: str) -> "Indexer":
 
         return indexer
     except Exception as e:
-        raise HTTPException(
-            status_code=404,
-            detail=f"Collection {collection_name} not found or could not be loaded: {str(e)}",
+        raise Exception(
+            f"Collection {collection_name} not found or could not be loaded: {str(e)}"
         )