No crash if the processor is asked to process an empty folder (#130)

* better default config file for processing

* removed useless run_index.load_results

* stop processing if crawl_result is empty

* removed useless import

Author: fabnemEPFL

examples/process/config.yaml

@@ -1,11 +1,11 @@
-data_path: examples/sample_data/ #put absolute path!
+data_path: examples/sample_data/ #put absolute path or relative to the root of the module
 dispatcher_config:
-  output_path: examples/process/outputs/ #put absolute path!
+  output_path: examples/process/outputs/ #put absolute path or relative to the root of the module
   use_fast_processors: false
   distributed: false
   dashboard_backend_url: null
   extract_images: true
-  scheduler_file: /mmore/scheduler-file.json #put absolute path!
+  scheduler_file: null # for instance /path/to/mmore/scheduler-file.json
   process_batch_sizes:
     - URLProcessor: 40
     - DOCXProcessor: 100

src/mmore/run_index.py

@@ -1,5 +1,4 @@
 import argparse
-import json
 import logging
 from dataclasses import dataclass
 from typing import Optional, Union
@@ -28,17 +27,6 @@ class IndexConfig:
     documents_path: str
 
 
-def load_results(path: str):
-    # Load the results computed and saved by 'run_process.py'
-    results = []
-    logger.info(f"Loading results from {path}")
-    with open(path, "rb") as f:
-        for line in f:
-            results.append(MultimodalSample.from_dict(json.loads(line)))
-    logger.info(f"Loaded {len(results)} results")
-    return results
-
-
 def index(
     config_file: Union[IndexConfig, str],
     documents_path: Optional[str] = None,

src/mmore/run_postprocess.py

@@ -32,6 +32,8 @@ def postprocess(config_file, input_data):
 
     # Load samples
     samples = _load_dataset(input_data)
+    if len(samples) == 0:
+        logger.warning("⚠️ Found no file to postprocess")
 
     # Run pipeline
     samples = pipeline(samples)

src/mmore/run_process.py

@@ -83,6 +83,10 @@ def process(config_file: str):
     crawl_time = crawl_end_time - crawl_start_time
     logger.info(f"Crawling completed in {crawl_time:.2f} seconds")
 
+    if len(crawl_result) == 0:
+        logger.warning("⚠️ Found no file to process")
+        return
+
     dispatcher_config: DispatcherConfig = config.dispatcher_config
 
     url = dispatcher_config.dashboard_backend_url

tests/test_indexer.py

@@ -9,7 +9,7 @@
 from mmore.index.indexer import Indexer, IndexerConfig
 
 # Import run_index from the correct package path:
-from mmore.run_index import index, load_results
+from mmore.run_index import index
 from mmore.type import MultimodalSample
 
 
@@ -32,21 +32,6 @@ def sample_jsonl(tmp_path):
     return path
 
 
-def test_load_results(sample_jsonl):
-    """
-    Tests that load_results() properly reads JSONL files and returns a list of MultimodalSample objects
-    """
-    results = load_results(str(sample_jsonl))
-    assert len(results) == 2, "Should load exactly 2 documents"
-    print(type(results[0]), MultimodalSample)
-    assert isinstance(results[0], MultimodalSample), (
-        "Should return MultimodalSample objects"
-    )
-    # If your code overrides the .id, don't check for '1':
-    assert "Document text 1" in results[0].text
-    assert results[1].metadata.get("author") == "Alice"
-
-
 @patch("mmore.run_index.Indexer.from_documents")
 def test_index_invocation(mock_from_documents, sample_jsonl):
     """