lint

paultltc · paultltc · commit 9e3ba39d2435 · 2025-06-12T20:20:16.000+02:00
diff --git a/pyproject.toml b/pyproject.toml
@@ -166,3 +166,8 @@ ignore = ["E501"]
 # Ignore `E402` (import violations) in all `__init__.py` files, and in selected subdirectories.
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["E402", "F401"]
+"run_index_api.py" = ["N803", "N806"]
+"run_retriever.py" = ["N815"]
+
+[tool.ruff]
+exclude = ["tests/"]
diff --git a/scripts/data_extractor.py b/scripts/data_extractor.py
@@ -1,16 +1,16 @@
+import os
+import zipfile
+
 import requests
 
 url = "https://corp.digitalcorpora.org/corpora/files/CC-MAIN-2021-31-PDF-UNTRUNCATED/zipfiles/0000-0999/0000.zip"
 response = requests.get(url)
 with open("0000.zip", "wb") as f:
     f.write(response.content)
 # Unzip the file
-import zipfile
 
 with zipfile.ZipFile("0000.zip", "r") as zip_ref:
     zip_ref.extractall("0000")
-# Read the PDF files
-import os
 
 print(os.listdir("0000"))
 
diff --git a/src/mmore/cli.py b/src/mmore/cli.py
@@ -33,7 +33,10 @@ def process(config_file: str):
     help="Path to the config file for post-processing.",
 )
 @click.option(
-    "--input-data", type=str, required=True, help="Path to the input JSONL file of documents."
+    "--input-data",
+    type=str,
+    required=True,
+    help="Path to the input JSONL file of documents.",
 )
 def postprocess(config_file: str, input_data: str):
     """Run the post-processors pipeline.
@@ -59,7 +62,11 @@ def postprocess(config_file: str, input_data: str):
     help="Path to the config file for indexing.",
 )
 @click.option(
-    "--documents-path", "-f", type=str, required=False, help="Path to the JSONL file of the (post)processed documents."
+    "--documents-path",
+    "-f",
+    type=str,
+    required=False,
+    help="Path to the JSONL file of the (post)processed documents.",
 )
 @click.option(
     "--collection-name",
@@ -93,10 +100,18 @@ def index(config_file: str, documents_path: str, collection_name: str):
     help="Dispatcher configuration file path.",
 )
 @click.option(
-    "--input-file", "-f", type=str, required=True, help="Path to the JSONL file of the input queries."
+    "--input-file",
+    "-f",
+    type=str,
+    required=True,
+    help="Path to the JSONL file of the input queries.",
 )
 @click.option(
-    "--output-file", "-o", type=str, required=True, help="Path to which save the results of the retriever as a JSON."
+    "--output-file",
+    "-o",
+    type=str,
+    required=True,
+    help="Path to which save the results of the retriever as a JSON.",
 )
 def retrieve(config_file: str, input_file: str, output_file: str):
     """Retrieve documents for specified queries.
@@ -171,8 +186,8 @@ def dashboard_backend(host, port):
     """Run the dashboard backend.
 
     Args:
-      host: 
-      port: 
+      host:
+      port:
 
     Returns:
 
diff --git a/src/mmore/process/processors/pdf_processor.py b/src/mmore/process/processors/pdf_processor.py
@@ -158,8 +158,8 @@ def _extract_images(pdf_doc, xref) -> Optional[Image.Image]:
             if self.config.custom_config.get("extract_images", True):
                 for img_info in page.get_images(full=False):
                     image = _extract_images(pdf_doc, img_info[0])
-                    if image and clean_image(
-                        image
+                    if (
+                        image and clean_image(image)
                     ):  # clean image filters images below size 512x512 and variance below 100, these are defaults and can be changed
                         embedded_images.append(image)
                         all_text.append(self.config.attachment_tag)
diff --git a/src/mmore/rag/llm.py b/src/mmore/rag/llm.py
@@ -74,7 +74,9 @@ def __post_init__(self):
                     else (
                         "COHERE"
                         if self.llm_name in _COHERE_MODELS
-                        else "HF" if self.base_url is None else None
+                        else "HF"
+                        if self.base_url is None
+                        else None
                     )
                 )
             )
diff --git a/src/mmore/run_index.py b/src/mmore/run_index.py
@@ -6,6 +6,10 @@
 
 from dotenv import load_dotenv
 
+from .index.indexer import Indexer, IndexerConfig
+from .type import MultimodalSample
+from .utils import load_config
+
 logger = logging.getLogger(__name__)
 INDEX_EMOJI = "🗂️"
 logging.basicConfig(
@@ -14,10 +18,6 @@
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
-from .index.indexer import Indexer, IndexerConfig
-from .type import MultimodalSample
-from .utils import load_config
-
 load_dotenv()
 
 
diff --git a/src/mmore/run_postprocess.py b/src/mmore/run_postprocess.py
@@ -2,6 +2,10 @@
 import logging
 from typing import List
 
+from .process.post_processor.pipeline import PPPipeline, PPPipelineConfig
+from .type import MultimodalSample
+from .utils import load_config
+
 PP_EMOJI = "🧹"
 logger = logging.getLogger(__name__)
 logging.basicConfig(
@@ -10,10 +14,6 @@
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
-from .process.post_processor.pipeline import PPPipeline, PPPipelineConfig
-from .type import MultimodalSample
-from .utils import load_config
-
 
 def _load_dataset(data_path: List[str]) -> List[MultimodalSample]:
     return [s for path in data_path for s in MultimodalSample.from_jsonl(path)]
diff --git a/src/mmore/run_process.py b/src/mmore/run_process.py
@@ -7,6 +7,12 @@
 import click
 import torch
 
+from .dashboard.backend.client import DashboardClient
+from .process.crawler import Crawler, CrawlerConfig
+from .process.dispatcher import Dispatcher, DispatcherConfig
+from .type import MultimodalSample
+from .utils import load_config
+
 PROCESS_EMOJI = "🚀"
 logger = logging.getLogger(__name__)
 logging.basicConfig(
@@ -15,12 +21,6 @@
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
-from .dashboard.backend.client import DashboardClient
-from .process.crawler import Crawler, CrawlerConfig
-from .process.dispatcher import Dispatcher, DispatcherConfig
-from .type import MultimodalSample
-from .utils import load_config
-
 overall_start_time = time.time()
 
 torch.backends.cuda.enable_mem_efficient_sdp(False)
diff --git a/src/mmore/run_rag.py b/src/mmore/run_rag.py
@@ -10,6 +10,9 @@
 from fastapi import FastAPI
 from langserve import add_routes
 
+from .rag.pipeline import RAGConfig, RAGPipeline
+from .utils import load_config
+
 RAG_EMOJI = "🧠"
 logger = logging.getLogger(__name__)
 logging.basicConfig(
@@ -18,9 +21,6 @@
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
-from .rag.pipeline import RAGConfig, RAGPipeline
-from .utils import load_config
-
 load_dotenv()
 
 
diff --git a/src/mmore/run_retriever.py b/src/mmore/run_retriever.py
@@ -12,6 +12,9 @@
 from pydantic import BaseModel, Field
 from tqdm import tqdm
 
+from .rag.retriever import Retriever, RetrieverConfig
+from .utils import load_config
+
 logger = logging.getLogger(__name__)
 RETRIVER_EMOJI = "🔍"
 logging.basicConfig(
@@ -20,9 +23,6 @@
     datefmt="%Y-%m-%d %H:%M:%S",
 )
 
-from .rag.retriever import Retriever, RetrieverConfig
-from .utils import load_config
-
 load_dotenv()
 
 
diff --git a/tests/test_indexer.py b/tests/test_indexer.py
@@ -22,7 +22,12 @@ def sample_jsonl(tmp_path):
     path = tmp_path / "sample_docs.jsonl"
     sample_data = [
         {"id": "1", "text": "Document text 1", "modalities": [], "metadata": {}},
-        {"id": "2", "text": "Document text 2", "modalities": [], "metadata": {"author": "Alice"}},
+        {
+            "id": "2",
+            "text": "Document text 2",
+            "modalities": [],
+            "metadata": {"author": "Alice"},
+        },
     ]
     with open(path, "w", encoding="utf-8") as f:
         for entry in sample_data:
@@ -36,7 +41,9 @@ def test_load_results(sample_jsonl):
     """
     results = load_results(str(sample_jsonl))
     assert len(results) == 2, "Should load exactly 2 documents"
-    assert isinstance(results[0], MultimodalSample), "Should return MultimodalSample objects"
+    assert isinstance(
+        results[0], MultimodalSample
+    ), "Should return MultimodalSample objects"
     # If your code overrides the .id, don't check for '1':
     assert "Document text 1" in results[0].text
     assert results[1].metadata.get("author") == "Alice"
@@ -65,21 +72,18 @@ def test_index_invocation(mock_from_documents, sample_jsonl):
 @patch("src.mmore.index.indexer.DenseModel.from_config")
 @patch("src.mmore.index.indexer.SparseModel.from_config")
 def test_indexer_integration(
-    mock_sparse_model,
-    mock_dense_model,
-    mock_milvus_client,
-    sample_jsonl
+    mock_sparse_model, mock_dense_model, mock_milvus_client, sample_jsonl
 ):
     """
     Tests the Indexer class with mocked embeddings & Milvus.
     """
     mock_dense_model.return_value.embed_documents.return_value = [
         np.array([0.01, 0.02]),
-        np.array([0.03, 0.04])
+        np.array([0.03, 0.04]),
     ]
     mock_sparse_model.return_value.embed_documents.return_value = [
         np.array([0, 1]),
-        np.array([1, 0])
+        np.array([1, 0]),
     ]
 
     # Mock Milvus
@@ -91,7 +95,9 @@ def test_indexer_integration(
     dense_cfg = MagicMock()
     sparse_cfg = MagicMock()
     db_cfg = MagicMock()
-    test_indexer_config = IndexerConfig(dense_model=dense_cfg, sparse_model=sparse_cfg, db=db_cfg)
+    test_indexer_config = IndexerConfig(
+        dense_model=dense_cfg, sparse_model=sparse_cfg, db=db_cfg
+    )
 
     # Load sample documents
     documents = MultimodalSample.from_jsonl(str(sample_jsonl))
@@ -101,11 +107,13 @@ def test_indexer_integration(
         config=test_indexer_config,
         documents=documents,
         collection_name="test_collection",
-        batch_size=2
+        batch_size=2,
     )
 
     # Verify the client did what we expect
-    assert client_instance.create_collection.called, "Should create collection if it does not exist"
+    assert (
+        client_instance.create_collection.called
+    ), "Should create collection if it does not exist"
     assert client_instance.insert.called, "Should insert documents into Milvus"
 
 
@@ -121,14 +129,17 @@ def test_index_documents_error(mock_milvus_client, sample_jsonl):
 
     # Minimal config
     test_indexer_config = IndexerConfig(
-        dense_model=MagicMock(),
-        sparse_model=MagicMock()
+        dense_model=MagicMock(), sparse_model=MagicMock()
     )
 
     # Patch the embeddings to return arrays
-    with patch("src.mmore.index.indexer.DenseModel.from_config") as mock_dense_model,\
-         patch("src.mmore.index.indexer.SparseModel.from_config") as mock_sparse_model:
-        mock_dense_model.return_value.embed_documents.return_value = [np.array([0.01, 0.02])]
+    with (
+        patch("src.mmore.index.indexer.DenseModel.from_config") as mock_dense_model,
+        patch("src.mmore.index.indexer.SparseModel.from_config") as mock_sparse_model,
+    ):
+        mock_dense_model.return_value.embed_documents.return_value = [
+            np.array([0.01, 0.02])
+        ]
         mock_sparse_model.return_value.embed_documents.return_value = [np.array([0, 1])]
 
         indexer = Indexer(
diff --git a/tests/test_postprocessors.py b/tests/test_postprocessors.py
diff --git a/tests/test_processors_local.py b/tests/test_processors_local.py

Original file line number	Diff line number	Diff line change
`@@ -74,7 +74,9 @@ def __post_init__(self):`
`74`	`74`	`else (`
`75`	`75`	`"COHERE"`
`76`	`76`	`if self.llm_name in _COHERE_MODELS`
`77`		`- else "HF" if self.base_url is None else None`
	`77`	`+ else "HF"`
	`78`	`+ if self.base_url is None`
	`79`	`+ else None`
`78`	`80`	`)`
`79`	`81`	`)`
`80`	`82`	`)`