Merge branch 'master' into cherry2

Author: fabnemEPFL

examples/index/config.yaml

@@ -9,4 +9,4 @@ indexer:
     uri: ./proc_demo.db
     name: my_db
 collection_name: my_docs
-documents_path: 'examples/process/outputs/merged/final_pp.jsonl'
+documents_path: 'examples/postprocessor/outputs/merged/final_pp.jsonl'

examples/postprocessor/config.yaml

@@ -1,7 +1,23 @@
 pp_modules:
-  - type: chunker 
+  - type: file_namer
+  - type: chunker
     args:
       chunking_strategy: sentence
+  - type: translator
+    args:
+      target_language: en
+      attachment_tag: <attachment>
+      confidence_threshold: 0.7
+      constrained_languages: 
+        - fr
+        - en
+  - type: metafuse
+    args:
+      metadata_keys:
+        - file_name
+      content_template: Content from {file_name}
+      position: beginning
+
 output:
-  output_path: examples/process/outputs/merged/
-  save_each_step: True
+  output_path: examples/postprocessor/outputs/merged/
+  save_each_step: True

examples/sample_data/txt/poeme.txt

@@ -0,0 +1,15 @@
+Demain, dès l’aube, à l’heure où blanchit la campagne,
+Je partirai. Vois-tu, je sais que tu m’attends.
+J’irai par la forêt, j’irai par la montagne.
+Je ne puis demeurer loin de toi plus longtemps.
+
+Je marcherai les yeux fixés sur mes pensées,
+Sans rien voir au dehors, sans entendre aucun bruit,
+Seul, inconnu, le dos courbé, les mains croisées,
+Triste, et le jour pour moi sera comme la nuit.
+
+Je ne regarderai ni l’or du soir qui tombe,
+Ni les voiles au loin descendant vers Harfleur,
+Et quand j’arriverai, je mettrai sur ta tombe
+Un bouquet de houx vert et de bruyère en fleur.
+

pyproject.toml

@@ -77,7 +77,7 @@ dependencies = [
     "langchain-anthropic==0.3.4",
     "langchain-aws==0.2.22",
     "langchain-cohere==0.4.2",
-    "langchain-huggingface==0.3.0",
+    "langchain-huggingface==0.1.2",
     "langchain-milvus==0.1.8",
     "langchain-mistralai==0.2.7",
     "langchain-nvidia-ai-endpoints",
@@ -88,7 +88,11 @@ dependencies = [
     "nltk>=3.9",
     "starlette==0.46",
     "typing_extensions==4.12.2",
-    "sympy==1.14.0"
+    "sympy==1.14.0",
+    "langid",
+    "mammoth==1.9.0",
+    "argostranslate",
+    "sentence-transformers",
 ]
 
 [project.optional-dependencies]
@@ -104,7 +108,7 @@ rag = [
   "langchain-anthropic==0.3.4",
   "langchain-aws",
   "langchain-cohere==0.4.2",
-  "langchain-huggingface==0.3.0",
+  "langchain-huggingface==0.1.2",
   "langchain-milvus==0.1.8",
   "langchain-mistralai==0.2.7",
   "langchain-nvidia-ai-endpoints",
@@ -164,4 +168,4 @@ ignore = ["E501", "E402"]   # Avoid enforcing line-length violations (`E501`) an
 
 [tool.ruff.lint.per-file-ignores]
 "__init__.py" = ["F401"]
-"run_index_api.py" = ["N803", "N806"]
+"run_index_api.py" = ["N803", "N806"]

src/mmore/process/post_processor/__init__.py

@@ -2,10 +2,8 @@
 
 from ...utils import load_config
 from .base import BasePostProcessor, BasePostProcessorConfig
-from .chunker import MultimodalChunker, MultimodalChunkerConfig
 from .filter import FILTER_TYPES, load_filter
 from .filter.base import BaseFilterConfig
-from .ner import NERecognizer, NERExtractorConfig
 from .tagger import TAGGER_TYPES, load_tagger
 from .tagger.base import BaseTaggerConfig
 
@@ -15,13 +13,33 @@
 def load_postprocessor(config: BasePostProcessorConfig) -> BasePostProcessor:
     if config.type in FILTER_TYPES:
         return load_filter(cast(BaseFilterConfig, config))
+
     elif config.type in TAGGER_TYPES:
         return load_tagger(cast(BaseTaggerConfig, config))
+
     elif config.type == "chunker":
+        from .chunker import MultimodalChunker, MultimodalChunkerConfig
+
         config_chunk = load_config(config.args, MultimodalChunkerConfig)
         return MultimodalChunker.from_config(config_chunk)
+
     elif config.type == "ner":
+        from .ner import NERecognizer, NERExtractorConfig
+
         config_ner = load_config(config.args, NERExtractorConfig)
         return NERecognizer.from_config(config_ner)
+
+    elif config.type == "translator":
+        from .translator import TranslatorConfig, TranslatorPostProcessor
+
+        config_translator = load_config(config.args, TranslatorConfig)
+        return TranslatorPostProcessor.from_config(config_translator)
+
+    elif config.type == "metafuse":
+        from .metafuse import MetaDataInfusor, MetaDataInfusorConfig
+
+        config_metafuse = load_config(config.args, MetaDataInfusorConfig)
+        return MetaDataInfusor.from_config(config_metafuse)
+
     else:
         raise ValueError(f"Unrecognized postprocessor type: {config.type}")

src/mmore/process/post_processor/metafuse/__init__.py

@@ -0,0 +1,3 @@
+from .base import MetaDataInfusor, MetaDataInfusorConfig, MetaDataPosition
+
+__all__ = ["MetaDataInfusor", "MetaDataInfusorConfig", "MetaDataPosition"]

src/mmore/process/post_processor/metafuse/base.py

@@ -0,0 +1,61 @@
+from collections import defaultdict
+from dataclasses import dataclass
+from enum import Enum
+from typing import List
+
+from mmore.process.post_processor.base import BasePostProcessor
+from mmore.type import MultimodalSample
+
+
+class MetaDataPosition(Enum):
+    BEGINNING = "beginning"
+    END = "end"
+
+
+@dataclass
+class MetaDataInfusorConfig:
+    metadata_keys: List[str]
+    content_template: str
+    position: str
+
+
+class MetaDataInfusor(BasePostProcessor):
+    def __init__(
+        self,
+        metadata_keys: List[str],
+        content_template: str,
+        position: MetaDataPosition,
+    ):
+        super().__init__(name="☕ Metadata Infusor")
+        self.metadata_keys = metadata_keys
+        self.content_template = content_template
+        self.position = position
+
+    @classmethod
+    def from_config(cls, config: MetaDataInfusorConfig):
+        metadata_infusor = MetaDataInfusor(
+            metadata_keys=config.metadata_keys,
+            content_template=config.content_template,
+            position=MetaDataPosition(config.position),
+        )
+        return metadata_infusor
+
+    def process(self, sample: MultimodalSample, **kwargs) -> List[MultimodalSample]:
+        format_mapping = defaultdict()
+        for key in self.metadata_keys:
+            value = sample.metadata.get(key, "")
+            format_mapping[key] = value
+
+        metadata_content = self.content_template.format_map(format_mapping)
+
+        match self.position:
+            case MetaDataPosition.BEGINNING:
+                new_content = metadata_content + "\n" + sample.text
+            case MetaDataPosition.END:
+                new_content = sample.text + "\n" + metadata_content
+            case _:
+                new_content = sample.text
+
+        return [
+            MultimodalSample(new_content, sample.modalities, sample.metadata, sample.id)
+        ]

src/mmore/process/post_processor/tagger/__init__.py

@@ -1,3 +1,5 @@
+from mmore.process.post_processor.tagger.file_namer import FileNamer
+
 from .base import BaseTaggerConfig
 from .lang_detector import LangDetector
 from .modalities import ModalitiesCounter
@@ -9,6 +11,7 @@
     "modalities_counter": ModalitiesCounter,
     "words_counter": WordsCounter,
     "lang_detector": LangDetector,
+    "file_namer": FileNamer,
 }
 TAGGER_TYPES = list(TAGGERS_LOADERS_MAP.keys())
 

src/mmore/process/post_processor/tagger/file_namer.py

@@ -0,0 +1,34 @@
+import os
+
+from .base import BaseTagger, BaseTaggerConfig
+
+
+class FileNamer(BaseTagger):
+    """
+    A tagger that extracts the file name from the sample's metadata.
+    This tagger is useful for identifying the source file of a sample, especially when dealing with multiple files.
+    It retrieves the file name from the `file_path` metadata key and uses it as a tag.
+    Attributes:
+        name (str): The name of the tagger.
+        metadata_key (str): The key in the sample's metadata from which to extract the file name.
+    """
+
+    def __init__(self, name: str = "🔤 File Namer", metadata_key: str = "file_name"):
+        """
+        Initializes the FileNamer tagger.
+        Args:
+            name (str): The name of the tagger.
+            metadata_key (str): The key in the sample's metadata from which to extract the file name.
+        """
+        super().__init__(name, metadata_key)
+
+    def tag(self, sample):
+        if "file_path" not in sample.metadata:
+            return "unknown"
+
+        return os.path.basename(str(sample.metadata["file_path"]))
+
+    @classmethod
+    def from_config(cls, config: BaseTaggerConfig):
+        file_namer = FileNamer()
+        return file_namer

src/mmore/process/post_processor/translator/__init__.py

@@ -0,0 +1,3 @@
+from .base import TranslatorConfig, TranslatorPostProcessor
+
+__all__ = ["TranslatorPostProcessor", "TranslatorConfig"]