Merge branch 'master' into feat/gh-actions

Author: paultltc

Dockerfile

@@ -4,12 +4,12 @@ ARG UV_ARGUMENTS=""
 # We select the image based on the platform argument
 
 # Define GPU image
-FROM "nvidia/cuda:12.2.2-base-ubuntu22.04" AS gpu
+FROM nvidia/cuda:12.2.2-base-ubuntu22.04 AS gpu
 ARG PLATFORM
 RUN echo "Using GPU image"
 
 # Define cpu image
-FROM ubuntu:22.04 as cpu
+FROM ubuntu:22.04 AS cpu
 ARG PLATFORM
 ARG UV_ARGUMENTS="--extra cpu"
 RUN echo "Using CPU-only image"
@@ -20,10 +20,17 @@ ARG PLATFORM
 
 COPY --from=ghcr.io/astral-sh/uv:0.5.8 /uv /uvx /bin/
 
-RUN apt-get update && \
-   apt-get install -y  --no-install-recommends  \
-      nano curl ffmpeg libsm6 libxext6 chromium-browser libnss3 libgconf-2-4 libxi6 libxrandr2 libxcomposite1 libxcursor1 libxdamage1 libxext6 libxfixes3 libxrender1 libasound2 libatk1.0-0 libgtk-3-0 libreoffice libjpeg-dev
+RUN apt-get update && apt-get install -y python3-pip && pip3 install uv
 
+RUN apt-get update && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+      tzdata nano curl ffmpeg libsm6 libxext6 chromium-browser libnss3 libgconf-2-4 \
+      libxi6 libxrandr2 libxcomposite1 libxcursor1 libxdamage1 libxfixes3 libxrender1 \
+      libasound2 libatk1.0-0 libgtk-3-0 libreoffice libjpeg-dev libpango-1.0-0 \
+      libpangoft2-1.0-0 weasyprint && \
+    ln -fs /usr/share/zoneinfo/Europe/Zurich /etc/localtime && \
+    dpkg-reconfigure --frontend noninteractive tzdata && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
 
 # Copy the project into the image
 ADD . /app
@@ -32,13 +39,16 @@ ADD . /app
 WORKDIR /app
 
 # Define the build argument with a default value of an empty string (optional)
-
-RUN uv sync --frozen ${UV_ARGUMENTS}
+COPY pyproject.toml ./
+# RUN uv sync --frozen ${UV_ARGUMENTS}
 
 
 # make uv's python the default python for the image
 ENV PATH="/app/.venv/bin:$PATH"
 
+# install mpmath
+RUN uv pip install -e . --system
+
 ENV DASK_DISTRIBUTED__WORKER__DAEMON=False
 
 ENTRYPOINT /bin/bash

README.md

@@ -29,7 +29,8 @@ Our package requires system dependencies. This snippet will take care of install
 sudo apt update
 sudo apt install -y ffmpeg libsm6 libxext6 chromium-browser libnss3 \
   libgconf-2-4 libxi6 libxrandr2 libxcomposite1 libxcursor1 libxdamage1 \
-  libxext6 libxfixes3 libxrender1 libasound2 libatk1.0-0 libgtk-3-0 libreoffice
+  libxext6 libxfixes3 libxrender1 libasound2 libatk1.0-0 libgtk-3-0 libreoffice \
+  libpango-1.0-0 libpangoft2-1.0-0 weasyprint
 ```
 
 #### Step 1 – Install MMORE
@@ -40,12 +41,6 @@ To install the package simply run:
 pip install -e .
 ```
 
-To install additional RAG-related dependencies, run:
-
-```bash
-pip install -e '.[rag]'
-```
-
 > :warning: This is a big package with a lot of dependencies, so we recommend to use `uv` to handle `pip` installations. [Check our tutorial on uv](./docs/uv.md).
 
 ### Minimal Example
@@ -54,13 +49,14 @@ You can use our predefined CLI commands to execute parts of the pipeline. Note t
 
 ```bash
 # Run processing
-mmore process --config-file examples/process/config.yaml
+python -m mmore process --config-file examples/process/config.yaml
+python -m mmore postprocess --config-file examples/postprocessor/config.yaml --input-data examples/process/outputs/merged/merged_results.jsonl
 
 # Run indexer
-mmore index --config-file examples/index/config.yaml
+python -m mmore index --config-file examples/index/config.yaml --documents-path examples/process/outputs/merged/final_pp.jsonl
 
 # Run RAG
-mmore rag --config-file examples/rag/api/rag_api.yaml
+python -m mmore rag --config-file examples/rag/config.yaml
 ```
 
 You can also use our package in python code as shown here:
@@ -70,35 +66,34 @@ from mmore.process.processors.pdf_processor import PDFProcessor
 from mmore.process.processors.base import ProcessorConfig
 from mmore.type import MultimodalSample
 
-pdf_file_paths = ["examples/sample_data/pdf/calendar.pdf"]
-out_file = "results/example.jsonl"
+pdf_file_paths = ["/path/to/examples/sample_data/pdf/calendar.pdf"] #write here the full path, not a relative path
+out_file = "/path/to/examples/process/outputs/example.jsonl"
 
-pdf_processor_config = ProcessorConfig(custom_config={"output_path": "results"})
+pdf_processor_config = ProcessorConfig(custom_config={"output_path": "examples/process/outputs"})
 pdf_processor = PDFProcessor(config=pdf_processor_config)
-result_pdf = pdf_processor.process_batch(pdf_file_paths, True, 1) # args: file_paths, fast mode (True/False), num_workers
+result_pdf = pdf_processor.process_batch(pdf_file_paths, False, 1) # args: file_paths, fast mode (True/False), num_workers
 
 MultimodalSample.to_jsonl(out_file, result_pdf)
 ```
 
 ---
 
-
 ### Usage
 
-To launch the MMORE pipeline follow the specialised instructions in the docs.
+To launch the MMORE pipeline, follow the specialised instructions in the docs.
 
 ![The MMORE pipelines archicture](https://github.com/user-attachments/assets/0cd61466-1680-43ed-9d55-7bd483a04a09)
 
 
 1. **:page_facing_up: Input Documents**  
-   Upload your multimodal documents (PDFs, videos, spreadsheets, and more) into the pipeline.
+   Upload your multimodal documents (PDFs, videos, spreadsheets, and m(m)ore) into the pipeline.
 
 2. [**:mag: Process**](./docs/process.md) 
-   Extracts and standardizes text, metadata, and multimedia content from diverse file formats. Easily extensible! You can add your own processors to handle new file types.  
+   Extracts and standardizes text, metadata, and multimedia content from diverse file formats. Easily extensible! You can add your own processors to handle new file types.
    *Supports fast processing for specific types.*
 
 3. [**:file_folder: Index**](./docs/index.md) 
-   Organizes extracted data into a **hybrid retrieval-ready Vector Store DB**, combining dense and sparse indexing through [Milvus](https://milvus.io/). Your vector DB can also be remotely hosted and then you only have to provide a standard API. 
+   Organizes extracted data into a **hybrid retrieval-ready Vector Store DB**, combining dense and sparse indexing through [Milvus](https://milvus.io/). Your vector DB can also be remotely hosted and then you only have to provide a standard API. There is also an [HTTP Index API](./docs/index_api.md) for adding new files on the fly with HTTP requests.
 
 4. [**:robot: RAG**](./docs/rag.md) 
    Use the indexed documents inside a **Retrieval-Augmented Generation (RAG) system**  that provides a [LangChain](https://www.langchain.com/) interface. Plug in any LLM with a compatible interface or add new ones through an easy-to-use interface.
@@ -108,7 +103,7 @@ To launch the MMORE pipeline follow the specialised instructions in the docs.
    *Coming soon*
    An easy way to evaluate the performance of your RAG system using Ragas.
 
-See [the `/docs` directory](/docs) for additional details on each modules and hands-on tutorials on parts of the pipeline.
+See [the `/docs` directory](./docs) for additional details on each modules and hands-on tutorials on parts of the pipeline.
 
 
 #### :construction: Supported File Types  
@@ -118,7 +113,7 @@ See [the `/docs` directory](/docs) for additional details on each modules and ha
 | **Text Documents** | DOCX, MD, PPTX, XLSX, TXT, EML           | CPU                      | :x:
 | **PDFs**           | PDF                                     | GPU/CPU                  | :white_check_mark:
 | **Media Files**    | MP4, MOV, AVI, MKV, MP3, WAV, AAC       | GPU/CPU                  | :white_check_mark:
-| **Web Content (TBD)**    | Webpages                                | GPU/CPU                  | :white_check_mark:
+| **Web Content**    | HTML                                    | CPU                      | :white_check_mark:
 
 
 ## Contributing
@@ -134,7 +129,3 @@ Don't hesitate to star the project :star: if you find it interesting! (you would
 ## License
 
 This project is licensed under the Apache 2.0 License, see the [LICENSE :mortar_board:](LICENSE) file for details.
-
-## Acknowledgements
-
-This project is part of the [**OpenMeditron**](https://huggingface.co/OpenMeditron) initiative developed in [LiGHT](https://www.light-laboratory.org/) lab at EPFL/Yale/CMU Africa in collaboration with the [**SwissAI**](https://www.swiss-ai.org/) initiative. Thank you Scott Mahoney, Mary-Anne Hartley

src/mmore/dashboard/dashboard_readme.md docs/dashboard.mdsrc/mmore/dashboard/dashboard_readme.md renamed to docs/dashboard.md

@@ -14,15 +14,15 @@
 Before setting up the dashboard, it is useful to understand how it works. You can think of the dashboard as being made up of 4 separate parts:
 
 <p align="center">
-  <img src="doc_images/image.png" width="1000">
+  <img src="doc_images/backend_image.png" width="1000">
 </p>
 
 |  |   |
 |------------|---|
-| **Frontend:** the actual dashboard user interface (UI), and what will be displayed on your screen. | ![Frontend](doc_images/image%201.png) |
-| **Database:** the database which stores information about the file processing. | ![Database](doc_images/image%202.png) |
-| **Processing Pipeline:** the pipeline processing your documents for which you want to be able to visualize on the dashboard. | ![Pipeline](doc_images/image%203.png) |
-| **Backend Server:** *backend* is what we call the server that acts like the middle man to the 3 elements above. It receives information from the processing pipeline, stores and retrieves data from the database and sends information to be displayed on the frontend dashboard. | ![Backend](doc_images/image%204.png) |
+| **Frontend:** the actual dashboard user interface (UI), and what will be displayed on your screen. | ![Frontend](doc_images/backend_image%201.png) |
+| **Database:** the database which stores information about the file processing. | ![Database](doc_images/backend_image%202.png) |
+| **Processing Pipeline:** the pipeline processing your documents for which you want to be able to visualize on the dashboard. | ![Pipeline](doc_images/backend_image%203.png) |
+| **Backend Server:** *backend* is what we call the server that acts like the middle man to the 3 elements above. It receives information from the processing pipeline, stores and retrieves data from the database and sends information to be displayed on the frontend dashboard. | ![Backend](doc_images/backend_image%204.png) |
 
 
 ## 2. Setup
@@ -35,15 +35,15 @@ Each element shown above is created in a different terminal. This means that you
 Official documentation for MongoDB setup can be found [here](https://www.mongodb.com/docs/manual/tutorial/install-mongodb-on-ubuntu/) (Ubuntu 22.04 Jammy release). 
 
 <p align="center">
-  <img src="doc_images/image%205.png" width="500">
+  <img src="doc_images/backend_image%205.png" width="500">
 </p>
 
 ### Manual Setup Instructions
 
 1. **Install required tools**
 
 ```bash
-sudo apt-get install gnupg curl
+sudo apt install gnupg curl
 ```
 
 - `gnupg`: Encryption tool for secure communication and data storage
@@ -74,25 +74,25 @@ This adds the official MongoDB repository to your package sources, specificall
 4. **Install MongoDB**
 
 ```bash
-sudo apt-get update
-sudo apt-get install -y mongodb-org
-sudo apt-get install -y mongodb-org=8.0.5 mongodb-org-database=8.0.5 mongodb-org-server=8.0.5 mongodb-mongosh mongodb-org-mongos=8.0.5 mongodb-org-tools=8.0.5
+sudo apt update
+sudo apt install -y mongodb-org=8.0.5
+sudo apt install -y mongodb-org-database mondogb-org-server mongodb-mongosh mongodb-org-mongos mongodb-org-tools
 ```
 
 > ✏️ **Note**: You will be prompted to select your timezone during installation. For instance for Switzerland, enter '8' for Europe and then '63' for the timezone.
 
 5. **Create Data Directory**
 
 ```bash
-mkdir -p ~/mongodb
+mkdir -p ~/mongodb
 ```
 
 Creates a directory in root folder to store MongoDB data files. 
 
 6. **Start the MongoDB Server**
 
 ```bash
-mongod --bind_ip_all --dbpath ~/mongodb
+mongod --bind_ip_all --dbpath ~/mongodb
 ```
 
 This starts MongoDB with the following configuration:
@@ -154,7 +154,7 @@ This script automatically checks if MongoDB is installed, installs it if needed
 This backend serves as the bridge between the **database,** the **frontend** and **processing pipeline**, providing a clean API to interact with the data without direct database access.
 
 <p align="center">
-  <img src="doc_images/image%206.png" width="800">
+  <img src="doc_images/backend_image%206.png" width="800">
 </p>
 
 ### Setup Instructions
@@ -166,13 +166,7 @@ This backend serves as the bridge between the **database,** the **frontend** and
 source .venv/bin/activate
 ```
 
-2. **Install Dependencies**
-
-```bash
-pip install -r src/mmore/dashboard/backend/backend_requirements.txt
-```
-
-3. **Configure MongoDB Connection**
+2. **Configure MongoDB Connection**
 
 ```bash
 export MONGODB_URL="mongodb://localhost:27017"
@@ -182,12 +176,12 @@ Sets the environment variable to tell the backend how to connect to MongoDB inst
 
 > 🚨 **Important**: Your MongoDB server should be active before starting the backend.
 
-4. **Start the Backend Server**
+3. **Start the Backend Server**
 
 Run the backend on port 8000
 
 ```bash
-python -m uvicorn src.mmore.dashboard.backend.main:app --host 0.0.0.0 --port 8000
+python3 -m mmore dashboard-backend --host 0.0.0.0 --port 8000
 ```
 
 This command:
@@ -199,7 +193,7 @@ This command:
 
 > 🚨 **Important**: Keep this terminal window open. The backend runs in the foreground and closing the terminal will shut down the server.
 
-5. **Verify the Backend is Running**
+4. **Verify the Backend is Running**
 
 You can check if the backend is running correctly by accessing [http://localhost:8000](http://localhost:8000). You should see a response like: `{"message": "Hello World"}`
 
@@ -215,53 +209,41 @@ The next step is to set up the frontend that will communicate with this backend
 This frontend serves as the user-facing component of the system, providing an interface for monitoring and controlling the processing pipeline without requiring direct interaction with the database or backend code.
 
 <p align="center">
-  <img src="doc_images/image%207.png" width="1000">
+  <img src="doc_images/backend_image%207.png" width="1000">
 </p>
 
 1. **Load Node Version Manager**
 
-```bash
-source /usr/local/nvm/nvm.sh
-```
-
-This loads Node Version Manager (NVM) into your current shell session. NVM is necessary because the frontend requires a specific version of Node.js that differs from the default version installed on the system.
+If it is not already installed, install NVM following the [instructions](https://github.com/nvm-sh/nvm?tab=readme-ov-file#installing-and-updating). Then make sure the Node Version Manager (NVM) is loaded into your current shell session. NVM is necessary because the frontend requires a specific version of Node.js that may differ from the default version installed on the system.
 
 2. **Install and Activate Node.js Version 23** 
 
 ```bash
-sudo -i # give root privileges 
-nvm install 23
-exit # exit root 
-nvm use 23
+nvm install 23
+nvm use 23
 ```
 
-This sequence:
-
-- Starts a shell with root privileges (necessary for the installation)
-- Uses NVM to install Node.js version 23
-- Exits the root shell
-- Sets version 23 as the active Node.js version for your current session
 3. **Install Dependencies**
 
 ```bash
 cd src/mmore/dashboard/frontend # navigte to frontend directory
-npm install
+npm install
 ```
 
 This command uses NPM (Node Package Manager) to install all JavaScript dependencies defined in the package.json file. These are libraries and frameworks needed by the frontend.
 
 4. **Configure Backend URL** 
 
 ```bash
-export VITE_BACKEND_API_URL="http://localhost:8000"
+export VITE_BACKEND_API_URL="http://0.0.0.0:8000"
 ```
 
 Sets an environment variable that tells the frontend where to find the backend API. Vite (the build tool) will use this variable during development.
 
 5. **Start Frontend Server**
 
 ```bash
-npm run dev
+npm run dev
 ```
 
 Executes the development script defined in package.json, and starts a local development server for the frontend application. The terminal will show the URL where the frontend is available (typically [http://localhost:5173](http://localhost:5173/)).
@@ -272,7 +254,7 @@ Executes the development script defined in package.json, and starts a local dev
 To complete the dashboard setup, you need to run a process module that will generate data for visualization in the UI. 
 
 <p align="center">
-  <img src="doc_images/image%208.png" width="1000">
+  <img src="doc_images/backend_image%208.png" width="1000">
 </p>
 
 1. **Modify Configuration File**
@@ -289,7 +271,7 @@ source .venv/bin/activate
 3. **Run the Process Module**
 
 ```bash
-python -m src.mmore.processing.run_processor --config examples/process/config.yaml
+python3 -m mmore process --config-file examples/process/config.yaml
 ```
 
 4. **Monitor the Dashboard**