Merge branch 'feat/document-citations' of https://github.com/rachedkko/mmore into feat/document-citations

# Conflicts:
#	src/mmore/run_index_api.py
#	src/mmore/run_retriever.py

Author: rachedkko

.dockerignore

@@ -1,3 +1,2 @@
 .venv
-tests
-test_data
+.git

.github/pyright/pyrightconfig.colvision.json

@@ -0,0 +1,16 @@
+{
+  "include": ["../../src/mmore/colvision", "../../src/mmore/tui"],
+  "exclude": ["../../build", "../../dist", "**/__pycache__", "*.egg-info"],
+  "executionEnvironments": [
+    {
+      "root": "../../src/mmore/colvision",
+      "extraPaths": ["../../src"]
+    },
+    {
+      "root": "../../src/mmore/tui",
+      "extraPaths": ["../../src"]
+    }
+  ],
+  "pythonVersion": "3.11",
+  "typeCheckingMode": "standard"
+}

.github/workflows/pyright.yml

@@ -0,0 +1,36 @@
+name: 📐 Pyright type checks
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  pyright:
+    name: "Pyright"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.11"
+
+      - name: Install uv
+        run: pipx install uv
+
+      - name: Install base dependencies
+        run: uv venv .venv && source .venv/bin/activate && uv pip install -e ".[process,index,rag,api,tui,cpu,dev,websearch]"
+
+      - name: Run Pyright (base)
+        continue-on-error: true
+        run: source .venv/bin/activate && pyright --project pyrightconfig.json
+
+      - name: Install colvision dependencies
+        run: uv venv .venv-colvision && source .venv-colvision/bin/activate && uv pip install -e ".[colvision,tui,cpu,dev]"
+
+      - name: Run Pyright (colvision)
+        continue-on-error: true
+        run: source .venv-colvision/bin/activate && pyright --project .github/pyright/pyrightconfig.colvision.json

.github/workflows/tests.yml

@@ -11,25 +11,27 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
         python-version: ["3.11", "3.12"]
 
+    name: test (py${{ matrix.python-version }})
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v6
 
-      - name: Install uv and create venv
-        run: |
-          pipx install uv
-          uv venv .venv
+      - name: Install uv
+        run: pipx install uv
 
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v6
         with:
           python-version: ${{ matrix.python-version }}
 
-      - name: Install dependencies (using uv)
+      - name: Install dependencies - process (using uv)
         run: |
+          uv venv .venv
           source .venv/bin/activate
           uv pip install -e ".[process,index,rag,api,cpu,dev,websearch]"
 
@@ -39,7 +41,18 @@ jobs:
           uv pip show cohere || echo "Cohere not installed"
           uv pip show langchain-cohere || echo "Langchain-cohere not installed"
 
-      - name: Run tests
+      - name: Run tests - process
         run: |
           source .venv/bin/activate
           pytest
+
+      - name: Install dependencies - colvision
+        run: |
+          uv venv .venv-colvision
+          source .venv-colvision/bin/activate
+          uv pip install -e ".[colvision,cpu,dev]"
+
+      - name: Run tests - colvision
+        run: |
+          source .venv-colvision/bin/activate
+          pytest tests/test_colvision.py

.gitignore

@@ -114,6 +114,7 @@ venv.bak/
 # Milvus DB
 db/
 *.db
+*.db.lock
 
 # Project files
 tmp/

README.md

@@ -16,7 +16,7 @@
 
 MMORE is an open-source, end-to-end pipeline to ingest, process, index, and retrieve knowledge from heterogeneous files: PDFs, Office docs, spreadsheets, emails, images, audio, video, and web pages. It standardizes content into a unified multimodal format, supports distributed CPU/GPU processing, and provides hybrid dense+sparse retrieval with an integrated RAG service (CLI, APIs). 
 
-👉 Read the paper for more details (OpenReview): [MMORE: Massive Multimodal Open RAG & Extraction](https://openreview.net/forum?id=6j1HjfIdKn)
+👉 Read the paper for more details (arXiv): [MMORE: Massive Multimodal Open RAG & Extraction](https://arxiv.org/abs/2509.11937)
 
 
 ### Documentation
@@ -66,6 +66,8 @@ brew install cairo pango gdk-pixbuf libffi
 uv pip install weasyprint
 ```
 
+You can also run MMORE on Windows by following our [Windows setup notes](docs/source/getting_started/windows.md).
+
 #### Step 1 – Install MMORE
 
 Dependencies are split by pipeline stage. Install only what you need:
@@ -103,6 +105,22 @@ uv pip install "mmore[process,cpu]"
 
 > :warning: **Check the instructions for contributors directly at [`docs/for_devs.md`](./docs/for_devs.md)**
 
+### Interactive TUI
+
+Prefer a guided experience over editing YAML by hand? Install the `tui` extra and launch the interactive Terminal UI:
+
+```bash
+uv sync --extra tui
+mmore tui
+```
+
+From the launcher you can:
+
+- run any stage (process / postprocess / index / rag / chat) interactively,
+- chain the full pipeline (process → postprocess → index → chat),
+- generate stage YAML configs through a guided wizard,
+- pick from existing example configs without leaving the terminal.
+
 ### Minimal Example
 
 You can use our predefined CLI commands to execute parts of the pipeline. Note that you might need to prepend `python -m` to the command if the package does not properly create bash aliases.
@@ -142,7 +160,7 @@ MultimodalSample.to_jsonl(out_file, result_pdf)
 
 To launch the MMORE pipeline, follow the specialised instructions in the docs.
 
-![The MMORE pipelines architecture](https://github.com/user-attachments/assets/0cd61466-1680-43ed-9d55-7bd483a04a09)
+![The MMORE pipelines architecture](docs/source/doc_images/pipeline_mmore+.png)
 
 
 1. **:page_facing_up: Input Documents**

docker/arch/Dockerfile

@@ -3,17 +3,15 @@ ARG UV_OVERRIDE
 
 # --- Base images ---
 FROM archlinux:latest AS gpu
-ENV UV_ARGUMENTS="--extra all,cu126"
-RUN echo "Using GPU image"
+ENV SYNC_ARGS="--extra all --extra cu126"
 
 FROM archlinux:latest AS cpu
-ENV UV_ARGUMENTS="--extra all,cpu"
-RUN echo "Using CPU-only image"
+ENV SYNC_ARGS="--extra all --extra cpu"
 
 # --- Build ---
 FROM ${DEVICE:-gpu} AS build
 ARG UV_OVERRIDE
-ENV UV_ARGUMENTS=${UV_OVERRIDE:-$UV_ARGUMENTS}
+ENV SYNC_ARGS=${UV_OVERRIDE:-$SYNC_ARGS}
 RUN echo "DisableSandbox" >> /etc/pacman.conf
 
 ARG USER_UID=1000
@@ -22,15 +20,22 @@ ARG USER_GID=1000
 # Install system dependencies
 RUN pacman -Syu --noconfirm && \
     pacman -S --noconfirm --needed \
-      base-devel python uv \
-      tzdata curl ffmpeg \
+      base-devel \
+      tzdata curl git ffmpeg \
       libsm libxext nss \
       libxi libxrandr libxcomposite libxcursor libxdamage libxfixes libxrender \
       alsa-lib atk gtk3 libreoffice-fresh libjpeg-turbo pango && \
     ln -fs /usr/share/zoneinfo/Europe/Zurich /etc/localtime && \
     echo "Europe/Zurich" > /etc/timezone && \
     pacman -Sc --noconfirm
 
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:0.7.13 /uv /uvx /bin/
+
+ENV UV_PYTHON=3.11 \
+    UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy
+
 # Create non-root user
 RUN groupadd --gid ${USER_GID} mmoreuser \
  && useradd --uid ${USER_UID} --gid ${USER_GID} -m mmoreuser
@@ -39,14 +44,14 @@ WORKDIR /app
 RUN chown mmoreuser:mmoreuser /app
 USER mmoreuser
 
-# Set up Python virtual environment
-ENV VIRTUAL_ENV=/app/.venv
-RUN uv venv --python 3.11 .venv \
- && uv pip install --no-cache weasyprint
-
 # Install dependencies (cached unless uv.lock changes)
 COPY --chown=mmoreuser:mmoreuser pyproject.toml uv.lock /app/
-RUN uv sync --frozen --no-install-project ${UV_ARGUMENTS}
+RUN --mount=type=cache,target=/home/mmoreuser/.cache/uv,uid=${USER_UID},gid=${USER_GID} \
+    uv sync --frozen --no-install-project ${SYNC_ARGS}
+
+# weasyprint needs to be installed using pip
+RUN --mount=type=cache,target=/home/mmoreuser/.cache/uv,uid=${USER_UID},gid=${USER_GID} \
+    uv pip install weasyprint
 
 # Install mmore from local source code
 COPY --chown=mmoreuser:mmoreuser . /app

docker/arch/README.md

@@ -19,9 +19,9 @@ CPU-only:
 sudo docker build -f docker/arch/Dockerfile --build-arg DEVICE=cpu -t mmore:arch-cpu .
 ```
 
-Custom extras (overrides the default `--extra all,cu126` or `--extra all,cpu`):
+Custom extras (overrides the default `--extra all --extra cu126` or `--extra all --extra cpu`):
 ```bash
-sudo docker build -f docker/arch/Dockerfile --build-arg UV_OVERRIDE="--extra all,cu126" -t mmore:arch .
+sudo docker build -f docker/arch/Dockerfile --build-arg UV_OVERRIDE="--extra process --extra rag --extra cpu" -t mmore:arch .
 ```
 
 Custom user UID/GID (e.g. for RCP):

docker/leap/Dockerfile

@@ -3,43 +3,48 @@ ARG UV_OVERRIDE
 
 # --- Base images ---
 FROM opensuse/leap:15.6 AS gpu
-ENV UV_ARGUMENTS="--extra all,cu126"
-RUN echo "Using GPU image"
+ENV SYNC_ARGS="--extra all --extra cu126"
 
 FROM opensuse/leap:15.6 AS cpu
-ENV UV_ARGUMENTS="--extra all,cpu"
-RUN echo "Using CPU-only image"
+ENV SYNC_ARGS="--extra all --extra cpu"
 
 # --- Build ---
 FROM ${DEVICE:-gpu} AS build
 ARG UV_OVERRIDE
-ENV UV_ARGUMENTS=${UV_OVERRIDE:-$UV_ARGUMENTS}
+ENV SYNC_ARGS=${UV_OVERRIDE:-$SYNC_ARGS}
 
 # Install system dependencies
 RUN zypper --non-interactive refresh && \
     zypper --non-interactive install -y \
-      python311 python311-pip \
-      nano curl ffmpeg \
+      nano curl git ffmpeg \
       libSM6 libXext6 mozilla-nss \
       libXi6 libXrandr2 libXcomposite1 libXcursor1 libXdamage1 libXfixes3 libXrender1 \
       libasound2 libatk-1_0-0 gtk3 libreoffice libjpeg8-devel libpango-1_0-0 && \
     ln -fs /usr/share/zoneinfo/Europe/Zurich /etc/localtime && \
     echo "Europe/Zurich" > /etc/timezone && \
     zypper clean --all
 
-WORKDIR /app
+# Install uv
+COPY --from=ghcr.io/astral-sh/uv:0.7.13 /uv /uvx /bin/
+
+ENV UV_PYTHON=3.11 \
+    UV_PROJECT_ENVIRONMENT=/app/.venv \
+    UV_LINK_MODE=copy
 
-# Set up Python virtual environment
-RUN python3.11 -m venv .venv \
- && .venv/bin/pip install --no-cache-dir uv weasyprint
+WORKDIR /app
 
 # Install dependencies (cached unless uv.lock changes)
 COPY pyproject.toml uv.lock /app/
-RUN .venv/bin/uv sync --frozen --no-install-project ${UV_ARGUMENTS}
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv sync --frozen --no-install-project ${SYNC_ARGS}
+
+# weasyprint needs to be installed using pip
+RUN --mount=type=cache,target=/root/.cache/uv \
+    uv pip install weasyprint
 
 # Install mmore from local source code
 COPY . /app
-RUN .venv/bin/uv pip install --no-cache --no-deps -e .
+RUN uv pip install --no-cache --no-deps -e .
 
 # --- Runtime ---
 ENV PATH="/app/.venv/bin:$PATH"

docker/leap/README.md

@@ -19,9 +19,9 @@ CPU-only:
 sudo docker build -f docker/leap/Dockerfile --build-arg DEVICE=cpu -t mmore:leap-cpu .
 ```
 
-Custom extras (overrides the default `--extra all,cu126` or `--extra all,cpu`):
+Custom extras (overrides the default `--extra all --extra cu126` or `--extra all --extra cpu`):
 ```bash
-sudo docker build -f docker/leap/Dockerfile --build-arg UV_OVERRIDE="--extra all,cu126" -t mmore:leap .
+sudo docker build -f docker/leap/Dockerfile --build-arg UV_OVERRIDE="--extra process --extra rag --extra cpu" -t mmore:leap .
 ```
 
 ## Run