Merge pull request #23 from artiz/claude/fleischwolf-rag-system-3knaso

feat(rag): add fleischwolf-rag — pluggable RAG subsystem

Author: artiz

.env.example

@@ -0,0 +1,71 @@
+# fleischwolf-rag configuration. Copy to `.env` and adjust.
+# Every key has a built-in default (shown here), so an empty .env still runs
+# with the offline-friendly stack (SQLite + Ollama).
+
+# --- Database (vector store) ---
+# sqlite | postgres | memory
+RAG_DB_BACKEND=sqlite
+# sqlite://<path>  or  postgres://user:pass@host:5432/dbname
+RAG_DATABASE_URL=sqlite://data/rag.db
+
+# --- Embeddings ---
+# ollama | gemini | onnx | hash
+RAG_EMBED_PROVIDER=ollama
+RAG_EMBED_MODEL=bge-m3
+RAG_EMBED_DIM=1024
+# word | token
+RAG_CHUNK_UNIT=word
+
+# Ollama (default provider)
+OLLAMA_BASE_URL=http://localhost:11434
+
+# Gemini (RAG_EMBED_PROVIDER=gemini)
+GEMINI_API_KEY=
+RAG_GEMINI_MODEL=gemini-embedding-001
+
+# Local ONNX (RAG_EMBED_PROVIDER=onnx; build with --features onnx-embed)
+RAG_EMBED_ONNX_PATH=models/embed/bge-m3.onnx
+RAG_EMBED_TOKENIZER=models/embed/tokenizer.json
+
+# --- LLM (OpenRouter) — for Multi-Query, HyDE, and answer synthesis ---
+# OpenRouter keys start with "sk-or-". Any OpenAI-compatible endpoint works:
+# with a native DeepSeek key (sk-...) use
+#   OPENROUTER_BASE_URL=https://api.deepseek.com and RAG_LLM_MODEL=deepseek-chat
+OPENROUTER_API_KEY=
+OPENROUTER_BASE_URL=https://openrouter.ai/api/v1
+RAG_LLM_MODEL=deepseek/deepseek-chat
+
+# --- Chunking ---
+RAG_CHUNK_SIZE=300
+RAG_CHUNK_OVERLAP=0.05
+
+# --- Retrieval ---
+# vector | bm25 | hybrid | multi-query | hyde
+RAG_RETRIEVAL_MODE=hybrid
+RAG_TOP_K=5
+RAG_RRF_K=60
+RAG_MULTIQUERY_N=4
+
+# --- Document source ---
+# folder | ftp | sftp   (ftp/sftp require --features remote-sources)
+RAG_SOURCE=folder
+RAG_SOURCE_PATH=./rag/data
+# For ftp/sftp:
+RAG_SOURCE_URL=
+RAG_SOURCE_USER=
+RAG_SOURCE_PASSWORD=
+
+# folder to store ingested documents in MD (for debugging / re-ingestion)
+RAG_DOCUMENTS_OUTPUT=./data/
+
+# --- Message queue (async ingestion) ---
+# memory | rabbitmq | redis   (rabbitmq/redis require the matching feature)
+RAG_QUEUE=memory
+RABBITMQ_URL=
+REDIS_URL=
+
+# --- REST API (fleischwolf-rag serve) ---
+RAG_HTTP_ADDR=127.0.0.1:3222
+# Comma-separated list of accepted API keys (X-Api-Key / Authorization: Bearer).
+# The server refuses to start with an empty list. CHANGE THESE.
+RAG_API_KEYS=dev-key-change-me

.gitignore

@@ -4,6 +4,11 @@ Cargo.lock
 # Secrets — never commit (holds CARGO_REGISTRY_TOKEN)
 .env
 .env.*
+# …but the documented template for fleischwolf-rag config is committed.
+!.env.example
+
+# Local RAG vector store (default RAG_DATABASE_URL=sqlite://data/rag.db)
+/data/
 
 # Local editable docling installs used by scripts/compare.sh & performance.sh
 # .venv-compare is the lightweight (no-torch) declarative-format env;
@@ -23,3 +28,4 @@ __pycache__/
 # Local docling ground-truth env & scratch dir for this migration work
 /.venv-gt
 /scratch-gt
+

Cargo.toml

@@ -6,6 +6,7 @@
 #   fleischwolf-pdf   — the PDF/image ML pipeline (pdfium text + ONNX layout/OCR)
 #   fleischwolf-asr   — the audio/ASR pipeline (symphonia decode + ONNX Whisper)
 #   fleischwolf-cli   — the `fleischwolf` command-line binary
+#   fleischwolf-rag   — pluggable RAG (chunk/embed/store/search) over the converter
 # See MIGRATION.md for the Python → Rust mapping and what is / isn't ported.
 
 [workspace]
@@ -18,6 +19,9 @@ members = [
     "crates/fleischwolf-pdf",
     # Node.js / Bun N-API bindings (published to npm, not crates.io).
     "crates/fleischwolf-node",
+    # RAG subsystem: chunking + embeddings + vector store + semantic search.
+    # A self-contained async (tokio) island; does not touch the sync crates above.
+    "crates/fleischwolf-rag",
 ]
 
 [workspace.package]

README.md

@@ -55,6 +55,22 @@ byte-for-byte against live docling, the ML pipeline against a deterministic
 snapshot baseline. See [`COMPARING.md`](./COMPARING.md) and
 `scripts/conformance.sh`.
 
+## RAG subsystem
+
+[`crates/fleischwolf-rag`](./crates/fleischwolf-rag) builds a pluggable
+Retrieval-Augmented-Generation layer on top of the converter: it turns documents
+into Markdown, chunks them (configurable size / overlap), embeds the chunks, and
+stores them in a vector database for semantic search. Every external dependency is
+a swappable trait — embedders (**Ollama**/Gemini/local-ONNX), vector stores
+(**SQLite+sqlite-vec**/PostgreSQL+pgvector), LLM (**OpenRouter**, DeepSeek-V3 by default),
+document sources (**folder**/FTP/SFTP), and message queues
+(**in-process**/RabbitMQ/Redis). It ships Hybrid, Multi-Query fusion and HyDE
+retrieval plus an evaluation harness to compare configurations and an
+API-key-protected REST API (`fleischwolf-rag serve`) for document info and
+search. Configure it via [`.env`](./.env.example); see the
+[crate README](./crates/fleischwolf-rag/README.md) for a quickstart on any
+documents folder.
+
 ## The API
 
 ```rust

crates/fleischwolf-node/examples/package.json

@@ -11,6 +11,6 @@
     "pdf": "node pdf-pipeline.mjs"
   },
   "dependencies": {
-    "fleischwolf": "^0.6.1"
+    "fleischwolf": "^0.9.0"
   }
 }

crates/fleischwolf-rag/Cargo.toml

@@ -0,0 +1,81 @@
+[package]
+name = "fleischwolf-rag"
+description = "Pluggable RAG subsystem for Fleischwolf: chunking, embeddings, vector store, and semantic search."
+version.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+homepage.workspace = true
+readme = "README.md"
+keywords = ["rag", "embeddings", "semantic-search", "vector", "docling"]
+categories = ["text-processing", "database"]
+# MSRV is raised over the workspace default by tokio/sqlx/reqwest; kept in step
+# with fleischwolf-pdf (ort) which already needs a recent toolchain.
+rust-version = "1.82"
+
+[[bin]]
+name = "fleischwolf-rag"
+path = "src/bin/fleischwolf-rag.rs"
+
+[features]
+# The default build is fully offline-testable: bundled SQLite + HTTP embedders.
+default = ["sqlite", "ollama"]
+sqlite = ["sqlx/sqlite", "dep:sqlite-vec", "dep:libsqlite3-sys"]
+postgres = ["sqlx/postgres"]
+# HTTP-only backends ride on reqwest (no native deps) and always compile; the
+# feature flags below just gate the small amount of provider-specific code.
+ollama = []
+gemini = []
+openrouter = []
+# Heavy / native-dependency backends. Real code, but not exercised in offline CI.
+onnx-embed = ["dep:ort", "dep:tokenizers"]
+remote-sources = ["dep:suppaftp", "dep:ssh2"]
+rabbitmq = ["dep:lapin", "dep:futures-lite"]
+redis = ["dep:redis", "dep:futures-lite"]
+
+[dependencies]
+# `>=` requirements on purpose while this crate rides an open PR: release.sh
+# bumps every sibling's exact pin on master with each release, which would
+# break this branch's merge-ref build. Post-merge, release.sh rewrites these
+# to exact pins automatically (its sed matches any quoted version here).
+fleischwolf = { path = "../fleischwolf", version = ">=0.12" }
+# Direct dep only for pdfium_backend::page_count (PDF page metrics); already in
+# the graph via fleischwolf, so this adds no compile cost.
+fleischwolf-pdf = { path = "../fleischwolf-pdf", version = ">=0.12" }
+# Counting slides/sheets in OOXML containers for page metrics (names only).
+zip = { version = "2.2", default-features = false, features = ["deflate"] }
+
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "sync", "fs", "time", "signal"] }
+axum = "0.8"
+async-trait = "0.1"
+thiserror = "2"
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+dotenvy = "0.15"
+clap = { version = "4", features = ["derive", "env"] }
+# rustls-tls so the crypto/TLS stack matches what the workspace already builds via
+# `ort` (tls-rustls) and sqlx below — no OpenSSL system dependency.
+reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+sqlx = { version = "0.8", default-features = false, features = ["runtime-tokio", "tls-rustls"] }
+# sqlite-vec: the `vec0` ANN virtual table, statically compiled into the bundled
+# SQLite. libsqlite3-sys must match sqlx's (cargo enforces one copy via `links`);
+# we need it directly for sqlite3_auto_extension to register the extension.
+sqlite-vec = { version = "0.1.6", optional = true }
+libsqlite3-sys = { version = "0.30", optional = true }
+pulldown-cmark = { version = "0.13", default-features = false }
+uuid = { version = "1", features = ["v4"] }
+sha2 = "0.10"
+tracing = "0.1"
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
+
+# --- feature-gated / optional backends ---
+ort = { version = "2.0.0-rc.12", default-features = false, features = ["std", "download-binaries", "tls-rustls"], optional = true }
+tokenizers = { version = "0.20", default-features = false, features = ["onig"], optional = true }
+suppaftp = { version = "6", optional = true }
+ssh2 = { version = "0.9", optional = true }
+lapin = { version = "2", optional = true }
+futures-lite = { version = "2", optional = true }
+redis = { version = "0.27", default-features = false, features = ["tokio-comp"], optional = true }
+
+[dev-dependencies]
+tokio = { version = "1", features = ["rt-multi-thread", "macros", "test-util"] }