fix(ci): include moltis memory crate in vendored source

Author: serhiicherepanov

.docker/moltis/.gitignore

@@ -1,6 +1,6 @@
 /target
 .superset
-memory
+/memory
 prompts
 .envrc
 .env

.docker/moltis/crates/memory/Cargo.toml

@@ -0,0 +1,36 @@
+[package]
+edition.workspace = true
+name              = "moltis-memory"
+version.workspace = true
+
+[features]
+default          = ["file-watcher"]
+file-watcher     = ["dep:notify-debouncer-full"]
+local-embeddings = ["dep:llama-cpp-2"]
+metrics          = ["dep:moltis-metrics"]
+
+[dependencies]
+anyhow                = { workspace = true }
+async-trait           = { workspace = true }
+chrono                = { features = ["serde"], workspace = true }
+directories.workspace = true
+llama-cpp-2           = { optional = true, workspace = true }
+moltis-agents         = { workspace = true }
+moltis-common         = { workspace = true }
+moltis-metrics        = { optional = true, workspace = true }
+notify-debouncer-full = { optional = true, workspace = true }
+reqwest               = { workspace = true }
+secrecy               = { workspace = true }
+serde                 = { workspace = true }
+serde_json            = { workspace = true }
+sha2                  = { workspace = true }
+sqlx                  = { workspace = true }
+tokio                 = { workspace = true }
+tracing               = { workspace = true }
+walkdir               = { workspace = true }
+
+[dev-dependencies]
+tempfile = { workspace = true }
+
+[lints]
+workspace = true

.docker/moltis/crates/memory/migrations/20240205100004_init.sql

@@ -0,0 +1,54 @@
+-- Memory system schema
+
+CREATE TABLE IF NOT EXISTS files (
+    path       TEXT    NOT NULL PRIMARY KEY,
+    source     TEXT    NOT NULL,
+    hash       TEXT    NOT NULL,
+    mtime      INTEGER NOT NULL,
+    size       INTEGER NOT NULL
+);
+
+CREATE TABLE IF NOT EXISTS chunks (
+    id         TEXT    NOT NULL PRIMARY KEY,
+    path       TEXT    NOT NULL,
+    source     TEXT    NOT NULL,
+    start_line INTEGER NOT NULL,
+    end_line   INTEGER NOT NULL,
+    hash       TEXT    NOT NULL,
+    model      TEXT    NOT NULL,
+    text       TEXT    NOT NULL,
+    embedding  BLOB,
+    updated_at TEXT    NOT NULL DEFAULT (datetime('now')),
+    FOREIGN KEY (path) REFERENCES files(path) ON DELETE CASCADE
+);
+
+CREATE TABLE IF NOT EXISTS embedding_cache (
+    provider     TEXT NOT NULL,
+    model        TEXT NOT NULL,
+    provider_key TEXT NOT NULL,
+    hash         TEXT NOT NULL,
+    embedding    BLOB NOT NULL,
+    dims         INTEGER NOT NULL,
+    updated_at   TEXT NOT NULL DEFAULT (datetime('now')),
+    PRIMARY KEY (provider, model, provider_key, hash)
+);
+
+CREATE VIRTUAL TABLE IF NOT EXISTS chunks_fts USING fts5(
+    text,
+    content=chunks,
+    content_rowid=rowid
+);
+
+-- Triggers to keep FTS in sync
+CREATE TRIGGER IF NOT EXISTS chunks_ai AFTER INSERT ON chunks BEGIN
+    INSERT INTO chunks_fts(rowid, text) VALUES (new.rowid, new.text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS chunks_ad AFTER DELETE ON chunks BEGIN
+    INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
+END;
+
+CREATE TRIGGER IF NOT EXISTS chunks_au AFTER UPDATE ON chunks BEGIN
+    INSERT INTO chunks_fts(chunks_fts, rowid, text) VALUES ('delete', old.rowid, old.text);
+    INSERT INTO chunks_fts(rowid, text) VALUES (new.rowid, new.text);
+END;

.docker/moltis/crates/memory/src/chunker.rs

@@ -0,0 +1,138 @@
+//! Split markdown text into overlapping chunks by line, targeting a token count.
+//!
+//! Tokens are approximated as whitespace-split words.
+
+/// A chunk produced by the chunker.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Chunk {
+    pub text: String,
+    pub start_line: usize,
+    pub end_line: usize,
+}
+
+/// Split `text` into chunks of approximately `chunk_size` tokens with `overlap` token overlap.
+///
+/// Lines are never split mid-line. Each chunk records its 1-based start and end line numbers.
+pub fn chunk_markdown(text: &str, chunk_size: usize, overlap: usize) -> Vec<Chunk> {
+    if text.is_empty() || chunk_size == 0 {
+        return vec![];
+    }
+
+    let lines: Vec<&str> = text.lines().collect();
+    if lines.is_empty() {
+        return vec![];
+    }
+
+    let line_tokens: Vec<usize> = lines
+        .iter()
+        .map(|l| l.split_whitespace().count().max(1)) // empty lines count as 1 token
+        .collect();
+
+    let mut chunks = Vec::new();
+    let mut start = 0;
+
+    while start < lines.len() {
+        let mut end = start;
+        let mut tokens = 0;
+
+        // Accumulate lines until we reach chunk_size tokens
+        while end < lines.len() && tokens + line_tokens[end] <= chunk_size {
+            tokens += line_tokens[end];
+            end += 1;
+        }
+
+        // If we couldn't fit even one line, take it anyway
+        if end == start {
+            end = start + 1;
+        }
+
+        let chunk_text: String = lines[start..end].join("\n");
+        chunks.push(Chunk {
+            text: chunk_text,
+            start_line: start + 1, // 1-based
+            end_line: end,         // 1-based inclusive
+        });
+
+        if end >= lines.len() {
+            break;
+        }
+
+        // Move start forward, keeping `overlap` tokens of context
+        let mut overlap_tokens = 0;
+        let mut new_start = end;
+        while new_start > start && overlap_tokens < overlap {
+            new_start -= 1;
+            overlap_tokens += line_tokens[new_start];
+        }
+
+        // Ensure progress
+        if new_start <= start {
+            new_start = start + 1;
+        }
+        start = new_start;
+    }
+
+    chunks
+}
+
+#[allow(clippy::unwrap_used, clippy::expect_used)]
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_empty() {
+        assert!(chunk_markdown("", 400, 80).is_empty());
+    }
+
+    #[test]
+    fn test_single_small_chunk() {
+        let text = "hello world\nfoo bar";
+        let chunks = chunk_markdown(text, 400, 80);
+        assert_eq!(chunks.len(), 1);
+        assert_eq!(chunks[0].start_line, 1);
+        assert_eq!(chunks[0].end_line, 2);
+        assert_eq!(chunks[0].text, text);
+    }
+
+    #[test]
+    fn test_multiple_chunks_with_overlap() {
+        // Create text with ~10 tokens per line, 5 lines = 50 tokens
+        let lines: Vec<String> = (0..10)
+            .map(|i| format!("line {} has several words in it here now ok", i))
+            .collect();
+        let text = lines.join("\n");
+
+        let chunks = chunk_markdown(&text, 20, 5);
+        assert!(chunks.len() > 1);
+
+        // Verify overlap: last lines of chunk N should overlap with first lines of chunk N+1
+        for i in 0..chunks.len() - 1 {
+            assert!(
+                chunks[i + 1].start_line <= chunks[i].end_line,
+                "chunk {} end_line {} should overlap with chunk {} start_line {}",
+                i,
+                chunks[i].end_line,
+                i + 1,
+                chunks[i + 1].start_line
+            );
+        }
+
+        // Verify all lines are covered
+        assert_eq!(chunks[0].start_line, 1);
+        assert_eq!(chunks.last().unwrap().end_line, 10);
+    }
+
+    #[test]
+    fn test_line_numbers_are_1_based() {
+        let text = "a\nb\nc";
+        let chunks = chunk_markdown(text, 1, 0);
+        assert_eq!(chunks[0].start_line, 1);
+        assert_eq!(chunks[0].end_line, 1);
+    }
+
+    #[test]
+    fn test_zero_chunk_size() {
+        assert!(chunk_markdown("hello", 0, 0).is_empty());
+    }
+}

.docker/moltis/crates/memory/src/config.rs

@@ -0,0 +1,76 @@
+use std::path::PathBuf;
+
+/// Citation mode for memory search results.
+#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
+pub enum CitationMode {
+    /// Always include citations in search results.
+    On,
+    /// Never include citations.
+    Off,
+    /// Auto: include citations when results come from multiple files.
+    #[default]
+    Auto,
+}
+
+impl std::str::FromStr for CitationMode {
+    type Err = std::convert::Infallible;
+
+    /// Parse from string (case-insensitive). Never fails - defaults to Auto.
+    fn from_str(s: &str) -> Result<Self, Self::Err> {
+        Ok(match s.to_lowercase().as_str() {
+            "on" | "true" | "yes" | "always" => Self::On,
+            "off" | "false" | "no" | "never" => Self::Off,
+            _ => Self::Auto,
+        })
+    }
+}
+
+/// Configuration for the memory subsystem.
+#[derive(Debug, Clone)]
+pub struct MemoryConfig {
+    /// Path to the SQLite database file (or `:memory:` for tests).
+    pub db_path: String,
+    /// Root data directory for writing memory files (e.g. `~/.moltis/`).
+    /// Required for `MemoryWriter` support. `None` disables writes.
+    pub data_dir: Option<PathBuf>,
+    /// Directories to scan for markdown files.
+    pub memory_dirs: Vec<PathBuf>,
+    /// Target chunk size in tokens (approximate, counted as whitespace-split words).
+    pub chunk_size: usize,
+    /// Overlap between consecutive chunks in tokens.
+    pub chunk_overlap: usize,
+    /// Weight for vector similarity in hybrid search (0.0–1.0).
+    pub vector_weight: f32,
+    /// Weight for keyword/FTS similarity in hybrid search (0.0–1.0).
+    pub keyword_weight: f32,
+    /// Path to a local GGUF model file for offline embeddings.
+    /// If `None`, the default model path will be used when `local-embeddings` feature is enabled.
+    pub local_model_path: Option<PathBuf>,
+    /// Whether to enable batch embedding via the OpenAI batch API (opt-in).
+    pub batch_embeddings: bool,
+    /// Minimum number of texts before switching to batch API (default: 50).
+    pub batch_threshold: usize,
+    /// Citation mode for search results.
+    pub citations: CitationMode,
+    /// Whether to enable LLM reranking for hybrid search results.
+    pub llm_reranking: bool,
+}
+
+impl Default for MemoryConfig {
+    fn default() -> Self {
+        Self {
+            db_path: "memory.db".into(),
+            data_dir: None,
+            memory_dirs: vec![PathBuf::from("memory")],
+            chunk_size: 400,
+            chunk_overlap: 80,
+            vector_weight: 0.7,
+            keyword_weight: 0.3,
+            local_model_path: None,
+            batch_embeddings: false,
+            batch_threshold: 50,
+            citations: CitationMode::default(),
+            llm_reranking: false,
+        }
+    }
+}

.docker/moltis/crates/memory/src/embeddings.rs

@@ -0,0 +1,29 @@
+/// Provider-agnostic embedding trait for generating vectors from text.
+use async_trait::async_trait;
+
+#[async_trait]
+pub trait EmbeddingProvider: Send + Sync {
+    /// Generate an embedding for a single text.
+    async fn embed(&self, text: &str) -> anyhow::Result<Vec<f32>>;
+
+    /// Generate embeddings for a batch of texts.
+    /// Default implementation calls `embed` sequentially.
+    async fn embed_batch(&self, texts: &[String]) -> anyhow::Result<Vec<Vec<f32>>> {
+        let mut results = Vec::with_capacity(texts.len());
+        for text in texts {
+            results.push(self.embed(text).await?);
+        }
+        Ok(results)
+    }
+
+    /// The model name used by this provider (e.g. "text-embedding-3-small").
+    fn model_name(&self) -> &str;
+
+    /// The dimensionality of the embeddings produced.
+    fn dimensions(&self) -> usize;
+
+    /// A stable key identifying this provider configuration for cache discrimination.
+    /// Different providers or the same provider with different settings should return
+    /// different keys.
+    fn provider_key(&self) -> &str;
+}