feat: Add source harvester for various data sources

- Introduced a new configuration file `config.example.yaml` for source harvester settings.
- Implemented fetchers for Azure DevOps Wiki, GitHub repositories, HTML documents, and Hugging Face datasets.
- Created a base fetcher class to standardize fetching logic across different sources.
- Added a pipeline to manage the fetching process and store the results in a JSONL format.
- Developed a training matrix generator for Hugging Face datasets, including scoring and prioritization logic.
- Included example YAML and Markdown files for configuration and output formats.
- Established a SQLite database for state management of fetched documents.
- Updated dependencies in `requirements.txt` and `pyproject.toml` for necessary libraries.

Author: makr-code

.continue/agents/new-config.yaml

@@ -3,27 +3,31 @@ version: 1.0.0
 schema: v1
 
 models:
-  - name: "Remote Agent (codestral, no-tools)"
+  - name: "Remote Agent (gemma4)"
     provider: ollama
     apiBase: "http://192.168.178.106:11434"
-    model: "codestral:latest"
+    model: "gemma4:latest"
+    capabilities:
+      - tool_use
+      - image_input
     roles:
       - chat
     chatOptions:
       baseAgentSystemMessage: |
         You are a senior coding agent for ThemisDB.
         Prioritize correctness, small safe patches, and verification steps.
-        This model does not support tool calling on this Ollama instance.
-        If a task requires external tools, clearly state the limitation and propose the safest fallback.
+        Use tools when available and keep responses grounded in retrieved context.
     completionOptions:
       temperature: 0.2
       topP: 0.9
       maxTokens: 4096
 
-  - name: "Remote Coding (codestral)"
+  - name: "Remote Coding (qwen2.5-coder-14b)"
     provider: ollama
     apiBase: "http://192.168.178.106:11434"
-    model: "codestral:latest"
+    model: "qwen2.5-coder:14b"
+    capabilities:
+      - tool_use
     roles:
       - edit
       - apply
@@ -33,6 +37,19 @@ models:
       topP: 0.9
       maxTokens: 4096
 
+  - name: "Remote Deep Coding (deepseek-coder-v2-16b)"
+    provider: ollama
+    apiBase: "http://192.168.178.106:11434"
+    model: "deepseek-coder-v2:16b"
+    roles:
+      - edit
+      - apply
+      - summarize
+    completionOptions:
+      temperature: 0.1
+      topP: 0.9
+      maxTokens: 6144
+
   - name: "Remote Autocomplete (codestral)"
     provider: ollama
     apiBase: "http://192.168.178.106:11434"
@@ -46,11 +63,11 @@ models:
       onlyMyCode: true
 
   # Prepared profiles (activate after model download + quick validation)
-  # 1) Agent candidate (enable only if /api/chat tools test succeeds)
-  # - name: "Remote Agent (gemma4)"
+  # 1) Additional agent candidate
+  # - name: "Remote Agent (qwen2.5-coder-14b)"
   #   provider: ollama
   #   apiBase: "http://192.168.178.106:11434"
-  #   model: "gemma4:latest"
+  #   model: "qwen2.5-coder:14b"
   #   capabilities:
   #     - tool_use
   #   roles:
@@ -64,11 +81,11 @@ models:
   #     topP: 0.9
   #     maxTokens: 4096
 
-  # 2) Strong coding profile
-  # - name: "Remote Coding (qwen2.5-coder-14b)"
+  # 2) Fallback coding profile
+  # - name: "Remote Coding (codestral)"
   #   provider: ollama
   #   apiBase: "http://192.168.178.106:11434"
-  #   model: "qwen2.5-coder:14b"
+  #   model: "codestral:latest"
   #   roles:
   #     - edit
   #     - apply
@@ -78,18 +95,14 @@ models:
   #     topP: 0.9
   #     maxTokens: 4096
 
-  # 3) Deep reasoning coding profile
-  # - name: "Remote Deep Coding (deepseek-coder-v2-16b)"
+  # 3) Deep coding fallback profile
+  # - name: "Remote Deep Coding Fallback (deepseek-coder-v2-16b)"
   #   provider: ollama
   #   apiBase: "http://192.168.178.106:11434"
   #   model: "deepseek-coder-v2:16b"
   #   roles:
   #     - edit
   #     - apply
-  #   completionOptions:
-  #     temperature: 0.1
-  #     topP: 0.9
-  #     maxTokens: 6144
 
   # 4) Fast fallback chat/autocomplete profile
   # - name: "Remote Fast (llama3.1-8b)"

ai_working/THEMISDB_COPILOT_GRAPH_RAG_ARCH_ANALYSIS_2026-06-10.md

@@ -0,0 +1,25 @@
+# ThemisDB as Copilot Backend: Graph-RAG + MCP + Re-Ranking
+
+## Scope
+- Create a practical architecture analysis for integrating ThemisDB as coding-assistant backend in VS Code/Copilot.
+- Include current state (IST), target state (SOLL), gap analysis, and implementation deep-dive.
+- Assume ThemisDB can host local inference models (llama.cpp family, CodeLlama, DeepSeek, Gemma).
+
+## Inputs Used
+- ARCHITECTURE.md
+- docs/architecture/FEATURE_FLAGS_REFERENCE.md
+- include/llama_cpp/*
+- include/search/llm_reranker.h + README
+- include/rag/reranker.h + src/rag/reranker.cpp
+- tools/copilot-ollama-router/*
+- audit/docs/implementation-history/LLM_LORA_QLORA_INTEGRATION_AUDIT.md
+
+## Deliverable Outline
+1. Executive summary
+2. IST architecture map
+3. SOLL architecture map
+4. SOLL-IST comparison table
+5. Deep-dive by capability
+6. Security and governance model
+7. Rollout phases and acceptance criteria
+8. Risks and mitigations

deployment/docker-compose.yml

@@ -0,0 +1,53 @@
+version: '3.8'
+
+services:
+  # 1. Database Service
+  db:
+    image: postgres:14-alpine
+    container_name: themisdb_postgres
+    environment:
+      POSTGRES_USER: user
+      POSTGRES_PASSWORD: password
+      POSTGRES_DB: themisdb
+    volumes:
+      # Persist data volume for the database
+      - db_data:/var/lib/postgresql/data
+    expose:
+      - "5432"
+
+  # 2. Authentication Service (Handles JWT generation and user auth)
+  auth_api:
+    build: ./services/auth_api # Path relative to the root of the project, or within a subfolder Dockerfile should expect
+    container_name: themisdb_auth_service
+    environment:
+      DATABASE_URL: postgresql://user:password@db:5432/themisdb
+      JWT_SECRET_KEY: ${JWT_SECRET_KEY} # Loaded from .env and passed through docker-compose
+      ALGORITHM: HS256
+    depends_on:
+      - db
+    ports:
+      # Expose port for external testing, if needed. Usually best left internal.
+      - "8001:8001" 
+    volumes:
+      # Optional: For hot reloading during development
+      - ./services/auth_api:/app
+
+  # 3. Main API Gateway / Gateway Service (Handles request routing and protected routes)
+  main_gateway:
+    build: . # Assuming main project structure is the root directory
+    container_name: themisdb_main_gateway
+    environment:
+      DATABASE_URL: postgresql://user:password@db:5432/themisdb
+      JWT_SECRET_KEY: ${JWT_SECRET_KEY}
+      AUTH_SERVICE_URL: http://auth_api:8001 # Use service name as hostname
+    depends_on:
+      - auth_api
+      - db
+    ports:
+      # Expose the main facing port
+      - "80:80"
+    volumes:
+      - .:/app # Assuming this maps the entire codebase for development
+
+volumes:
+  db_data: