fix(mcp): resolve MCP service breakage from missing deps, asyncio errors, and lazy imports

- Move cachetools to core dependencies (was hidden in vectordb extra, needed
  by BaseRepository which every service imports)
- Make HuggingFaceEmbeddings import lazy in rag/embeddings.py so RAG service
  works with OpenAI API embeddings without requiring sentence-transformers/PyTorch
- Add run_async_safely() utility to prevent "asyncio.run() cannot be called
  from a running event loop" errors in rag_service and discovery/scheduler
- Fix _PromptsChangeHandler to inherit from FileSystemEventHandler (provides
  required dispatch() method for watchdog)
- Fix CLI lazy loading to check sys.argv for mcp/db/setup commands before
  argparse, and make MCP handler pipeline arg optional
- Fix docker-compose.dev.yml: use docker/mcp/Dockerfile with explicit command
  override and PYTHONPATH for source mounts
- Remove unnecessary vectordb extra from all service groups (cachetools is
  now a core dep)
- Remove redundant pydocstyle pre-commit hook (ruff already handles docstring
  linting and the standalone hook conflicted with the codebase's style)
- Fix pre-existing ruff B028 and W505 lint issues in touched files
- Bump version to 0.3.0-alpha.3
- Add comprehensive regression tests for MCP config, async utils, and imports

Author: acertainKnight

.pre-commit-config.yaml

@@ -33,14 +33,7 @@ repos:
         args: ['-c', 'pyproject.toml']
         additional_dependencies: ['bandit[toml]']
 
-  # Docstring checks
-  - repo: https://github.com/pycqa/pydocstyle
-    rev: 6.3.0
-    hooks:
-      - id: pydocstyle
-        args: [--convention=google]
-        exclude: ^tests/
-
+  # Docstring checks handled by ruff (pydocstyle rules via [tool.ruff.lint.pydocstyle])
   # Import sorting handled by ruff (select includes "I" rules)
 
   # Run unit tests before push (catches failures before CI)

docker-compose.dev.yml

@@ -255,11 +255,12 @@ services:
     profiles: ["microservices"]
     build:
       context: .
-      dockerfile: Dockerfile
-      # target: development  # Not using multi-stage build
+      dockerfile: docker/mcp/Dockerfile
+      target: development
       args:
-        SERVICE_EXTRAS: "service-mcp"  # Includes api,pdf,mcp,discovery,langchain,vectordb,embeddings,memory
+        SERVICE_EXTRAS: "service-mcp"
     container_name: thoth-dev-mcp
+    command: ["python", "-m", "thoth", "mcp", "http", "--host", "0.0.0.0", "--port", "8001", "--log-level", "DEBUG"]
     user: "1000:1000"  # Run as host user for proper file permissions
     env_file:
       - .env
@@ -284,6 +285,9 @@ services:
       # Development settings (REQUIRED - dev-specific configuration)
       - THOTH_LOG_LEVEL=DEBUG
 
+      # Prioritize mounted source over installed package for hot-reload
+      - PYTHONPATH=/app/src:${PYTHONPATH:-}
+
       # Hot-reload configuration
       # Automatically reload when vault/thoth/_thoth/settings.json changes
       # No container restart needed!

pyproject.toml

@@ -7,7 +7,7 @@ name = "thoth"
 description = "Thoth AI Research Agent - An autonomous system for academic research paper processing and knowledge management"
 readme = "README.md"
 requires-python = ">=3.12, <3.13"
-version = '0.3.0-alpha.2'
+version = '0.3.0-alpha.3'
 authors = [{name="Nick Hallmark", email="NHALLMARK3@BLOOMBERG.NET"}]
 
 dependencies = [
@@ -23,6 +23,7 @@ dependencies = [
     "fuzzywuzzy>=0.18.0",
     "python-levenshtein>=0.27.3",
     "matplotlib>=3.10.3",
+    "cachetools>=5.3.0",  # Required by BaseRepository (used by all services)
 ]
 
 [project.optional-dependencies]
@@ -101,9 +102,8 @@ langchain = [
 ]
 
 # Vector database utilities
-vectordb = [
-    "cachetools>=5.3.0",  # Required by BaseRepository
-]
+# NOTE: cachetools was moved to core dependencies since BaseRepository needs it
+vectordb = []
 
 # Embeddings (VERY HEAVY: ~2GB with PyTorch)
 embeddings = [
@@ -125,35 +125,33 @@ memory = [
 # ==============================================================================
 
 # thoth-api: REST API server (~1.5GB)
-# Core API server - does NOT need vectordb/embeddings by default
-# Add them if you want RAG features with local embeddings
+# Core API server - cachetools is now a core dep so vectordb not needed here
 service-api = [
     "thoth[api,discovery,pdf,langchain]",
 ]
 
 # thoth-dashboard: Dashboard sync (~1.5GB)
-# Needs vectordb for cachetools (BaseRepository), but NOT embeddings
 service-dashboard = [
-    "thoth[api,discovery,pdf,langchain,vectordb]",
+    "thoth[api,discovery,pdf,langchain]",
 ]
 
 # thoth-discovery: Discovery scheduler (~1.5GB)
-# Needs langchain for AnthropicClient + vectordb for cachetools + pdf for pypdf (pipelines)
+# Needs langchain for AnthropicClient + pdf for pypdf (pipelines)
 service-discovery = [
-    "thoth[api,discovery,pdf,langchain,vectordb]",
+    "thoth[api,discovery,pdf,langchain]",
 ]
 
 # thoth-mcp: MCP server with 60+ research tools (~2GB)
+# thoth-mcp: MCP protocol interface (~1.5GB)
 # Provides MCP protocol interface to Thoth research tools
-# Includes discovery and document processing capabilities
 # embeddings and memory (letta) are OPTIONAL - users can add if needed
 service-mcp = [
     "thoth[api,pdf,mcp,discovery,langchain]",
 ]
 
 # thoth-monitor: PDF processing (~2GB)
 # Watches for PDFs and processes them
-# embeddings and vectordb are OPTIONAL - add if you want local embeddings
+# embeddings are OPTIONAL - add if you want local embeddings
 service-monitor = [
     "thoth[api,pdf,discovery,mcp,langchain]",
 ]

src/thoth/cli/main.py

@@ -35,28 +35,35 @@ def _configure_safe_environment() -> None:
 def main() -> None:
     """Main entry point for the Thoth CLI."""
     logger.info('===== main(): Function called, parsing arguments =====')
-    parser = argparse.ArgumentParser(
-        description='Thoth - Academic PDF processing system'
-    )
-    subparsers = parser.add_subparsers(
-        dest='command', help='Command to run', required=True
-    )
 
-    # Always register setup command
-    setup_cli.configure_subparser(subparsers)
+    # Quick-check the command from sys.argv before argparse to enable lazy
+    # loading.  This avoids importing heavy dependencies (ThothPipeline,
+    # initialize_thoth, etc.) for lightweight commands like setup, db, and mcp.
+    import sys
 
-    # Parse args early to check if we need other modules
-    args = parser.parse_args()
+    raw_command = sys.argv[1] if len(sys.argv) > 1 else None
+
+    if raw_command in ('setup', 'db', 'mcp'):
+        parser = argparse.ArgumentParser(
+            description='Thoth - Academic PDF processing system'
+        )
+        subparsers = parser.add_subparsers(
+            dest='command', help='Command to run', required=True
+        )
+
+        # Only register the subparser we actually need
+        setup_cli.configure_subparser(subparsers)
 
-    # Skip heavy imports for setup and db commands
-    if args.command in ['setup', 'db']:
-        # For db command, only import database module
-        if args.command == 'db':
+        if raw_command == 'db':
             from . import database
 
             database.configure_subparser(subparsers)
-            # Re-parse after adding db subparser
-            args = parser.parse_args()
+        elif raw_command == 'mcp':
+            from . import mcp
+
+            mcp.configure_subparser(subparsers)
+
+        args = parser.parse_args()
 
         if hasattr(args, 'func'):
             if inspect.iscoroutinefunction(args.func):
@@ -65,7 +72,15 @@ def main() -> None:
                 args.func(args)
         return
 
-    # Import all other CLI modules (only when not running setup/db)
+    # Full CLI mode — register all subparsers and initialize pipeline
+    parser = argparse.ArgumentParser(
+        description='Thoth - Academic PDF processing system'
+    )
+    subparsers = parser.add_subparsers(
+        dest='command', help='Command to run', required=True
+    )
+
+    # Import all CLI modules (full pipeline mode)
     from . import (
         database,
         discovery,

src/thoth/cli/mcp.py

@@ -5,11 +5,15 @@
 with different transport configurations.
 """
 
+from __future__ import annotations
+
 import sys
+from typing import TYPE_CHECKING
 
 from loguru import logger
 
-from thoth.pipeline import ThothPipeline
+if TYPE_CHECKING:
+    from thoth.pipeline import ThothPipeline
 
 
 def configure_subparser(subparsers):
@@ -103,7 +107,7 @@ def configure_subparser(subparsers):
     legacy_parser.set_defaults(func=run_http_server)
 
 
-def run_stdio_server(args, _pipeline: ThothPipeline):
+def run_stdio_server(args, _pipeline: ThothPipeline | None = None):
     """Run MCP server with stdio transport only."""
     from thoth.mcp.launcher import run_stdio_server
 
@@ -115,7 +119,7 @@ def run_stdio_server(args, _pipeline: ThothPipeline):
     run_stdio_server()
 
 
-def run_http_server(args, _pipeline: ThothPipeline):
+def run_http_server(args, _pipeline: ThothPipeline | None = None):
     """Run MCP server with HTTP transport only."""
     import sys
 
@@ -129,7 +133,7 @@ def run_http_server(args, _pipeline: ThothPipeline):
     run_http_server(host=args.host, port=args.port)
 
 
-def run_full_server(args, _pipeline: ThothPipeline):
+def run_full_server(args, _pipeline: ThothPipeline | None = None):
     """Run MCP server with all transports."""
     import sys
 

src/thoth/discovery/scheduler.py

@@ -434,8 +434,7 @@ def _load_schedule_state(self) -> dict[str, Any]:
 
     def _load_from_postgres(self) -> dict[str, Any]:
         """Load discovery schedule from PostgreSQL."""
-        import asyncpg  # noqa: I001
-        import asyncio
+        import asyncpg
 
         db_url = (
             getattr(self.config.secrets, 'database_url', None)
@@ -467,16 +466,10 @@ async def load():
             finally:
                 await conn.close()
 
-        # Use asyncio.run() to create a new event loop - safe from any thread
-        try:
-            return asyncio.run(load())
-        except RuntimeError:
-            # If there's already an event loop running, create a new one in a thread
-            import concurrent.futures
+        # Use run_async_safely to handle both sync and async calling contexts.
+        from thoth.utilities.async_utils import run_async_safely
 
-            with concurrent.futures.ThreadPoolExecutor() as executor:
-                future = executor.submit(lambda: asyncio.run(load()))
-                return future.result()
+        return run_async_safely(load())
 
     def _save_schedule_state(self) -> None:
         """
@@ -489,8 +482,7 @@ def _save_schedule_state(self) -> None:
 
     def _save_to_postgres(self) -> None:
         """Save discovery schedule to PostgreSQL."""
-        import asyncpg  # noqa: I001
-        import asyncio
+        import asyncpg
 
         db_url = (
             getattr(self.config.secrets, 'database_url', None)
@@ -535,16 +527,10 @@ async def save():
             finally:
                 await conn.close()
 
-        # Use asyncio.run() to create a new event loop - safe from any thread
-        try:
-            asyncio.run(save())
-        except RuntimeError:
-            # If there's already an event loop running, create a new one in a thread
-            import concurrent.futures
-
-            with concurrent.futures.ThreadPoolExecutor() as executor:
-                future = executor.submit(lambda: asyncio.run(save()))
-                future.result()
+        # Use run_async_safely to handle both sync and async calling contexts.
+        from thoth.utilities.async_utils import run_async_safely
+
+        run_async_safely(save())
 
     def sync_with_discovery_manager(self) -> None:
         """

src/thoth/rag/embeddings.py

@@ -8,12 +8,16 @@
 import os
 from typing import Any
 
-from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_openai import OpenAIEmbeddings
 from loguru import logger
 
 from thoth.config import config
 
+# HuggingFaceEmbeddings is imported lazily to avoid requiring the heavy
+# sentence-transformers / PyTorch dependencies when using API-based embeddings
+# (OpenAI).  Install with: uv sync --extra embeddings
+HuggingFaceEmbeddings = None  # type: ignore[assignment,misc]
+
 
 class EmbeddingManager:
     """
@@ -133,6 +137,23 @@ def _init_openai_embeddings(self) -> None:
 
     def _init_local_embeddings(self) -> None:
         """Initialize local sentence-transformers embeddings."""
+        global HuggingFaceEmbeddings
+        if HuggingFaceEmbeddings is None:
+            try:
+                from langchain_huggingface import (
+                    HuggingFaceEmbeddings as _HFE,
+                )
+
+                HuggingFaceEmbeddings = _HFE
+            except ImportError:
+                raise ImportError(
+                    'Local embeddings require langchain-huggingface and '
+                    'sentence-transformers. Install with: uv sync --extra embeddings\n'
+                    'Alternatively, configure an OpenAI embedding model '
+                    '(e.g., "openai/text-embedding-3-small") in settings to '
+                    'use API-based embeddings without heavy dependencies.'
+                ) from None
+
         try:
             # Create local embeddings using HuggingFace/sentence-transformers
             # with safer configuration to prevent segfaults