pyproject.toml

[build-system]
requires = ["setuptools >= 60.0.0", "wheel", "setuptools-scm>=8.0"]
build-backend = "setuptools.build_meta"

[project]
name = "thoth"
description = "Thoth AI Research Agent - An autonomous system for academic research paper processing and knowledge management"
readme = "README.md"
requires-python = ">=3.12, <3.13"
version = '0.5.3-alpha'
authors = [{name="Nick Hallmark", email="NHALLMARK3@BLOOMBERG.NET"}]

dependencies = [
    # Absolute bare minimum - all services need these
    "click>=8.1.8", # For CLI
    "loguru>=0.7.3", # For logging
    "python-dotenv>=1.0.1", # For environment variables
    "pydantic>=2.9.2", # For data validation
    "pydantic-settings>=2.5.0", # For settings management
    "deprecated>=1.2.18", # For deprecation warnings
    "cryptography>=45.0.2",
    "seaborn>=0.13.2",
    "fuzzywuzzy>=0.18.0",
    "python-levenshtein>=0.27.3",
    "matplotlib>=3.10.3",
    "cachetools>=5.3.0",  # Required by BaseRepository (used by all services)
]

[project.optional-dependencies]
# ==============================================================================
# Base Component Groups
# ==============================================================================

# API Server components
api = [
    "fastapi>=0.115.12",
    "uvicorn>=0.32.0",
    "sqlalchemy>=2.0.39",
    "asyncpg", # PostgreSQL async driver
    "psycopg2-binary>=2.9.10", # PostgreSQL synchronous driver for pdf_monitor
    "aiofiles>=24.1.0",
    "httpx>=0.27.0",
    "requests>=2.32.3",
    "arrow>=1.3.0",
    "json-repair>=0.44.1",
    "jinja2>=3.1.6",
    "cryptography>=44.0.0", # For workflow credentials encryption
    "python-multipart>=0.0.9", # For file upload support in FastAPI
]

# Discovery module (web scraping, academic APIs)
discovery = [
    "beautifulsoup4>=4.13.2",
    "lxml>=5.3.0",
    "feedparser>=6.0.11", # For arXiv RSS feeds
    "networkx>=3.4.2",
    "scholarly>=1.7.11",
    "bibtexparser>=1.4.2",
    "selenium>=4.29.0",
    "playwright>=1.48.0", # Browser automation for workflows
    "fake-useragent>=2.1.0",
    "duckduckgo-search>=5.2",
    "roman-numerals-py>=3.1.0",
    "aiohttp>=3.8.5",
    # New plugins for top-tier AI/ML venues
    "openreview-py>=1.43.0", # For ICLR and OpenReview conferences
    "acl-anthology>=0.2.12", # For ACL, EMNLP, NAACL, and NLP venues
]

# PDF processing (file watching, extraction, OCR)
pdf = [
    "watchdog>=6.0.0",
    "pypdf>=5.3.1",
    "mistralai>=1.5.1",
]

# MCP Server components
mcp = [
    "fastapi>=0.115.12",
    "uvicorn>=0.32.0",
    # langchain-mcp-adapters removed - we register MCP tools directly with Letta
    # langchain-core removed - was only needed for adapters
]

# LangChain ecosystem (document processing, LLM orchestration)
langchain = [
    "langchain>=0.3.21",
    "langchain-core>=0.3.45",
    "langchain-text-splitters>=0.3.7",
    "langchain-openai>=0.3.9",
    "langchain-anthropic>=0.1.8",
    "langgraph>=0.4.7",
    "langgraph-checkpoint>=2.0.26",
    "langgraph-sdk>=0.1.70",
    "langgraph-prebuilt>=0.2.1",
    "langsmith>=0.3.16",
    "openrouter>=0.0.19",
    "openai>=1.57.0",
    "instructor>=1.8.3",
    "tiktoken>=0.9.0",
    "websockets>=15.0.1",
    "grandalf>=0.8",
]

# Vector database utilities
# NOTE: cachetools was moved to core dependencies since BaseRepository needs it
vectordb = []

# Embeddings (VERY HEAVY: ~2GB with PyTorch)
embeddings = [
    "langchain-huggingface>=0.1.0",
    "sentence-transformers>=3.0.0",
]

# Reranking (optional - for production-grade reranking via Cohere API)
reranking = [
    "cohere>=5.0",
]

# Query routing (optional - for adaptive RAG with semantic query classification)
routing = [
    "semantic-router>=0.1.0",
]

# External knowledge processing (convert various formats to markdown)
knowledge = [
    "mammoth>=1.0.0",      # DOCX to HTML conversion
    "ebooklib>=0.18",      # EPUB parsing and extraction
    "markdownify>=0.11.6", # HTML to Markdown conversion
]

# Letta client SDK for communicating with the Letta server container
# Only the lightweight client - the full letta server runs in its own container
memory = [
    "letta-client>=1.0.0",
]

# ==============================================================================
# Service-Specific Composite Groups (for Docker containers)
# ==============================================================================

# thoth-api: REST API server (~1.5GB)
# Includes memory (letta-client) for filesystem sync with Letta agents
service-api = [
    "thoth[api,discovery,pdf,langchain,memory]",
]

# thoth-dashboard: Dashboard sync (~1.5GB)
service-dashboard = [
    "thoth[api,discovery,pdf,langchain]",
]

# thoth-discovery: Discovery scheduler (~1.5GB)
# Needs langchain for AnthropicClient + pdf for pypdf (pipelines)
service-discovery = [
    "thoth[api,discovery,pdf,langchain]",
]

# thoth-mcp: MCP server with 60+ research tools (~2GB)
# thoth-mcp: MCP protocol interface (~1.5GB)
# Provides MCP protocol interface to Thoth research tools
service-mcp = [
    "thoth[api,pdf,mcp,discovery,langchain,memory,knowledge]",
]

# thoth-monitor: PDF processing (~2GB)
# Watches for PDFs and processes them
# embeddings are OPTIONAL - add if you want local embeddings
service-monitor = [
    "thoth[api,pdf,discovery,mcp,langchain,knowledge]",
]

service-all = [
    "thoth[api,pdf,mcp,discovery,langchain,memory,knowledge]",
]

# Optional heavy dependencies (only install if you need local models)
service-heavy = [
    "thoth[api,pdf,mcp,discovery,langchain,vectordb,embeddings,memory]",
]

# ==============================================================================
# Development and Testing
# ==============================================================================

dev = [
    "black>=25.1.0",
    "build>=1.2.2.post1",
    "codespell>=2.4.1",
    "mypy>=1.15.0",
    "pre-commit>=4.2.0",
    "pytest>=8.3.5",
    "pytest-asyncio>=0.26.0",
    "pytest-cov>=6.1.1",
    "pytest-mock>=3.14.0",
    "pytest-timeout>=2.1.0",
    "ruff>=0.11.11",
    "twine>=6.1.0",
]

test = [
    "pytest>=8.3.5",
    "pytest-asyncio>=0.26.0",
    "pytest-cov>=6.1.1",
    "pytest-mock>=3.14.0",
    "pytest-timeout>=2.1.0",
    "pytest-benchmark>=5.1.0", # Performance benchmarking
    "hypothesis>=6.122.0", # Property-based testing
    "psutil>=7.0.0", # Memory profiling
    "respx>=0.22.0",
    "freezegun>=1.5.5",
]

setup = [
    "textual>=0.80.0",  # TUI framework for setup wizard
    "rich>=13.0.0",     # Rich text and formatting
    "psutil>=7.0.0",    # System information and process management
    "httpx>=0.27.0",    # HTTP client for checking service availability
    "requests>=2.32.3", # Alternative HTTP client (used by some detectors)
    "pydantic>=2.0.0",  # Settings validation (config.py)
    "pydantic-settings>=2.0.0",  # Settings.from_json_file
    # Minimal langchain deps for model fetching (ModelRegistry in openrouter.py)
    "langchain-core>=0.3.45",  # Required by OpenRouterClient base class
    "langchain-openai>=0.3.9", # OpenRouterClient extends ChatOpenAI
]

jupyter = [
    "ipykernel>=6.29.5",
    "jupyter>=1.1.1",
    "notebook>=7.4.2",
]

viz = [
    "matplotlib>=3.9.2",
    "plotly>=5.20.0",
    "grandalf>=0.8",
]

utils = [
    "psutil>=7.0.0",
    "jsonpatch>=1.33",
]

# Full local development
all = [
    "thoth[service-api,service-dashboard,service-discovery,service-mcp,service-monitor,dev,test,viz,utils]",
]

[tool.setuptools.packages.find]
where = ["src"]
# Note: scripts/ directory is intentionally excluded (not in src/)
# Migration scripts and one-off utilities should not be part of installed package

### STANDARDIZED CODE REQUIREMENTS ###################################################
### pytest ###########################################################################
[tool.pytest.ini_options]
testpaths = ["tests"]
asyncio_default_fixture_loop_scope = "function"

### ruff #############################################################################
[tool.ruff]
line-length    = 88
target-version = "py312"

[tool.ruff.lint]
select = [
    "F", "E", "W", "I", "UP", "B", "ARG", "RUF",
    "N801", "N802", "N803", "N804", "N805", "N806",
    "PD002", "PD003", "PD004", "PD007", "PD008",
    "PD009", "PD010", "PD011", "PD012", "PD013", "PD015",
]
ignore = ["E501"]
extend-unsafe-fixes = ["UP034"]

[tool.ruff.lint.per-file-ignores]
# FastAPI Depends() in argument defaults is intentional
"src/thoth/server/routers/auth.py" = ["B008"]
"src/thoth/server/routers/browser_workflows.py" = ["B008"]
"src/thoth/server/routers/mcp_servers.py" = ["B008"]
"src/thoth/server/routers/operations.py" = ["B008"]
"src/thoth/server/routers/research.py" = ["B008"]
"src/thoth/server/routers/tools.py" = ["B008"]
"src/thoth/server/routers/websocket.py" = ["B008"]
# BaseRepository uses Generic[T] for backward compatibility
"src/thoth/repositories/base.py" = ["UP046"]

[tool.ruff.lint.pydocstyle]
convention = "numpy"

[tool.ruff.lint.pycodestyle]
max-doc-length = 88

[tool.ruff.lint.flake8-quotes]
inline-quotes = "single"

[tool.ruff.format]
quote-style               = "single"
indent-style              = "space"
skip-magic-trailing-comma = false
line-ending               = "auto"
docstring-code-format = true

### Black (for auto-wrapping) ########################################################
[tool.black]
line-length    = 88
target-version = ["py312"]

### sqlfluff ########################################################################
[tool.sqlfluff.core]
dialect    = "postgres"
templater  = "jinja"
exclude_rules = "L015,L026,L027,L031,L034,L042,L054"
ignore        = "templating"

[tool.sqlfluff.indentation]
tab_space_size = 2

[tool.sqlfluff.rules.capitalisation.keywords]
capitalisation_policy = "upper"

[tool.sqlfluff.rules.capitalisation.identifiers]
extended_capitalisation_policy = "lower"
unquoted_identifiers_policy    = "all"

[tool.sqlfluff.rules.layout.long_lines]
ignore_comment_lines = true

[tool.sqlfluff.rules.capitalisation.functions]
extended_capitalisation_policy = "upper"

### codespell ########################################################################
[tool.codespell]
exclude-file = ".codespellignore"

[dependency-groups]
dev = [
    "nbstripout>=0.8.1",
]

### Bandit Security Configuration ################################################
[tool.bandit]
exclude_dirs = ["tests", ".venv", "venv"]
skips = [
    "B101", "B601",  # assert_used, paramiko
    "B404",  # subprocess import (CLI runs docker/compose)
    "B603", "B607",  # subprocess without shell / partial path (docker CLI)
    "B110",  # try/except pass (intentional in TUI/setup paths)
    "B608",  # graph.py citation INSERT uses fixed subquery templates, params bound
]

### MyPy Configuration ###########################################################
[tool.mypy]
python_version = "3.10"
warn_return_any = true
warn_unused_configs = true
ignore_missing_imports = true
disallow_untyped_defs = false
check_untyped_defs = true

[[tool.mypy.overrides]]
module = "tests.*"
disallow_untyped_defs = false

### Coverage Configuration #######################################################
[tool.coverage.run]
source = ["src"]
omit = [
    "*/tests/*",
    "*/__pycache__/*",
    "*/site-packages/*",
]

[tool.coverage.report]
precision = 2
show_missing = true
skip_covered = false
exclude_lines = [
    "pragma: no cover",
    "def __repr__",
    "raise AssertionError",
    "raise NotImplementedError",
    "if __name__ == .__main__.:",
    "if TYPE_CHECKING:",
    "@abstractmethod",
]