Crawllama/pyproject.toml at 1.4.11 · arn-c0de/Crawllama · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
# Project + tooling configuration for CrawlLama (managed with uv).
#
# Dependency management is handled by uv via this file and uv.lock.
# - Core runtime deps live in [project.dependencies].
# - Optional features (LLM providers, API server, OSINT, LinkedIn, testing)
#   live in [project.optional-dependencies] and are installed with
#   `uv sync --extra <name>` (e.g. `uv sync --extra api --extra openai`).
# - Transitive security pins live in [tool.uv] override-dependencies.

[project]
name = "crawllama"
# Keep in sync with _version.py (the runtime single source of truth).
version = "1.4.11"
description = "Local AI Search and Answer Agent with RAG, OSINT and multi-provider LLM support."
readme = "README.md"
requires-python = ">=3.10"
license = { file = "LICENSE" }

# ===== CORE (Always installed) =====
dependencies = [
    "requests>=2.33.1",
    "beautifulsoup4>=4.14.3",
    "html5lib>=1.1",
    "urllib3>=2.6.3",          # fix decompression bomb CVE-2025-69223 (Dependabot alert #32)
    "ddgs>=9.14.1",
    # chromadb is used only as an embedded PersistentClient (tools/rag.py) — the
    # local, in-process vector store. CVE-2026-45829 "ChromaToast" (Dependabot
    # alert #71, Critical) is a pre-auth RCE in the Chroma *FastAPI server*
    # (/api/v2/.../collections with trust_remote_code=true), which CrawlLama
    # never runs/exposes, so it is not exploitable here. No patched release
    # exists yet (latest is 1.5.9); bump once Chroma ships a fix.
    "chromadb>=1.5.8",
    "pyasn1>=0.6.4",
    "tenacity>=9.1.4",
    "rich>=15.0.0",
    "python-dotenv>=1.2.2",
    "cryptography>=50.0.0",    # fixes through PKCS#7 Bleichenbacher oracle CVE-2026-69247 (#95)
    "redis>=7.4.0",
    "fakeredis>=2.35.1",
    "wikipedia>=1.4.0",
    "tiktoken>=0.12.0",
    "aiohttp>=3.14.3",         # security fixes through 3.14.3, including malformed-response C-parser OOB read, WebSocket compression negotiation, and rejected-upgrade request smuggling (#92-#94); overrides langchain-community transitive dep
    "pysocks>=1.7.1",          # SOCKS5 support for requests (Tor mode, socks5h remote DNS)
    "aiohttp-socks>=0.10.1",   # SOCKS5 connector for aiohttp (Tor mode, rdns=True)
    "pydantic>=2.13.3",
    "psutil>=7.2.2",
    "pyperclip>=1.11.0",       # clipboard support for health dashboard log viewer
    "brotli>=1.2.0",           # ensure safe decompression (fixes DoS CVE)
    "filelock>=3.29.0",        # fix TOCTOU symlink vulns CVE-2025-68146 (#33) / CVE-2026-22701
    "langchain>=1.3.9",        # core agent infrastructure (StructuredTool, LangGraph); fix path traversal/sandbox escape in file-search middleware & loaders (#77)
    "langchain-core>=1.3.0",   # fix SSRF CVE-2026-26013 (#41), f-string validation CVE-2026-40087 (#63)
    "langchain-community>=0.4.1",
    "langgraph>=1.1.9",
]

[project.optional-dependencies]
# ===== API (FastAPI server features) =====
api = [
    "fastapi>=0.136.0",
    "starlette>=1.3.1",        # Range header DoS (CVE-2025-62727) + request.form() limit DoS (#76) + request.url.hostname authority poisoning (#75)
    "uvicorn>=0.45.0",
]

# ===== OSINT (OSINT & Social Media Intelligence) =====
osint = [
    "phonenumbers>=9.0.28",
    "dnspython>=2.8.0",
    "tweepy>=4.16.0",
    "python-instagram>=1.3.2",
]

# ===== LINKEDIN_API (Optional LinkedIn API - requires LinkedIn account) =====
# NOTE: optional and conflict-prone. linkedin-api==2.3.1 declares lxml<6.0.0,
# but lxml<6.1.0 is vulnerable to XXE (CVE-2026-41066), so we force lxml>=6.1.0
# via [tool.uv] override-dependencies below.
# See docs/osint/SOCIAL_INTELLIGENCE.md for setup and ToS implications.
linkedin = [
    "linkedin-api==2.3.1",
    "lxml>=6.1.0",            # fix XXE CVE-2026-41066; overrides linkedin-api's <6.0.0 cap
]

# ===== LLM providers =====
# Ollama (local) needs no extra packages — langchain/langgraph are in CORE.
ollama = []
# httpx[socks] lets the SDKs' HTTP layer speak SOCKS5 when Tor mode is on.
openai = ["openai>=2.32.0", "httpx[socks]>=0.27.0"]
anthropic = ["anthropic>=0.96.0", "httpx[socks]>=0.27.0"]
groq = ["groq>=1.2.0", "httpx[socks]>=0.27.0"]

# ===== TESTING (Development & Testing) =====
testing = [
    "pytest>=9.0.3",           # fix vulnerable tmpdir handling CVE-2025-71176 (#64)
    "pytest-mock>=3.15.1",
    "pytest-cov>=7.1.0",
    "pytest-asyncio>=1.3.0",
    "pytest-timeout>=2.4.0",   # pytest.ini sets timeout=60; without the plugin the option is ignored
]

# Convenience aggregate of the common runtime features (linkedin excluded — it
# is conflict-prone and pulls an older lxml). Install with `uv sync --extra all`.
all = [
    "crawllama[api,osint,ollama,openai,anthropic,groq]",
]

[tool.uv]
# Application, not a distributable library: uv manages the venv and runs the
# entry points (main.py / app.py) without building/installing the project.
package = false

# --- Security overrides (transitive dependencies) ---
# These force-pin packages that arrive transitively (via chromadb, langchain*,
# langgraph, sentence-transformers, rich, …) to versions that carry security
# fixes, regardless of what the intermediate dependencies request.
# NOTE: marshmallow 4.3.0 conflicts with dataclasses-json (<4.0.0) at the
# resolver level, but all imports work; the security fix takes priority.
override-dependencies = [
    "marshmallow==4.3.0",              # fix DoS CVE-2025-68480 (#22); GHSA-428g-f7cq-pgp5
    "protobuf>=7.34.1",                # fix JSON recursion depth bypass CVE-2026-0994 (#37); via chromadb
    "langsmith>=0.8.18",               # fix SSRF CVE-2026-25528 (#38) + GHSA-rr7j-v2q5-chgv (#66) + TracingMiddleware arbitrary file read GHSA-f4xh-w4cj-qxq8 (#78); via langchain/langgraph
    "pillow>=12.2.0",                  # fix PSD OOB write CVE-2026-25990 (#40) + FITS bomb CVE-2026-40192 (#65); via sentence-transformers
    "langgraph-checkpoint>=4.0.2",     # fix BaseCache deserialization RCE (#42); via langchain/langgraph
    "orjson>=3.11.8",                  # fix nested-JSON recursion DoS (#44); via chromadb, langchain*, langgraph
    "pygments>=2.20.0",                # fix ReDoS in GUID/ID regex CVE-2026-4539 (#50); via rich
    "langchain-text-splitters>=1.1.2", # fix split_text_from_url SSRF redirect bypass GHSA-fv5p-p927-qmxr (#67)
    "transformers>=5.5.4",             # fix Trainer RCE via torch.load CVE-2026-1839 (#61); via sentence-transformers
    "lxml>=6.1.0",                     # fix XXE in iterparse/ETCompatXMLParser CVE-2026-41066 (#70); via ddgs (and overrides linkedin-api's <6.0.0 cap)
    "pydantic-settings>=2.14.2",       # fix NestedSecretsSettingsSource symlink escape / secrets_dir_max_size bypass GHSA-4xgf-cpjx-pc3j (#79); via langchain/langsmith
    "h2>=4.4.1",                       # fix duplicate Host header request smuggling CVE-2026-71554 (#96); via ddgs -> httpx[http2]
]

[tool.ruff]
target-version = "py312"
line-length = 120
extend-exclude = ["venv", ".venv", "data", "logs", "wiki", "docs"]

[tool.ruff.lint]
# Start conservative: pyflakes (F), pycodestyle errors (E), bugbear (B),
# isort (I), pyupgrade (UP). Expand over time.
select = ["E", "F", "B", "I", "UP"]
ignore = [
    "E501",  # line length is handled by the formatter
    "B008",  # function calls in argument defaults (FastAPI Depends() idiom)
    "UP042", # str+Enum classes: switching to StrEnum changes str() output
]

[tool.ruff.lint.per-file-ignores]
"tests/**" = ["B", "E402"]

[tool.mypy]
python_version = "3.12"
warn_unused_ignores = true
warn_redundant_casts = true
ignore_missing_imports = true
# Adopt gradually: do not fail on untyped defs yet across the legacy codebase.
disallow_untyped_defs = false
exclude = "(venv|\\.venv|data|logs|wiki|docs|tests)/"