voidaccess/requirements.in at main · Averroeskw/voidaccess · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
# VoidAccess production dependencies (unpinned)
# Run: pip-compile requirements.in --output-file requirements.txt

bs4
pysocks
requests
streamlit
python-dotenv
langchain-openai
langchain-ollama
backports.tarfile
langchain-anthropic
langchain_community
langchain_google_genai

# Phase 1A — persistent database
sqlalchemy[asyncio]>=2.0
alembic
psycopg2-binary

# Phase 1B — async scraper
aiohttp
aiohttp-socks
trafilatura

# Phase 1C — recursive crawler
sentence-transformers

# Phase 1D — expanded sources
telethon

# Phase 2 — entity extraction pipeline
spacy
# run: python -m spacy download en_core_web_sm

# Phase 3 — graph relationship mapping
networkx
pyvis

# Phase 4 — monitoring + vector memory
chromadb
apscheduler
pyyaml
filelock

# Phase 5 — industry-standard exports + REST API
fastapi
uvicorn[standard]
stix2
python-multipart
python-jose[cryptography]>=3.3.0
passlib[bcrypt]>=1.7.4
slowapi
limits

# Phase 6 — advanced capabilities
langdetect

# Phase 7 — JS rendering for dark web sites (requires playwright browsers)
playwright==1.44.0

# Auth — token blacklist
redis>=5.0