Crawllama/.env.example at 1.4.11 · arn-c0de/Crawllama · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
# =============================================================================
# CrawLlama Environment Variables
# =============================================================================
# Copy this file to .env and fill in your actual API keys and configuration
# IMPORTANT: Never commit .env to version control!

# -----------------------------------------------------------------------------
# Search API Keys (Optional - DuckDuckGo is used by default)
# -----------------------------------------------------------------------------
BRAVE_API_KEY=
SERPER_API_KEY=

# -----------------------------------------------------------------------------
# OSINT API Keys (Optional - for enhanced breach/leak detection)
# -----------------------------------------------------------------------------
# Have I Been Pwned - Free API key (required for HIBP breach checks)
# Get yours at: https://haveibeenpwned.com/API/Key
HIBP_API_KEY=

# LeakCheck - Free tier available (3 requests/day)
# Get yours at: https://leakcheck.io/
LEAKCHECK_API_KEY=

# DeHashed - Paid API access
# Get yours at: https://www.dehashed.com/
DEHASHED_USERNAME=
DEHASHED_API_KEY=

# Snusbase - Paid API access
# Get yours at: https://snusbase.com/
SNUSBASE_API_KEY=

# IntelX - Intelligence data platform
# Get yours at: https://intelx.io/
INTELX_API_KEY=

# -----------------------------------------------------------------------------
# LLM API Keys (Optional - for cloud LLM providers)
# -----------------------------------------------------------------------------
# OpenAI API key (for GPT models)
# Get yours at: https://platform.openai.com/api-keys
OPENAI_API_KEY=

# Anthropic API key (for Claude models)
# Get yours at: https://console.anthropic.com/
ANTHROPIC_API_KEY=

# Groq API key (for fast inference)
# Get yours at: https://console.groq.com/
GROQ_API_KEY=

# -----------------------------------------------------------------------------
# Proxy Configuration (Optional - for anonymization)
# -----------------------------------------------------------------------------
HTTP_PROXY=
HTTPS_PROXY=
NO_PROXY=localhost,127.0.0.1

# -----------------------------------------------------------------------------
# Tor Mode (Optional - route ALL outbound web traffic through Tor)
# -----------------------------------------------------------------------------
# TOR_MODE=1 sends every HTTP(S) request (crawler, RAG/tool fetches, OSINT
# lookups, cloud LLM calls) through the Tor SOCKS5 proxy below. Requires a
# running Tor service (apt install tor / brew install tor). On startup the
# Tor circuit is verified via check.torproject.org; if Tor is unreachable,
# the application refuses to start (fail fast — no silent direct fallback).
# Loopback/LAN targets (e.g. a local Ollama server) stay direct: that
# traffic never leaves your network and Tor cannot reach it anyway.
# These env vars override the "tor" section in config.json (leave empty to
# use the config.json values; defaults: 127.0.0.1:9050).
TOR_MODE=
TOR_SOCKS_HOST=
TOR_SOCKS_PORT=

# -----------------------------------------------------------------------------
# FastAPI Security (Required for production)
# -----------------------------------------------------------------------------
# API key for accessing CrawLlama API endpoints
# IMPORTANT: Must be at least 32 characters long!
# Generate with: python -c "import secrets; print(secrets.token_urlsafe(32))"
CRAWLLAMA_API_KEY=your_secure_api_key_here_min_32_chars

# Enable development mode (disables API key requirement)
# WARNING: Only use in development! Never in production!
CRAWLLAMA_DEV_MODE=false

# Rate limiting (requests per minute)
RATE_LIMIT=60

# CORS allowed origins (comma-separated)
ALLOWED_ORIGINS=http://localhost:3000,http://localhost:8080

# Allowed hosts (comma-separated)
ALLOWED_HOSTS=localhost,127.0.0.1,0.0.0.0

# -----------------------------------------------------------------------------
# Redis Configuration (Optional - for distributed rate limiting)
# -----------------------------------------------------------------------------
# Redis connection URL (leave empty to use in-memory fallback)
REDIS_URL=redis://localhost:6379/0

# Redis rate limiting secret (for distributed setups)
RATE_LIMIT_SECRET=

# -----------------------------------------------------------------------------
# Development & Debugging
# -----------------------------------------------------------------------------
DEBUG=false