config.example.yaml

# ============================================================================
# LLMTrace Proxy — Example Configuration
# ============================================================================
#
# Copy this file to config.yaml and adjust values for your environment.
#
# Usage:
#   llmtrace-proxy --config config.yaml          # start the proxy
#   llmtrace-proxy validate --config config.yaml  # validate without starting
#   llmtrace-proxy --help                          # show CLI help
#   llmtrace-proxy --version                       # show version
#
# Environment variable overrides (highest precedence after CLI flags):
#   LLMTRACE_CONFIG                — path to this config file
#   LLMTRACE_LISTEN_ADDR           — override listen_addr
#   LLMTRACE_UPSTREAM_URL          — override upstream_url
#   LLMTRACE_STORAGE_PROFILE       — override storage.profile
#   LLMTRACE_STORAGE_DATABASE_PATH — override storage.database_path
#   LLMTRACE_CLICKHOUSE_URL        — override storage.clickhouse_url
#   LLMTRACE_CLICKHOUSE_DATABASE   — override storage.clickhouse_database
#   LLMTRACE_POSTGRES_URL          — override storage.postgres_url
#   LLMTRACE_REDIS_URL             — override storage.redis_url
#   LLMTRACE_ML_ENABLED            — enable/disable ML security analysis ("1"/"0")
#   LLMTRACE_ML_PRELOAD            — enable/disable ML model preloading ("1"/"0")
#   LLMTRACE_ML_CACHE_DIR          — override security_analysis.ml_cache_dir
#   LLMTRACE_LOG_LEVEL             — override logging.level
#   LLMTRACE_LOG_FORMAT            — override logging.format
#   RUST_LOG                       — fine-grained tracing filter (e.g. "llmtrace_proxy=debug,info")
# ============================================================================

# ---------------------------------------------------------------------------
# Network
# ---------------------------------------------------------------------------

# Address and port to bind the proxy server to.
listen_addr: "0.0.0.0:8080"

# Upstream LLM provider URL (OpenAI, vLLM, Ollama, etc.).
upstream_url: "https://api.openai.com"

# ---------------------------------------------------------------------------
# Storage
# ---------------------------------------------------------------------------

storage:
  # Profile: "lite" (SQLite, zero infrastructure), "memory" (in-memory, lost on restart),
  # or "production" (ClickHouse + PostgreSQL + Redis — see docker-compose.yml).
  profile: "lite"
  # Database file path (used by the "lite" profile).
  database_path: "llmtrace.db"

  # --- Production profile settings (only used when profile = "production") ---
  # ClickHouse HTTP URL for trace/span analytical storage.
  # clickhouse_url: "http://localhost:8123"
  # ClickHouse database name (created automatically on startup).
  # clickhouse_database: "llmtrace"
  # PostgreSQL connection URL for metadata (tenants, configs, audit events).
  # postgres_url: "postgres://llmtrace:llmtrace@localhost:5432/llmtrace"
  # Redis connection URL for hot-query cache and sessions.
  # redis_url: "redis://127.0.0.1:6379"

  # Automatically run pending database migrations on startup.
  # Default: true (convenient for development). Set to false in production
  # and use `llmtrace-proxy migrate --config config.yaml` explicitly.
  auto_migrate: true

# ---------------------------------------------------------------------------
# Logging
# ---------------------------------------------------------------------------

logging:
  # Log level: trace, debug, info, warn, error.
  # Can be overridden by LLMTRACE_LOG_LEVEL env var or --log-level CLI flag.
  # RUST_LOG env var takes highest precedence for fine-grained control.
  level: "info"
  # Output format: "text" (human-readable) or "json" (structured, machine-parseable).
  # Can be overridden by LLMTRACE_LOG_FORMAT env var or --log-format CLI flag.
  format: "text"

# ---------------------------------------------------------------------------
# Timeouts
# ---------------------------------------------------------------------------

# Request timeout in milliseconds (covers full upstream round-trip).
timeout_ms: 30000

# Connection timeout in milliseconds (TCP connect phase only).
connection_timeout_ms: 5000

# Maximum number of concurrent connections.
max_connections: 1000

# ---------------------------------------------------------------------------
# TLS (for the proxy listener itself)
# ---------------------------------------------------------------------------

enable_tls: false
# tls_cert_file: "/etc/ssl/certs/proxy.crt"
# tls_key_file: "/etc/ssl/private/proxy.key"

# ---------------------------------------------------------------------------
# Feature toggles
# ---------------------------------------------------------------------------

# Enable regex-based security analysis (prompt injection, PII detection).
enable_security_analysis: true

# Enable trace storage to the configured storage backend.
enable_trace_storage: true

# Enable streaming SSE passthrough for "stream": true requests.
enable_streaming: true

# Maximum request body size in bytes (default: 50 MB).
max_request_size_bytes: 52428800

# Maximum response body size to collect for trace storage (bytes).
# Responses larger than this are truncated in traces but forwarded to the
# client in full. Default: 50MB.
# max_response_size_bytes: 52428800

# Timeout for async security analysis in milliseconds.
security_analysis_timeout_ms: 5000

# Timeout for async trace storage in milliseconds.
trace_storage_timeout_ms: 10000

# ---------------------------------------------------------------------------
# Rate limiting
# ---------------------------------------------------------------------------

rate_limiting:
  enabled: true
  requests_per_second: 100
  burst_size: 200
  window_seconds: 60
  # Per-tenant overrides — keyed by tenant UUID
  tenant_overrides:
    # "tenant-uuid-here":
    #   requests_per_second: 500
    #   burst_size: 1000

# ---------------------------------------------------------------------------
# Circuit breaker — degrades to pure pass-through on repeated failures
# ---------------------------------------------------------------------------

circuit_breaker:
  enabled: true
  # Number of consecutive failures before opening the circuit.
  failure_threshold: 10
  # Time in ms to wait before trying a probe call (half-open state).
  recovery_timeout_ms: 30000
  # Number of probe calls allowed in half-open state.
  half_open_max_calls: 3

# ---------------------------------------------------------------------------
# Alert engine — webhook notifications for security findings
# ---------------------------------------------------------------------------

alerts:
  # Enable the alert engine. When enabled, security findings that exceed the
  # configured thresholds will trigger notifications to the configured channels.
  enabled: false

  # ---------- Legacy mode (backward compatible) ----------
  # If no `channels` are defined, these top-level fields are used:
  # webhook_url: "https://hooks.slack.com/services/T00/B00/xxx"
  # min_severity: "High"
  # min_security_score: 70

  # ---------- Multi-channel mode ----------
  # When `channels` is defined, each channel has its own type, URL, and severity filter.
  # Supported types: webhook, slack, pagerduty (email: coming soon)
  channels: []
  #   - type: slack
  #     url: "https://hooks.slack.com/services/T00/B00/xxx"
  #     min_severity: "Medium"          # Slack gets Medium and above
  #     min_security_score: 50
  #
  #   - type: pagerduty
  #     routing_key: "your-pagerduty-routing-key"
  #     min_severity: "Critical"        # PagerDuty only for Critical
  #     min_security_score: 90
  #
  #   - type: webhook
  #     url: "https://your-server.com/alerts"
  #     min_severity: "High"
  #     min_security_score: 70

  # Global cooldown in seconds between repeated alerts for the same finding type.
  cooldown_seconds: 300

  # Optional: alert escalation (re-send at higher severity if unacknowledged).
  # escalation:
  #   enabled: false
  #   escalate_after_seconds: 600

# ---------------------------------------------------------------------------
# Cost estimation — model pricing for per-request cost tracking
# ---------------------------------------------------------------------------

cost_estimation:
  # Enable cost estimation on traced requests.
  enabled: true

  # Optional: external pricing file. When set, the proxy loads model pricing
  # from this YAML file at startup and reloads it on SIGHUP — no rebuild needed.
  # If the file is missing or invalid, built-in defaults are used.
  # pricing_file: "config/pricing.yaml"

  # Inline custom model pricing overrides (take highest precedence).
  # custom_models:
  #   my-fine-tuned-model:
  #     input_per_million: 5.0
  #     output_per_million: 10.0

# ---------------------------------------------------------------------------
# Cost caps — per-agent budget & token enforcement
# ---------------------------------------------------------------------------

cost_caps:
  # Enable cost cap enforcement. When enabled, requests are checked against
  # budget limits (USD) and per-request token caps before being forwarded.
  enabled: false

  # Default budget caps applied to all tenants/agents unless overridden.
  # Each cap specifies a time window and a hard limit. Requests that would
  # push spend above the hard limit are rejected with HTTP 429.
  # Soft limits trigger an alert but still allow the request through.
  default_budget_caps:
    - window: hourly
      hard_limit_usd: 10.0
      soft_limit_usd: 8.0
    - window: daily
      hard_limit_usd: 100.0
      # soft_limit_usd: 80.0

  # Default per-request token caps (applied before the request is forwarded).
  # default_token_cap:
  #   max_prompt_tokens: 8192
  #   max_completion_tokens: 4096
  #   max_total_tokens: 16384

  # Per-agent overrides. An agent is identified by the X-LLMTrace-Agent-ID header.
  # agents:
  #   - agent_id: "heavy-agent"
  #     budget_caps:
  #       - window: daily
  #         hard_limit_usd: 500.0
  #         soft_limit_usd: 400.0
  #     token_cap:
  #       max_prompt_tokens: 16384
  #       max_completion_tokens: 8192

# ---------------------------------------------------------------------------
# gRPC ingestion gateway — high-throughput trace ingestion via tonic
# ---------------------------------------------------------------------------

grpc:
  # Enable the gRPC ingestion endpoint. When enabled, the proxy starts a
  # tonic gRPC server on a separate listen address that accepts traces in
  # the LLMTrace-native protobuf format (see crates/llmtrace-proto/llmtrace.proto).
  # Supports both unary batch and client-side streaming RPCs.
  enabled: false
  # Address and port to bind the gRPC server to.
  listen_addr: "0.0.0.0:50051"

# ---------------------------------------------------------------------------
# Streaming security analysis — real-time analysis during SSE streaming
# ---------------------------------------------------------------------------

streaming_analysis:
  # Enable incremental regex-based security checks during SSE streaming.
  # When enabled, the proxy runs lightweight pattern matching every N tokens
  # while the stream is still in progress. Findings are tagged with
  # "detection": "streaming" metadata and critical issues trigger alerts
  # mid-stream rather than waiting for stream completion.
  enabled: false
  # Number of completion tokens between each incremental analysis check.
  # Lower values detect threats faster but add marginal CPU overhead per chunk.
  token_interval: 50
  # Enable output-side analysis during SSE streaming (PII, secrets, toxicity
  # on response content in real-time). Requires output_safety.enabled = true.
  output_enabled: false
  # If a critical output safety finding is detected mid-stream, inject a
  # warning into the SSE stream and terminate. Use with caution — this will
  # cut off the LLM response mid-generation.
  early_stop_on_critical: false

# ---------------------------------------------------------------------------
# Output safety — toxicity detection and response content analysis (R6)
# ---------------------------------------------------------------------------

output_safety:
  # Enable output safety analysis on LLM responses. When enabled, the proxy
  # analyses response content for toxicity, PII leakage, and secret exposure.
  enabled: false
  # Enable toxicity detection on response content. Uses a BERT-based classifier
  # (unitary/toxic-bert) or falls back to keyword-based detection.
  toxicity_enabled: false
  # Confidence threshold for toxicity detection (0.0–1.0). Categories scoring
  # above this threshold are reported as findings.
  toxicity_threshold: 0.7
  # Block (replace) the entire response if critical toxicity is detected
  # (severe_toxic, threat, or score >= 0.9). Use with caution.
  block_on_critical: false

  # --- Hallucination detection (R9) ---
  # Enable hallucination detection on response content. Uses a two-stage
  # pipeline: a sentinel heuristic gates expensive cross-encoder inference,
  # then each response sentence is scored against the user's prompt for
  # factual consistency.
  hallucination_enabled: false
  # HuggingFace model ID for the cross-encoder hallucination detector.
  # The model scores (premise, hypothesis) pairs for factual consistency.
  hallucination_model: "vectara/hallucination_evaluation_model"
  # Threshold below which a sentence is considered potentially hallucinated
  # (0.0–1.0). Lower values are more permissive; higher values flag more.
  hallucination_threshold: 0.5
  # Minimum response length (in characters) to run hallucination detection.
  # Responses shorter than this are skipped to avoid false positives on
  # brief answers and save compute.
  hallucination_min_response_length: 50

# ---------------------------------------------------------------------------
# ML-based security analysis
# ---------------------------------------------------------------------------

# security_analysis:
#   # Enable ML-based prompt injection detection (requires binary compiled with `ml` feature).
#   ml_enabled: false
#   # HuggingFace model ID for prompt injection detection.
#   ml_model: "protectai/deberta-v3-base-prompt-injection-v2"
#   # Confidence threshold for ML detection (0.0–1.0).
#   ml_threshold: 0.8
#   # Maximum text size (bytes) passed to the analysis pipeline.
#   # Inputs larger than this are truncated before normalization and
#   # regex scanning. Default: 1MB.
#   # max_analysis_text_bytes: 1048576
#   # Local cache directory for downloaded ML models.
#   ml_cache_dir: "~/.cache/llmtrace/models"
#   # Pre-load ML models at proxy startup (eliminates cold-start latency on first request).
#   ml_preload: true
#   # Timeout in seconds for downloading ML models at startup.
#   ml_download_timeout_seconds: 300
#   # Enable ML-based NER for PII detection (person names, organisations, locations).
#   ner_enabled: false
#   # HuggingFace model ID for NER-based PII detection.
#   ner_model: "dslim/bert-base-NER"
#   # Enable dedicated jailbreak detection (runs alongside prompt injection detection).
#   # Detects DAN/character jailbreaks, system prompt extraction, privilege escalation,
#   # and encoding-based evasion (base64, ROT13, leetspeak, reversed text).
#   jailbreak_enabled: true
#   # Confidence threshold for jailbreak detection (0.0–1.0).
#   jailbreak_threshold: 0.7

# ---------------------------------------------------------------------------
# Boundary token injection defense — structural prevention for indirect
# prompt injection via tool outputs
# ---------------------------------------------------------------------------

# When enabled, the proxy wraps untrusted message content (tool role messages)
# with structural delimiter tags before forwarding to the upstream LLM provider.
# This helps the model distinguish instructions from external data, reducing
# indirect prompt injection attack success rate by ~10x (BIPIA benchmark,
# KDD 2025: ASR from 6.17% to 0.53%).
#
# The defense is complementary to detection (regex + DeBERTa ensemble) — it
# makes injections structurally harder to execute even if they evade detection.
#
# Rollout recommendation: enabled + shadow_mode: true first, then disable
# shadow_mode after validating metrics for 24+ hours.

# boundary_defense:
#   # Master toggle (default: false — opt-in).
#   enabled: false
#   # Shadow mode: compute the modified body and record all metrics, but
#   # forward the original bytes to upstream. Safe for production validation.
#   shadow_mode: false
#   # Message roles whose content will be wrapped with boundary delimiters.
#   # For OpenAI-compatible APIs, "tool" covers tool-call results.
#   wrap_roles: ["tool"]
#   # Delimiter tag name. Synthetic tag unlikely to appear in real data.
#   delimiter: "llmtrace-boundary"
#   # Append a random hex nonce to the delimiter per request to prevent
#   # attackers from pre-crafting payloads that close the boundary.
#   randomize_nonce: false
#   # Inject an explicit instruction into the system prompt telling the
#   # model to treat delimited content as untrusted data.
#   inject_system_reminder: true
#   # Custom reminder text (leave empty to use built-in default).
#   # system_reminder_text: ""
#   # IS-060 PR-2 — datamarking transform (Microsoft Spotlighting "Option C").
#   # Replaces whitespace inside detected Data zones with a Unicode
#   # Private Use Area marker codepoint, telling the upstream model
#   # (via a system-reminder addendum) that the marked text is data,
#   # not instructions. Defaults to disabled. When first enabled,
#   # operators MUST flip shadow_mode = false ONLY after one nightly
#   # cycle confirms zero upstream 4xx delta vs prior nightly.
#   datamarking:
#     enabled: false           # default: false (no-op)
#     shadow_mode: true        # compute + emit metrics, forward original bytes
#     marker_strategy:
#       kind: randomized       # sample fresh PUA codepoint per request
#     # For deterministic nightly diffs, pin a fixed marker instead:
#     # marker_strategy:
#     #   kind: fixed
#     #   value: "\uE000"

# ---------------------------------------------------------------------------
# Graceful shutdown — connection draining and task completion
# ---------------------------------------------------------------------------

shutdown:
  # Maximum seconds to wait for in-flight background tasks (trace capture,
  # security analysis) to complete after a SIGTERM/SIGINT signal. If tasks
  # are still running after this timeout the process force-exits.
  # In Kubernetes, set terminationGracePeriodSeconds > timeout_seconds
  # (e.g. 60s grace period with 30s shutdown timeout gives 30s headroom).
  timeout_seconds: 30

# ---------------------------------------------------------------------------
# Health check endpoint
# ---------------------------------------------------------------------------

health_check:
  enabled: true
  path: "/health"
  interval_seconds: 10
  timeout_ms: 5000
  retries: 3

# ---------------------------------------------------------------------------
# LLM Judge — optional third detector (LLM-as-a-Judge)
# ---------------------------------------------------------------------------
#
# Adds a dedicated language-model verdict alongside the regex and
# DeBERTa detectors. Fail-open: misbehaviour never changes enforcement
# versus the no-judge baseline. Disabled out of the box.
#
# Full setup guide with shadow-mode rollout + metrics:
# https://docs.llmtrace.io/guides/llm-judge/
#
# Evaluation report (gpt-4o-mini on 27 labeled corpora):
# https://docs.llmtrace.io/research/results/judge_evaluation_gpt4o_mini_2026-04-20/
#
# API keys are read from environment variables at startup:
#   LLMTRACE_JUDGE_OPENAI_API_KEY        for backend: openai
#   LLMTRACE_JUDGE_ANTHROPIC_API_KEY     for backend: anthropic
#   (vLLM does not require a key)
#
# judge:
#   enabled: false
#   backend: cascade                          # "cascade" | "deberta" | "openai" | "anthropic" | "vllm"
#
#   # Cascade = DeBERTa fast-judge (local) escalating to a slow tier on
#   # ambiguous-band verdicts. Ship today with slow_backend: null
#   # (fast-only); flip to vllm / openai / anthropic when the slow
#   # model is available.
#   cascade:
#     fast_backend: deberta
#     slow_backend: null                      # or "vllm" | "openai" | "anthropic"
#     ambiguous_low: 0.3
#     ambiguous_high: 0.7
#
#   deberta:
#     model_id: "protectai/deberta-v3-base-prompt-injection-v2"
#     threshold: 0.5
#     # cache_dir: "~/.cache/llmtrace/models"
#
#   openai:
#     base_url: "https://api.openai.com"      # OpenAI-compatible gateway (OpenRouter, Azure, LiteLLM)
#     model: "gpt-4o-mini"
#     max_tokens: 512
#     temperature: 0.1
#   anthropic:
#     model: "claude-3-5-haiku-20241022"
#     max_tokens: 512
#     temperature: 0.1
#   vllm:
#     base_url: "http://localhost:8000"
#     model: "security-judge-v1"
#     max_tokens: 512
#     temperature: 0.1
#     allow_plaintext: false                  # true only for loopback hosts
#
#   worker:
#     channel_buffer: 1000
#     max_concurrency: 4
#     timeout_ms: 30000
#     max_analysis_text_bytes: 65536
#     total_deadline_ms: 45000
#   retry:
#     max_retries: 2
#     backoff_base_ms: 1000
#   promotion:
#     min_confidence: 0.7                     # pre-calibration placeholder
#     min_security_score: 60
#     require_ensemble_support: true
#     shadow: false                           # set true during initial rollout
#
#   system_prompt: ""                         # "" uses built-in hardened default
#   min_score_threshold: 30                   # only judge prompts with prior score >= this
#   persist_verdicts: true