fix(litellm-amd): re-enable master-key auth + rotate sk-lemonade

dream-server/.env.schema.json

@@ -91,6 +91,11 @@
       "description": "LiteLLM API gateway master key",
       "secret": true
     },
+    "LITELLM_LEMONADE_API_KEY": {
+      "type": "string",
+      "description": "Outbound API key LiteLLM sends when calling the local Lemonade LLM backend (AMD installs only). Generated per-install at phase 06.",
+      "secret": true
+    },
     "OPENCLAW_TOKEN": {
       "type": "string",
       "description": "OpenClaw agent framework token",

dream-server/Makefile

@@ -29,6 +29,7 @@ test: ## Run unit and contract tests
 	@echo ""
 	@echo "=== AMD/Lemonade contracts ==="
 	@bash tests/contracts/test-amd-lemonade-contracts.sh
+	@bash tests/test-litellm-amd-auth-enforced.sh
 	@echo ""
 	@echo "=== Overlay/plist contracts ==="
 	@bash tests/contracts/test-overlay-map-coherence.sh

dream-server/bin/dream-host-agent.py

@@ -1675,19 +1675,20 @@ def _do_model_activate(self, model_id: str):
             # Regenerate LiteLLM lemonade config so it routes to the new model.
             # Only written on AMD installs where lemonade.yaml exists.
             if lemonade_yaml.exists():
+                lemonade_api_key = os.environ.get("LITELLM_LEMONADE_API_KEY", "sk-lemonade")
                 lemonade_yaml.write_text(
                     f"model_list:\n"
                     f"  - model_name: default\n"
                     f"    litellm_params:\n"
                     f"      model: openai/extra.{gguf_file}\n"
                     f"      api_base: http://llama-server:8080/api/v1\n"
-                    f"      api_key: sk-lemonade\n"
+                    f"      api_key: {lemonade_api_key}\n"
                     f"\n"
                     f"  - model_name: \"*\"\n"
                     f"    litellm_params:\n"
                     f"      model: openai/extra.{gguf_file}\n"
                     f"      api_base: http://llama-server:8080/api/v1\n"
-                    f"      api_key: sk-lemonade\n"
+                    f"      api_key: {lemonade_api_key}\n"
                     f"\n"
                     f"litellm_settings:\n"
                     f"  drop_params: true\n"
@@ -1982,13 +1983,14 @@ def _write_lemonade_config(install_dir: Path, gguf_file: str):
     Mirrors bootstrap-upgrade.sh lines 369-382.
     """
     config_path = install_dir / "config" / "litellm" / "lemonade.yaml"
+    lemonade_api_key = os.environ.get("LITELLM_LEMONADE_API_KEY", "sk-lemonade")
     content = (
         "model_list:\n"
         "  - model_name: \"*\"\n"
         "    litellm_params:\n"
         f"      model: openai/extra.{gguf_file}\n"
         "      api_base: http://llama-server:8080/api/v1\n"
-        "      api_key: sk-lemonade\n"
+        f"      api_key: {lemonade_api_key}\n"
         "\n"
         "litellm_settings:\n"
         "  drop_params: true\n"

dream-server/config/litellm/lemonade.yaml

@@ -10,6 +10,7 @@ model_list:
     litellm_params:
       model: openai/extra.GGUF_FILENAME_HERE
       api_base: http://llama-server:8080/api/v1
+      # api_key is overwritten per-install by phase 06 / bootstrap-upgrade to LITELLM_LEMONADE_API_KEY
       api_key: sk-lemonade
 
 litellm_settings:

dream-server/docker-compose.amd.yml

@@ -65,11 +65,11 @@ services:
 
   # Services route through LiteLLM (DREAM_MODE=lemonade sets LLM_API_URL=http://litellm:4000).
   # LiteLLM handles the /api/v1 translation to Lemonade internally.
-  # Auth disabled via extensions/services/litellm/compose.amd.yaml (loads after compose.yaml).
+  # LiteLLM enforces auth via LITELLM_MASTER_KEY=${LITELLM_KEY}; clients must present LITELLM_KEY.
 
   open-webui:
     environment:
-      - OPENAI_API_KEY=no-key
+      - OPENAI_API_KEY=${LITELLM_KEY}
       - ENABLE_IMAGE_GENERATION=${ENABLE_IMAGE_GENERATION:-true}
 
   dashboard-api:

dream-server/extensions/services/dashboard-api/tests/test_model_activate.py

@@ -1,6 +1,7 @@
 """Tests for AMD model activation helpers in dream-host-agent.py."""
 
 import importlib.util
+import re
 import subprocess
 import sys
 from pathlib import Path
@@ -116,7 +117,11 @@ def test_writes_correct_content(self, tmp_path):
         content = (litellm_dir / "lemonade.yaml").read_text()
         assert "model: openai/extra.Qwen3.5-9B-Q4_K_M.gguf" in content
         assert "api_base: http://llama-server:8080/api/v1" in content
-        assert "api_key: sk-lemonade" in content
+        # Per-install rotation (#521): assert api_key field is present and non-empty,
+        # not the legacy literal "sk-lemonade".
+        api_key_match = re.search(r'^\s*api_key:\s*(\S+)$', content, re.MULTILINE)
+        assert api_key_match, "api_key field missing from emitted config"
+        assert api_key_match.group(1), "api_key value is empty"
         assert 'model_name: "*"' in content
         assert "drop_params: true" in content
 

dream-server/extensions/services/litellm/compose.amd.yaml

@@ -1,14 +1,12 @@
-# LiteLLM AMD overlay — disable auth for local-only Lemonade installs.
+# LiteLLM AMD overlay — Langfuse-aware entrypoint for Lemonade installs.
+# Inherits LITELLM_MASTER_KEY from base compose.yaml so proxy auth is enforced;
+# the literal `${LITELLM_KEY}` written to .env at install time is non-empty.
 # All LiteLLM ports bind to 127.0.0.1 — no external exposure.
-# LiteLLM checks `is not None` (not truthiness), so empty string still
-# enables auth. The only way to disable it is to unset the env var entirely.
-# Loads after compose.yaml via GPU overlay discovery, so entrypoint wins.
 services:
   litellm:
     entrypoint: ["/bin/sh", "-c"]
     command:
       - |
-        unset LITELLM_MASTER_KEY
         if [ "$$LANGFUSE_ENABLED" = "true" ]; then
           python3 -c "
         import yaml

dream-server/extensions/services/perplexica/compose.yaml

@@ -8,7 +8,7 @@ services:
     environment:
       - SEARXNG_API_URL=http://searxng:8080
       - OPENAI_BASE_URL=${LLM_API_URL:-http://llama-server:8080}/v1
-      - OPENAI_API_KEY=${OPENAI_API_KEY:-no-key}
+      - OPENAI_API_KEY=${LITELLM_KEY:-${OPENAI_API_KEY:-no-key}}
       - HOSTNAME=0.0.0.0
     volumes:
       - perplexica-data:/home/perplexica/data

dream-server/extensions/services/privacy-shield/compose.yaml

@@ -10,7 +10,7 @@ services:
       - no-new-privileges:true
     environment:
       - TARGET_API_URL=${LLM_API_URL:-http://llama-server:8080}/v1
-      - TARGET_API_KEY=${TARGET_API_KEY:-not-needed}
+      - TARGET_API_KEY=${LITELLM_KEY:-${TARGET_API_KEY:-not-needed}}
       - SHIELD_PORT=${SHIELD_PORT:-8085}
       - SHIELD_API_KEY_PATH=${SHIELD_API_KEY_PATH:-/data/shield_api_key}
       - PII_CACHE_ENABLED=${PII_CACHE_ENABLED:-true}

dream-server/installers/phases/06-directories.sh

@@ -203,6 +203,7 @@ Fix with: sudo chown -R \$(id -u):\$(id -g) $INSTALL_DIR/config $INSTALL_DIR/dat
     WEBUI_SECRET=$(_env_get WEBUI_SECRET "$(openssl rand -hex 32 2>/dev/null || head -c 32 /dev/urandom | xxd -p)")
     N8N_PASS=$(_env_get N8N_PASS "$(openssl rand -base64 16 2>/dev/null || head -c 16 /dev/urandom | base64)")
     LITELLM_KEY=$(_env_get LITELLM_KEY "sk-dream-$(openssl rand -hex 16 2>/dev/null || head -c 16 /dev/urandom | xxd -p)")
+    LITELLM_LEMONADE_API_KEY=$(_env_get LITELLM_LEMONADE_API_KEY "sk-dream-lemonade-$(openssl rand -hex 16 2>/dev/null || head -c 16 /dev/urandom | xxd -p)")
     LIVEKIT_SECRET=$(_env_get LIVEKIT_API_SECRET "$(openssl rand -base64 32 2>/dev/null || head -c 32 /dev/urandom | base64)")
     DASHBOARD_API_KEY=$(_env_get DASHBOARD_API_KEY "$(openssl rand -hex 32 2>/dev/null || head -c 32 /dev/urandom | xxd -p)")
     DREAM_AGENT_KEY=$(_env_get DREAM_AGENT_KEY "$(openssl rand -hex 32 2>/dev/null || head -c 32 /dev/urandom | xxd -p)")
@@ -327,6 +328,9 @@ HSA_XNACK=1
 ROCBLAS_USE_HIPBLASLT=1
 AMDGPU_TARGET=gfx1151
 LLAMA_CPP_REF=b8763
+
+#=== LiteLLM → Lemonade outbound key (AMD only) ===
+LITELLM_LEMONADE_API_KEY=${LITELLM_LEMONADE_API_KEY}
 AMD_ENV
 fi)
 $(if [[ "$GPU_BACKEND" == "sycl" ]]; then cat << INTEL_ENV
@@ -446,13 +450,13 @@ model_list:
     litellm_params:
       model: openai/extra.${_active_gguf}
       api_base: http://llama-server:8080/api/v1
-      api_key: sk-lemonade
+      api_key: ${LITELLM_LEMONADE_API_KEY}
 
   - model_name: "*"
     litellm_params:
       model: openai/extra.${_active_gguf}
       api_base: http://llama-server:8080/api/v1
-      api_key: sk-lemonade
+      api_key: ${LITELLM_LEMONADE_API_KEY}
 
 litellm_settings:
   drop_params: true

dream-server/installers/windows/lib/env-generator.ps1

@@ -131,6 +131,7 @@ function New-DreamEnv {
     $webuiSecret     = Get-EnvOrNew "WEBUI_SECRET"       (New-SecureHex -Bytes 32)
     $n8nPass         = Get-EnvOrNew "N8N_PASS"           (New-SecureBase64 -Bytes 16)
     $litellmKey      = Get-EnvOrNew "LITELLM_KEY"        "sk-dream-$(New-SecureHex -Bytes 16)"
+    $litellmLemonadeApiKey = Get-EnvOrNew "LITELLM_LEMONADE_API_KEY" "sk-dream-lemonade-$(New-SecureHex -Bytes 16)"
     $livekitSecret   = Get-EnvOrNew "LIVEKIT_API_SECRET" (New-SecureBase64 -Bytes 32)
     $livekitApiKey   = Get-EnvOrNew "LIVEKIT_API_KEY"    (New-SecureHex -Bytes 16)
     $dashboardApiKey = Get-EnvOrNew "DASHBOARD_API_KEY"  (New-SecureHex -Bytes 32)
@@ -290,6 +291,7 @@ DREAM_AGENT_KEY=$dreamAgentKey
 N8N_USER=admin@dreamserver.local
 N8N_PASS=$n8nPass
 LITELLM_KEY=$litellmKey
+$(if ($GpuBackend -eq "amd") { "LITELLM_LEMONADE_API_KEY=$litellmLemonadeApiKey" })
 LIVEKIT_API_KEY=$livekitApiKey
 LIVEKIT_API_SECRET=$livekitSecret
 OPENCLAW_TOKEN=$openclawToken

dream-server/scripts/bootstrap-upgrade.sh

@@ -464,19 +464,24 @@ if [[ -n "$DOCKER_CMD" ]] && $DOCKER_CMD ps --filter name=dream-llama-server --f
         # reference the exact ID, not a wildcard passthrough.
         if $DOCKER_CMD ps --filter name=dream-litellm --format '{{.Names}}' 2>/dev/null | grep -q dream-litellm; then
             log "Updating LiteLLM config for new model: extra.${FULL_GGUF_FILE}"
+            # Read per-install lemonade key from .env; fall back to literal so
+            # older installs without the key still produce a valid config (lemonade
+            # itself ignores the value).
+            LITELLM_LEMONADE_API_KEY=$(grep '^LITELLM_LEMONADE_API_KEY=' "$ENV_FILE" 2>/dev/null | cut -d= -f2- | tr -d '"'"'")
+            : "${LITELLM_LEMONADE_API_KEY:=sk-lemonade}"
             cat > "$INSTALL_DIR/config/litellm/lemonade.yaml" << LITELLM_UPGRADE_EOF
 model_list:
   - model_name: default
     litellm_params:
       model: openai/extra.${FULL_GGUF_FILE}
       api_base: http://llama-server:8080/api/v1
-      api_key: sk-lemonade
+      api_key: ${LITELLM_LEMONADE_API_KEY}
 
   - model_name: "*"
     litellm_params:
       model: openai/extra.${FULL_GGUF_FILE}
       api_base: http://llama-server:8080/api/v1
-      api_key: sk-lemonade
+      api_key: ${LITELLM_LEMONADE_API_KEY}
 
 litellm_settings:
   drop_params: true

dream-server/tests/contracts/test-amd-lemonade-contracts.sh

@@ -50,13 +50,14 @@ else
 fi
 
 # ---------------------------------------------------------------------------
-# 4. LiteLLM AMD overlay blanks LITELLM_MASTER_KEY
+# 4. LiteLLM AMD overlay does NOT unset LITELLM_MASTER_KEY (auth must be enforced)
 # ---------------------------------------------------------------------------
-echo "[contract] LiteLLM auth disabled for AMD"
-if grep -q 'unset LITELLM_MASTER_KEY' extensions/services/litellm/compose.amd.yaml 2>/dev/null; then
-    pass "litellm compose.amd.yaml: LITELLM_MASTER_KEY unset in entrypoint"
+echo "[contract] LiteLLM auth enforced on AMD"
+if grep -qE '^[[:space:]]*unset[[:space:]]+LITELLM_MASTER_KEY' \
+        extensions/services/litellm/compose.amd.yaml 2>/dev/null; then
+    fail "litellm compose.amd.yaml: 'unset LITELLM_MASTER_KEY' is an auth bypass — must be removed"
 else
-    fail "litellm compose.amd.yaml: must unset LITELLM_MASTER_KEY (empty string still enables auth)"
+    pass "litellm compose.amd.yaml: no 'unset LITELLM_MASTER_KEY' (auth enforced)"
 fi
 
 # ---------------------------------------------------------------------------

dream-server/tests/test-litellm-amd-auth-enforced.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+# Regression guard for #519: ensure LiteLLM auth is enforced on AMD installs
+# and that open-webui no longer ships a hardcoded "no-key" credential.
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+cd "$ROOT_DIR"
+
+PASS=0
+FAIL=0
+
+pass() { echo "[PASS] $1"; PASS=$((PASS + 1)); }
+fail() { echo "[FAIL] $1"; FAIL=$((FAIL + 1)); }
+
+echo "[guard] LiteLLM AMD overlay must not unset LITELLM_MASTER_KEY"
+if grep -qE '^[[:space:]]*unset[[:space:]]+LITELLM_MASTER_KEY' \
+        extensions/services/litellm/compose.amd.yaml; then
+    fail "compose.amd.yaml: 'unset LITELLM_MASTER_KEY' present — auth bypass regression"
+else
+    pass "compose.amd.yaml: no 'unset LITELLM_MASTER_KEY'"
+fi
+
+echo "[guard] open-webui must not hardcode OPENAI_API_KEY=no-key on AMD"
+if grep -qE '^[[:space:]]*-[[:space:]]*OPENAI_API_KEY=no-key' docker-compose.amd.yml; then
+    fail "docker-compose.amd.yml: hardcoded OPENAI_API_KEY=no-key — open-webui will fail auth"
+else
+    pass "docker-compose.amd.yml: no hardcoded OPENAI_API_KEY=no-key"
+fi
+
+# Bundled extension fixes (#519 downstream consumers): when LiteLLM auth is
+# enforced on AMD-local, every extension that routes through LLM_API_URL must
+# present LITELLM_KEY by default. Use a fallback chain so user-supplied keys
+# still win, and so non-AMD/non-LiteLLM installs are unchanged.
+echo "[guard] perplexica must use LITELLM_KEY fallback chain"
+if grep -qF 'OPENAI_API_KEY=${LITELLM_KEY:-${OPENAI_API_KEY:-no-key}}' \
+        extensions/services/perplexica/compose.yaml; then
+    pass "perplexica: OPENAI_API_KEY uses LITELLM_KEY fallback chain"
+else
+    fail "perplexica: OPENAI_API_KEY missing LITELLM_KEY fallback — would 401 on AMD-local"
+fi
+
+echo "[guard] privacy-shield must use LITELLM_KEY fallback chain"
+if grep -qF 'TARGET_API_KEY=${LITELLM_KEY:-${TARGET_API_KEY:-not-needed}}' \
+        extensions/services/privacy-shield/compose.yaml; then
+    pass "privacy-shield: TARGET_API_KEY uses LITELLM_KEY fallback chain"
+else
+    fail "privacy-shield: TARGET_API_KEY missing LITELLM_KEY fallback — would 401 on AMD-local"
+fi
+
+echo
+echo "Passed: $PASS  Failed: $FAIL"
+[[ $FAIL -eq 0 ]]