fix(installer): fall back when AMD devices are unavailable

Closes #1131

Author: Lightheartdevs

dream-server/docs/TROUBLESHOOTING.md

@@ -38,6 +38,28 @@ sudo nvidia-ctk runtime configure --runtime=docker
 sudo systemctl restart docker
 ```
 
+### AMD GPU Devices Missing in LXD/LXC
+
+**Error:** installer reports missing `/dev/kfd`, `/dev/dri`, or `/dev/dri/renderD*`, or AMD services never become healthy inside an LXD container.
+
+**Cause:** LXD/LXC containers can expose enough host CPU/sysfs information for Dream Server to recognize AMD/Strix Halo hardware while still hiding the actual GPU device nodes that Docker must mount into ROCm containers.
+
+**Fix for GPU acceleration:** pass the GPU devices from the LXD host into the container, then re-run the installer.
+
+```bash
+lxc config set <container> security.nesting=true
+lxc config device add <container> gpu gpu
+lxc config device add <container> kfd unix-char path=/dev/kfd
+```
+
+**CPU fallback:** if you do not need GPU acceleration inside the container, run:
+
+```bash
+GPU_BACKEND=cpu ./install.sh
+```
+
+Fresh installs now fall back to CPU mode automatically when AMD is auto-detected but the required device nodes are unavailable inside a container. If you explicitly force `GPU_BACKEND=amd`, the installer fails fast with passthrough guidance instead.
+
 ---
 
 ## Startup Issues

dream-server/installers/lib/compose-select.sh

@@ -51,6 +51,11 @@ resolve_compose_config() {
                 COMPOSE_FLAGS="-f docker-compose.base.yml"
                 COMPOSE_FILE="docker-compose.base.yml"
             fi
+        elif [[ "$GPU_BACKEND" == "cpu" ]]; then
+            if [[ -f "$SCRIPT_DIR/docker-compose.base.yml" && -f "$SCRIPT_DIR/docker-compose.cpu.yml" ]]; then
+                COMPOSE_FLAGS="-f docker-compose.base.yml -f docker-compose.cpu.yml"
+                COMPOSE_FILE="docker-compose.cpu.yml"
+            fi
         elif [[ "$TIER" == "SH_LARGE" || "$TIER" == "SH_COMPACT" ]]; then
             if [[ -f "$SCRIPT_DIR/docker-compose.base.yml" && -f "$SCRIPT_DIR/docker-compose.amd.yml" ]]; then
                 COMPOSE_FLAGS="-f docker-compose.base.yml -f docker-compose.amd.yml"
@@ -66,11 +71,6 @@ resolve_compose_config() {
                 COMPOSE_FLAGS="-f docker-compose.base.yml -f docker-compose.intel.yml"
                 COMPOSE_FILE="docker-compose.intel.yml"
             fi
-        elif [[ "$GPU_BACKEND" == "cpu" ]]; then
-            if [[ -f "$SCRIPT_DIR/docker-compose.base.yml" && -f "$SCRIPT_DIR/docker-compose.cpu.yml" ]]; then
-                COMPOSE_FLAGS="-f docker-compose.base.yml -f docker-compose.cpu.yml"
-                COMPOSE_FILE="docker-compose.cpu.yml"
-            fi
         else
             if [[ -f "$SCRIPT_DIR/docker-compose.base.yml" && -f "$SCRIPT_DIR/docker-compose.nvidia.yml" ]]; then
                 COMPOSE_FLAGS="-f docker-compose.base.yml -f docker-compose.nvidia.yml"

dream-server/installers/lib/detection.sh

@@ -146,6 +146,132 @@ calculate_llama_cpu_budget() {
     echo "$limit $reservation $available"
 }
 
+ds_in_container() {
+    [[ -f /.dockerenv ]] && return 0
+    [[ -f /run/.containerenv ]] && return 0
+    if command -v systemd-detect-virt >/dev/null 2>&1; then
+        systemd-detect-virt --container --quiet 2>/dev/null && return 0
+    fi
+    if [[ -f /proc/1/cgroup ]] && grep -qiE '(docker|containerd|kubepods|lxc|libpod)' /proc/1/cgroup 2>/dev/null; then
+        return 0
+    fi
+    if [[ -f /proc/1/environ ]] && awk -v RS='\0' -F= '$1 == "container" { found = 1 } END { exit !found }' /proc/1/environ 2>/dev/null; then
+        return 0
+    fi
+    return 1
+}
+
+ds_container_label() {
+    if command -v systemd-detect-virt >/dev/null 2>&1; then
+        local virt
+        virt=$(systemd-detect-virt --container 2>/dev/null || true)
+        [[ -n "$virt" && "$virt" != "none" ]] && { echo "$virt"; return 0; }
+    fi
+    if [[ -f /proc/1/environ ]]; then
+        local env_container
+        env_container=$(awk -v RS='\0' -F= '$1 == "container" { print $2; exit }' /proc/1/environ 2>/dev/null || true)
+        [[ -n "$env_container" ]] && { echo "$env_container"; return 0; }
+    fi
+    if [[ -f /.dockerenv ]]; then
+        echo "docker"
+    elif [[ -f /run/.containerenv ]]; then
+        echo "container"
+    elif [[ -f /proc/1/cgroup ]] && grep -qiE 'lxc|lxd' /proc/1/cgroup 2>/dev/null; then
+        echo "lxc"
+    else
+        echo "container"
+    fi
+}
+
+amd_gpu_missing_runtime_devices() {
+    local root="${DREAM_AMD_DEVICE_ROOT:-/dev}"
+    local kfd="$root/kfd"
+    local dri="$root/dri"
+    local missing=()
+
+    if [[ -n "${DREAM_AMD_DEVICE_ROOT:-}" ]]; then
+        [[ -e "$kfd" ]] || missing+=("$kfd")
+    else
+        [[ -c "$kfd" ]] || missing+=("$kfd")
+    fi
+
+    if [[ ! -d "$dri" ]]; then
+        missing+=("$dri")
+    elif ! compgen -G "$dri/renderD*" >/dev/null; then
+        missing+=("$dri/renderD*")
+    fi
+
+    printf '%s\n' "${missing[@]}"
+}
+
+amd_gpu_runtime_devices_available() {
+    [[ -z "$(amd_gpu_missing_runtime_devices)" ]]
+}
+
+amd_gpu_missing_devices_csv() {
+    local missing
+    missing="$(amd_gpu_missing_runtime_devices | xargs || true)"
+    printf '%s' "${missing// /, }"
+}
+
+show_amd_gpu_device_guidance() {
+    local missing="${1:-$(amd_gpu_missing_devices_csv)}"
+    ai_warn "AMD GPU device nodes unavailable: ${missing:-unknown}"
+    if ds_in_container; then
+        local container_label
+        container_label="$(ds_container_label)"
+        ai "Container environment detected (${container_label}). LXD/LXC containers can see CPU/sysfs clues while GPU device nodes stay hidden."
+        ai "For LXD GPU acceleration, pass the devices from the host, for example:"
+        ai "  lxc config device add <container> gpu gpu"
+        ai "  lxc config device add <container> kfd unix-char path=/dev/kfd"
+    else
+        ai "On native Linux, make sure the amdgpu/amdkfd modules are loaded and /dev/dri/renderD* exists."
+        ai "  sudo modprobe amdgpu"
+        ai "  sudo modprobe amdkfd"
+    fi
+}
+
+apply_cpu_gpu_fallback() {
+    local reason="${1:-AMD GPU runtime devices are unavailable.}"
+    ai_warn "$reason"
+    ai "Using CPU mode so installation can complete without GPU passthrough."
+
+    GPU_BACKEND="cpu"
+    GPU_NAME="None (CPU fallback)"
+    GPU_VRAM=0
+    GPU_COUNT=0
+    GPU_MEMORY_TYPE="none"
+    GPU_DEVICE_ID=""
+    HAS_NPU=false
+    [[ "${DREAM_MODE:-local}" == "lemonade" ]] && DREAM_MODE="local"
+    BACKEND_ID="cpu"
+    CAP_LLM_BACKEND="cpu"
+    CAP_GPU_VENDOR="cpu"
+    CAP_GPU_NAME="$GPU_NAME"
+    CAP_GPU_VRAM_MB=0
+    CAP_GPU_COUNT=0
+    CAP_GPU_MEMORY_TYPE="none"
+    CAP_RECOMMENDED_TIER=""
+    CAP_COMPOSE_OVERLAYS=""
+}
+
+select_cpu_fallback_tier() {
+    local ram_gb="${1:-0}"
+    if ! [[ "$ram_gb" =~ ^[0-9]+$ ]]; then
+        ram_gb=0
+    fi
+
+    if [[ "$ram_gb" -ge 96 ]]; then
+        echo "3"
+    elif [[ "$ram_gb" -ge 48 ]]; then
+        echo "2"
+    elif [[ "$ram_gb" -lt 12 ]]; then
+        echo "0"
+    else
+        echo "1"
+    fi
+}
+
 detect_gpu() {
     GPU_BACKEND="cpu"  # default to CPU-only fallback
     GPU_MEMORY_TYPE="none"

dream-server/installers/lib/ui.sh

@@ -265,6 +265,7 @@ check_service() {
   local url=$2
   local max_attempts=${3:-30}
   local timeout=${4:-10}  # Timeout per request (default 10s)
+  local container_name=${5:-}
   local spin='⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏'
   local i=0
   local lore_idx=$(( RANDOM % ${#LORE_MESSAGES[@]} ))
@@ -294,6 +295,24 @@ check_service() {
     local curl_exit=$?
     elapsed=$((elapsed + backoff))
 
+    if [[ -n "$container_name" ]]; then
+      local docker_cmd="${DOCKER_CMD:-docker}"
+      local -a docker_cmd_arr=()
+      read -r -a docker_cmd_arr <<< "$docker_cmd"
+      [[ ${#docker_cmd_arr[@]} -gt 0 ]] || docker_cmd_arr=(docker)
+      local container_state=""
+      if command -v "${docker_cmd_arr[0]}" >/dev/null 2>&1; then
+        container_state=$("${docker_cmd_arr[@]}" inspect --format '{{.State.Status}}' "$container_name" 2>/dev/null || echo "missing")
+        case "$container_state" in
+          exited|dead|missing)
+            printf "\r  ${RED}✗${NC} %-55s\n" "$name container $container_state"
+            ai_warn "$name container is $container_state; not retrying health probe."
+            return 1
+            ;;
+        esac
+      fi
+    fi
+
     # Distinguish between timeout (124), connection refused (7),
     # and transient startup errors (56 = recv error, 52 = empty reply)
     if [[ $curl_exit -eq 124 ]]; then

dream-server/installers/phases/02-detection.sh

@@ -27,6 +27,15 @@
 dream_progress 12 "detection" "Detecting GPU hardware"
 chapter "SYSTEM DETECTION"
 
+GPU_BACKEND_REQUESTED="${GPU_BACKEND:-}"
+GPU_BACKEND_FORCED=false
+[[ "${GPU_BACKEND_REQUESTED,,}" == "amd" ]] && GPU_BACKEND_FORCED=true
+GPU_BACKEND_FORCED_CPU=false
+[[ "${GPU_BACKEND_REQUESTED,,}" == "cpu" ]] && GPU_BACKEND_FORCED_CPU=true
+TIER_REQUESTED="${TIER:-}"
+TIER_FORCED=false
+[[ -n "$TIER_REQUESTED" ]] && TIER_FORCED=true
+
 # Cloud mode: skip GPU detection entirely
 if [[ "${DREAM_MODE:-local}" == "cloud" ]]; then
     ai "Cloud mode — skipping GPU detection"
@@ -108,22 +117,43 @@ DISK_AVAIL=$(df -BG "$HOME" | tail -1 | awk '{print $4}' | tr -d 'G')
 log "Available disk: ${DISK_AVAIL}GB"
 
 # GPU Detection
-ai "Detecting GPU..."
-detect_gpu || true
+if [[ "$GPU_BACKEND_FORCED_CPU" == "true" ]]; then
+    ai "GPU_BACKEND=cpu requested - skipping GPU detection"
+    apply_cpu_gpu_fallback "GPU_BACKEND=cpu was requested."
+else
+    ai "Detecting GPU..."
+    detect_gpu || true
 
-if [[ "${CAP_PROFILE_LOADED:-false}" == "true" ]]; then
-    case "${CAP_LLM_BACKEND:-}" in
-        amd)   GPU_BACKEND="amd" ;;
-        intel) GPU_BACKEND="intel" ;;
-        cpu)   GPU_BACKEND="cpu" ;;
-        apple) GPU_BACKEND="apple" ;;
-        *) GPU_BACKEND="nvidia" ;;
-    esac
-    [[ -n "${CAP_GPU_MEMORY_TYPE:-}" ]] && GPU_MEMORY_TYPE="${CAP_GPU_MEMORY_TYPE}"
-    [[ -n "${CAP_GPU_NAME:-}" ]] && GPU_NAME="${CAP_GPU_NAME}"
-    [[ -n "${CAP_GPU_VRAM_MB:-}" ]] && GPU_VRAM="${CAP_GPU_VRAM_MB}"
-    [[ -n "${CAP_GPU_COUNT:-}" ]] && GPU_COUNT="${CAP_GPU_COUNT}"
-    log "Capabilities override detection: backend=${GPU_BACKEND}, memory=${GPU_MEMORY_TYPE}, tier=${CAP_RECOMMENDED_TIER:-unknown}"
+    if [[ "${CAP_PROFILE_LOADED:-false}" == "true" ]]; then
+        case "${CAP_LLM_BACKEND:-}" in
+            amd)   GPU_BACKEND="amd" ;;
+            intel) GPU_BACKEND="intel" ;;
+            cpu)   GPU_BACKEND="cpu" ;;
+            apple) GPU_BACKEND="apple" ;;
+            *) GPU_BACKEND="nvidia" ;;
+        esac
+        [[ -n "${CAP_GPU_MEMORY_TYPE:-}" ]] && GPU_MEMORY_TYPE="${CAP_GPU_MEMORY_TYPE}"
+        [[ -n "${CAP_GPU_NAME:-}" ]] && GPU_NAME="${CAP_GPU_NAME}"
+        [[ -n "${CAP_GPU_VRAM_MB:-}" ]] && GPU_VRAM="${CAP_GPU_VRAM_MB}"
+        [[ -n "${CAP_GPU_COUNT:-}" ]] && GPU_COUNT="${CAP_GPU_COUNT}"
+        log "Capabilities override detection: backend=${GPU_BACKEND}, memory=${GPU_MEMORY_TYPE}, tier=${CAP_RECOMMENDED_TIER:-unknown}"
+    fi
+
+    if [[ "$GPU_BACKEND" == "amd" ]] && ! amd_gpu_runtime_devices_available; then
+        _amd_missing_devices="$(amd_gpu_missing_devices_csv)"
+        if [[ "${GPU_BACKEND_FORCED:-false}" == "true" ]]; then
+            ai_bad "GPU_BACKEND=amd was explicitly requested, but required AMD device nodes are missing."
+            show_amd_gpu_device_guidance "$_amd_missing_devices"
+            error "Cannot continue with AMD GPU mode until device passthrough is available."
+        elif ds_in_container; then
+            ai_warn "AMD hardware was detected, but this container cannot access the AMD GPU devices."
+            show_amd_gpu_device_guidance "$_amd_missing_devices"
+            apply_cpu_gpu_fallback "Falling back to CPU mode because AMD GPU passthrough is unavailable in this container."
+        else
+            ai_warn "AMD GPU runtime devices not ready yet: ${_amd_missing_devices:-unknown}"
+            ai "Continuing for now; AMD tuning will try to load kernel modules before services start."
+        fi
+    fi
 fi
 
 BACKEND_ID="$GPU_BACKEND"
@@ -155,7 +185,7 @@ fi
 #-----------------------------------------------------------------------------
 # If detect_gpu found no working GPU, check if it's a fixable driver/Secure Boot issue
 # (Only for NVIDIA — AMD APU is handled above)
-if [[ $GPU_COUNT -eq 0 && "$GPU_BACKEND" != "amd" ]] && ! $DRY_RUN; then
+if [[ "${GPU_BACKEND_FORCED_CPU:-false}" != "true" && $GPU_COUNT -eq 0 && "$GPU_BACKEND" != "amd" ]] && ! $DRY_RUN; then
     fix_nvidia_secure_boot || true
 fi
 

dream-server/installers/phases/11-services.sh

@@ -25,6 +25,58 @@ if $DRY_RUN; then
 else
     cd "$INSTALL_DIR" || exit 1
 
+    _phase11_env_set() {
+        local key="$1" value="$2" env_file="$INSTALL_DIR/.env" tmp_file
+        [[ -f "$env_file" ]] || return 0
+        tmp_file="${env_file}.tmp.$$"
+        awk -v k="$key" -v v="$value" '
+            BEGIN { found = 0 }
+            index($0, k "=") == 1 { print k "=" v; found = 1; next }
+            { print }
+            END { if (!found) print k "=" v }
+        ' "$env_file" > "$tmp_file" && cat "$tmp_file" > "$env_file" && rm -f "$tmp_file"
+    }
+
+    _phase11_apply_cpu_fallback() {
+        local missing="$1"
+        show_amd_gpu_device_guidance "$missing"
+        apply_cpu_gpu_fallback "Falling back to CPU mode before launching services."
+
+        if [[ "${TIER_FORCED:-false}" != "true" ]]; then
+            TIER="$(select_cpu_fallback_tier "${RAM_GB:-0}")"
+            log "CPU fallback tier selected: $TIER"
+        fi
+
+        load_backend_contract "cpu" || true
+        LLM_HEALTHCHECK_URL="${BACKEND_PUBLIC_HEALTH_URL:-http://localhost:8080/health}"
+        LLM_PUBLIC_API_PORT="${BACKEND_PUBLIC_API_PORT:-8080}"
+        OPENCLAW_PROVIDER_NAME_DEFAULT="${BACKEND_PROVIDER_NAME:-local-llama}"
+        OPENCLAW_PROVIDER_URL_DEFAULT="${BACKEND_PROVIDER_URL:-http://llama-server:8080/v1}"
+        resolve_tier_config
+        GPU_BACKEND="cpu"
+
+        _phase11_env_set GPU_BACKEND "$GPU_BACKEND"
+        _phase11_env_set DREAM_MODE "local"
+        _phase11_env_set LLM_API_URL "http://llama-server:8080"
+        _phase11_env_set LLM_MODEL "$LLM_MODEL"
+        _phase11_env_set GGUF_FILE "$GGUF_FILE"
+        _phase11_env_set MAX_CONTEXT "$MAX_CONTEXT"
+        _phase11_env_set CTX_SIZE "$MAX_CONTEXT"
+        _phase11_env_set AUDIO_STT_MODEL "Systran/faster-whisper-base"
+        _phase11_env_set LLAMA_SERVER_IMAGE "${LLAMA_SERVER_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-b8248}"
+        ai_ok "Rewrote .env for CPU fallback"
+    }
+
+    if [[ "${GPU_BACKEND:-}" == "amd" ]] && ! amd_gpu_runtime_devices_available; then
+        _amd_missing_devices="$(amd_gpu_missing_devices_csv)"
+        if [[ "${GPU_BACKEND_FORCED:-false}" == "true" ]]; then
+            ai_bad "GPU_BACKEND=amd was explicitly requested, but required AMD device nodes are missing."
+            show_amd_gpu_device_guidance "$_amd_missing_devices"
+            exit 1
+        fi
+        _phase11_apply_cpu_fallback "$_amd_missing_devices"
+    fi
+
     # Re-resolve compose flags against the actual install directory.
     # Phase 03 may have disabled services (e.g., ComfyUI on Tier 0) after
     # COMPOSE_FLAGS was first set in Phase 02, making the cached value stale.