fix(cookbook): prefer native llama-server on local Windows

Author: zoomdbz

routes/cookbook_helpers.py

@@ -559,7 +559,7 @@ def _bash_squote(v: str) -> str:
 # Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve.
 # Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper.
 _SERVE_CMD_ALLOWLIST = {
-    "vllm", "llama-server", "llama_server", "llama.cpp", "ollama",
+    "vllm", "llama-server", "llama-server.exe", "llama_server", "llama.cpp", "ollama",
     "python", "python3",
     "sglang", "lmdeploy",
     "node", "npx",

routes/cookbook_routes.py

@@ -73,6 +73,9 @@ def _mask_secret(value: str) -> str:
             return "stored"
         return f"{value[:4]}...{value[-4:]}"
 
+    def _client_host_platform() -> str:
+        return "windows" if IS_WINDOWS else ""
+
     def _decrypt_secret(value: str | None) -> str:
         if not value:
             return ""
@@ -226,11 +229,15 @@ def _state_for_client(state):
         """Return cookbook state without raw secrets for browser clients."""
         _strip_task_secrets(state)
         env = state.get("env") if isinstance(state, dict) else None
+        if isinstance(state, dict) and not isinstance(env, dict):
+            env = {}
+            state["env"] = env
         if isinstance(env, dict):
             token = _decrypt_secret(env.get("hfToken"))
             env.pop("hfToken", None)
             env["hfTokenConfigured"] = bool(token)
             env["hfTokenMasked"] = _mask_secret(token)
+            env["hostPlatform"] = _client_host_platform()
         return state
 
     def _state_for_storage(state, on_disk=None):
@@ -249,6 +256,7 @@ def _state_for_storage(state, on_disk=None):
                 env.pop("hfToken", None)
             env.pop("hfTokenMasked", None)
             env.pop("hfTokenConfigured", None)
+            env.pop("hostPlatform", None)
         return state
 
     def _load_stored_hf_token() -> str:
@@ -1387,7 +1395,8 @@ async def model_serve(request: Request, req: ServeRequest):
             runner_lines.append(_HF_TOKEN_STATUS_SNIPPET)
             handled_ollama_serve = False
             # Auto-install inference engine if missing
-            if "llama_cpp" in req.cmd or "llama-server" in req.cmd:
+            local_windows_llama_cmd = local_windows and ("llama_cpp" in req.cmd or "llama-server" in req.cmd)
+            if ("llama_cpp" in req.cmd or "llama-server" in req.cmd) and not local_windows_llama_cmd:
                 # Prefer the NATIVE llama-server binary — its minja templating
                 # renders modern GGUF chat templates that the Python bindings'
                 # Jinja2 rejects (do_tojson ensure_ascii). Build it once from
@@ -2110,8 +2119,8 @@ async def get_cookbook_state(request: Request):
             try:
                 return _state_for_client(json.loads(_cookbook_state_path.read_text(encoding="utf-8")))
             except Exception:
-                return {}
-        return {}
+                return _state_for_client({})
+        return _state_for_client({})
 
     @router.post("/api/cookbook/state")
     async def save_cookbook_state(request: Request):

static/js/cookbook.js

@@ -73,7 +73,7 @@ function _platformIcon(platform) {
   return '';
 }
 
-export let _envState = { env: 'none', envPath: '', hfToken: '', hfTokenConfigured: false, hfTokenMasked: '', gpus: '', remoteHost: '', servers: [], modelPaths: [], platform: '', defaultServer: '' };
+export let _envState = { env: 'none', envPath: '', hfToken: '', hfTokenConfigured: false, hfTokenMasked: '', gpus: '', remoteHost: '', servers: [], modelPaths: [], platform: '', hostPlatform: '', defaultServer: '' };
 let _lastCacheHostVal = null;
 let _cookbookOpeningSpinners = [];
 export function _lastCacheHost() { return _lastCacheHostVal; }
@@ -210,8 +210,13 @@ function _getPort(hostOrTask) {
 
 /** Get platform for a given host (or task object). Returns 'windows', 'termux', 'linux', or '' */
 export function _getPlatform(hostOrTask) {
-  if (!hostOrTask) return _envState.platform || '';
-  if (typeof hostOrTask === 'object') return hostOrTask.platform || _getPlatform(hostOrTask.remoteServerKey || hostOrTask.remoteHost);
+  if (hostOrTask === 'local') return _envState.hostPlatform || '';
+  if (!hostOrTask) return _envState.remoteHost ? (_envState.platform || '') : (_envState.hostPlatform || '');
+  if (typeof hostOrTask === 'object') {
+    const taskHost = hostOrTask.remoteServerKey || hostOrTask.remoteHost || '';
+    if (!taskHost || taskHost === 'local') return _envState.hostPlatform || '';
+    return hostOrTask.platform || _getPlatform(taskHost);
+  }
   const selected = hostOrTask === _envState.remoteHost ? _selectedServer() : null;
   const srv = selected || _serverByVal(hostOrTask);
   return srv?.platform || '';
@@ -532,25 +537,27 @@ export function _buildServeCmd(f, modelName, backend) {
     // GPU list — read from gpus (button strip); fall back to gpu_id for
     // backward-compat with older saved presets that pre-date the removal.
     const gpuId = (f.gpus || f.gpu_id || '').toString().trim();
-    const py = _isWindows() ? 'python' : 'python3';
+    const _isWin = _isWindows();
+    const _localWindows = _isWin && !_envState.remoteHost;
+    const py = _isWin ? 'python' : 'python3';
     // CPU-only serve (-ngl 0): drop the GPU-only flags, otherwise the command
     // mixes "zero GPU layers" with CUDA unified-memory + flash-attn and fails to
     // start (issue #1291). Only affects the ngl=0 path; GPU serving is unchanged.
     const _cpuOnly = String(f.ngl).trim() === '0';
     const lcPrefix = (() => {
       let p = '';
-      if (f.unified_mem && !_cpuOnly && !_isWindows()) p += `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 `;
-      if (gpuId && !_isWindows()) p += `CUDA_VISIBLE_DEVICES=${gpuId} `;
+      if (f.unified_mem && !_cpuOnly && (!_isWin || _localWindows)) p += `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1 `;
+      if (gpuId && (!_isWin || _localWindows)) p += `CUDA_VISIBLE_DEVICES=${gpuId} `;
       return p;
     })();
-    if (f.unified_mem && !_cpuOnly && _isWindows()) cmd += `$env:GGML_CUDA_ENABLE_UNIFIED_MEMORY="1"; `;
-    if (gpuId && _isWindows()) cmd += `$env:CUDA_VISIBLE_DEVICES="${gpuId}"; `;
-    if (!_isWindows()) {
+    if (f.unified_mem && !_cpuOnly && _isWin && !_localWindows) cmd += `$env:GGML_CUDA_ENABLE_UNIFIED_MEMORY="1"; `;
+    if (gpuId && _isWin && !_localWindows) cmd += `$env:CUDA_VISIBLE_DEVICES="${gpuId}"; `;
+    if (!_isWin) {
       // Resolve GGUF path once, fail loudly if nothing matched (prevents
       // `--model ""` which causes confusing downstream errors).
       cmd += `MODEL_FILE=${ggufPath} && { [ -n "$MODEL_FILE" ] && [ -f "$MODEL_FILE" ]; } || { echo "ERROR: No GGUF found on this host. Either download the model here, or switch to the server where it's cached."; exit 1; } && `;
     }
-    const modelArg = _isWindows() ? `"${ggufPath}"` : `"$MODEL_FILE"`;
+    const modelArg = _isWin ? `"${ggufPath}"` : `"$MODEL_FILE"`;
     // Prefer the native llama-server binary on Linux — its minja templating
     // renders modern GGUF chat templates that the Python bindings' Jinja2
     // rejects (do_tojson ensure_ascii). Fall back to llama_cpp.server.
@@ -613,11 +620,16 @@ export function _buildServeCmd(f, modelName, backend) {
       // llama-cpp-python takes the projector via --clip_model_path.
       _lcpExtra += ` --clip_model_path "${f._mmproj_path}"`;
     }
+    const _lcServer = `${lcPrefix}llama-server --model ${modelArg} --host 0.0.0.0 --port ${f.port || '8080'} -ngl ${f.ngl || '99'} -c ${f.ctx || '8192'}${_lcExtra}`;
     const _lcpServer = `${lcPrefix}${py} -m llama_cpp.server --model ${modelArg} --host 0.0.0.0 --port ${f.port || '8080'} --n_gpu_layers ${f.ngl || '99'} --n_ctx ${f.ctx || '8192'}${_lcpExtra}`;
-    if (_isWindows()) {
+    if (_localWindows) {
+      // Local Windows serve is launched through Git Bash, so use the native
+      // llama-server shape and let PATH resolve the CUDA Release wrapper.
+      cmd += _lcServer;
+    } else if (_isWin) {
       cmd += _lcpServer;
     } else {
-      cmd += `${lcPrefix}llama-server --model ${modelArg} --host 0.0.0.0 --port ${f.port || '8080'} -ngl ${f.ngl || '99'} -c ${f.ctx || '8192'}${_lcExtra}`;
+      cmd += _lcServer;
       cmd += ` || ${_lcpServer}`;
     }
   } else if (backend === 'ollama') {
@@ -2099,13 +2111,14 @@ function _renderRecipes() {
     const isLocal = !s.host || s.host.toLowerCase() === 'local';
     if (isLocal) {
       s.host = '';
+      s.platform = _envState.hostPlatform || _envState.platform || '';
       if (_localSeen) return false;
       _localSeen = true;
     }
     return true;
   });
   if (!_localSeen) {
-    _es.servers.unshift({ host: '', env: _es.env || 'none', envPath: _es.envPath || '', modelDir: '~/.cache/huggingface/hub' });
+    _es.servers.unshift({ host: '', env: _es.env || 'none', envPath: _es.envPath || '', modelDir: '~/.cache/huggingface/hub', platform: _envState.hostPlatform || _envState.platform || '' });
   }
   if (_es.remoteHost && !_es.servers.some(s => s.host === _es.remoteHost)) {
     _es.servers.push({ host: _es.remoteHost, env: _es.env || 'none', envPath: _es.envPath || '', modelDir: '~/.cache/huggingface/hub' });

static/js/cookbookRunning.js

@@ -667,6 +667,7 @@ function _stripStateSecrets(state) {
   if (safe.env && typeof safe.env === 'object') {
     const { hfToken, ...env } = safe.env;
     if (hfToken) env.hfToken = hfToken;
+    delete env.hostPlatform;
     safe.env = env;
   }
   if (Array.isArray(safe.tasks)) safe.tasks = safe.tasks.map(_stripTaskSecrets);

tests/test_cookbook_cpu_only_serve.py

@@ -1,4 +1,4 @@
-"""Regression guard for issue #1291 — CPU-only serve still emitted GPU-only flags.
+"""Regression guard for issue #1291 - CPU-only serve still emitted GPU-only flags.
 
 The llama.cpp serve command builder (static/js/cookbook.js) added
 `--flash-attn on` and exported `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1` from
@@ -16,6 +16,7 @@
 
 SRC = Path(__file__).resolve().parent.parent / "static/js/cookbook.js"
 SERVE_SRC = Path(__file__).resolve().parent.parent / "static/js/cookbookServe.js"
+ROOT = SRC.parent.parent.parent
 
 
 def test_cpu_only_drops_gpu_only_flags():
@@ -51,3 +52,39 @@ def test_windows_diffusers_uses_python_not_python3():
     assert "const diffusersPy = _isWindows() ? 'python' : _py3Bin;" in text
     assert "cmd += `${diffusersPy} scripts/diffusion_server.py" in text
     assert "cmd += `python3 scripts/diffusion_server.py" not in text
+
+
+def test_local_windows_platform_comes_from_backend_host_state():
+    text = SRC.read_text(encoding="utf-8")
+    routes = (ROOT / "routes/cookbook_routes.py").read_text(encoding="utf-8")
+    running = (SRC.parent / "cookbookRunning.js").read_text(encoding="utf-8")
+
+    assert "hostPlatform" in text
+    assert "navigator.platform" not in text
+    assert "hostOrTask === 'local'" in text
+    assert "if (hostOrTask === 'local') return _envState.hostPlatform || '';" in text
+    assert "return _envState.hostPlatform || _envState.platform || ''" not in text
+    assert 'return "windows" if IS_WINDOWS else ""' in routes
+    assert 'env["hostPlatform"] = _client_host_platform()' in routes
+    assert "return _state_for_client({})" in routes
+    assert 'env.pop("hostPlatform", None)' in routes
+    assert "delete env.hostPlatform;" in running
+
+
+def test_local_windows_llamacpp_prefers_native_llama_server():
+    text = SRC.read_text(encoding="utf-8")
+    helpers = (ROOT / "routes/cookbook_helpers.py").read_text(encoding="utf-8")
+
+    assert "const _localWindows = _isWin && !_envState.remoteHost;" in text
+    assert "const gpuId = (f.gpus || f.gpu_id || '').toString().trim();" in text
+    assert "const _lcServer = `${lcPrefix}llama-server --model" in text
+    assert "if (_localWindows) {" in text
+    assert "cmd += _lcServer;" in text
+    assert '"llama-server.exe"' in helpers
+
+
+def test_local_windows_llama_server_skips_source_bootstrap():
+    routes = (ROOT / "routes/cookbook_routes.py").read_text(encoding="utf-8")
+
+    assert 'local_windows_llama_cmd = local_windows and ("llama_cpp" in req.cmd or "llama-server" in req.cmd)' in routes
+    assert 'if ("llama_cpp" in req.cmd or "llama-server" in req.cmd) and not local_windows_llama_cmd:' in routes