From 547edabe325b4d76257f8890f1703830822c2626 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 08:34:18 +0300 Subject: [PATCH 01/23] =?UTF-8?q?spec:=20lobes=20never=20advertises=20a=20?= =?UTF-8?q?capability=20it=20cannot=20serve=20(#92=20=C2=B7=20#91=20=C2=B7?= =?UTF-8?q?=20#74=20=C2=B7=20#69)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Unifies five open items into one invariant — advertised implies reachable — after live investigation on the DGX Spark rig proved the issues describe the symptoms correctly but the causes wrongly: * #92 is not a regression of #87. `reachable_origin` shipped in 0.38.0 (PR #90); the rig runs a gateway image built 2026-07-03 carrying lobes 0.36.0, so the fix was never deployed and nothing detects the skew. Host :8000 is not dead either — it is an unrelated uvicorn service (reachy-mini-dae). * #91 is not a backend-reload window. `handle_post` rewrites the model id once, before the failover loop, while `order_backends` offers every same-task generate backend as a failover target. Cortex 5xx (EngineDeadError, 04:08:34) -> the same body is retried against Gemma -> Gemma correctly 404s -> the "4xx = client error, no failover" rule relays it as terminal. The primary container's own logs show the symmetric case at 04:34:01 and 04:39:01. * Two unfiled bugs of the same shape: AUDIO_URL never reaches the gateway container (stt/tts advertise ready=true on a path that 404s), and /v1/models advertises phantom backends (6 models, 4 containers). Decisions recorded in the frame: no cross-backend failover at all; readiness becomes a background cached probe; phantom backends stopped by both a config gate and the readiness filter; #69's DSpark criterion closed answered-negative. Evidence: coolthor/gemma-4-12B-it-NVFP4A16 genuinely perceives images (red -> "Red", blue -> "Blue", ground-truth checked). Audio perception is blocked by the AUDIO_URL bug and is gated behind its fix. Frame: .devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- .devague/current | 2 +- ...dvertises-a-capability-it-cannot-serv.json | 451 ++++++++++++++++++ ...-advertises-a-capability-it-cannot-serv.md | 81 ++++ 3 files changed, 533 insertions(+), 1 deletion(-) create mode 100644 .devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json create mode 100644 docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md diff --git a/.devague/current b/.devague/current index c10e082..6fb4fdf 100644 --- a/.devague/current +++ b/.devague/current @@ -1 +1 @@ -lobes-preserves-qwen-thinking-traces-across-multi +lobes-never-advertises-a-capability-it-cannot-serv diff --git a/.devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json b/.devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json new file mode 100644 index 0000000..8fb56c4 --- /dev/null +++ b/.devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json @@ -0,0 +1,451 @@ +{ + "slug": "lobes-never-advertises-a-capability-it-cannot-serv", + "title": "lobes never advertises a capability it cannot serve: every endpoint, model and role in the machine-readable contract is proven reachable, and a fleet fault degrades to a retryable 503 instead of a terminal 404", + "schema_version": 1, + "status": "exported", + "created": "2026-07-09T05:14:35Z", + "updated": "2026-07-09T05:33:40Z", + "claims": [ + { + "id": "c1", + "kind": "announcement", + "text": "lobes never advertises a capability it cannot serve: every endpoint, model and role in the machine-readable contract is proven reachable, and a fleet fault degrades to a retryable 503 instead of a terminal 404", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h1", + "text": "On the live rig, for EVERY role in the contract, the advertised endpoint+path answers a real request with a non-404 status; and no model listed by GET /v1/models returns a 404 'does not exist' from POST /v1/chat/completions on the same origin.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c2", + "kind": "audience", + "text": "Programmatic consumers of the lobes contract \u2014 Colleague's resolve_role_base_url dialing GET /capabilities, and any long-running agent loop driving /v1/chat/completions through the gateway \u2014 plus the lobes operator who runs the fleet on the DGX Spark.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h2", + "text": "Colleague's resolve_role_base_url, given ONLY GET /capabilities and no COLLEAGUE_*_BASE_URL override, reaches cortex and senses and gets an answer \u2014 the scenario that failed on 2026-07-06 and again on 2026-07-09.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c3", + "kind": "before_state", + "text": "lobes advertises CONFIGURATION and calls it REACHABILITY, and nothing ever dials what it advertises. GET /capabilities returns endpoint=http://localhost:8000 with ready=true for every role while the gateway is published on :8001 and :8000 is an unrelated uvicorn service (reachy-mini-dae) answering {\"detail\":\"Not Found\"}. GET /v1/models lists 6 models while only 4 vLLM containers run. RoleInfo.ready is a config fact (==loaded), not a probe \u2014 its own docstring says so. And the deployed gateway silently runs lobes 0.36.0 while main is 0.39.0, so the #87 fix that shipped in 0.38.0 is not even in the running image.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h3", + "text": "A test asserts that the origin the gateway advertises never equals its own internal listen port when a published port differs, and that GET /capabilities and 'lobes capabilities --json' return byte-identical endpoint values for the same deployment.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c4", + "kind": "before_state", + "text": "A fleet fault is laundered into a terminal client error. handle_post rewrites the model id ONCE before the failover loop; order_backends offers every same-task generate backend as a failover candidate ([primary, multimodal, multimodal-coder, middle] for cortex). When the cortex vLLM returns >=500 (EngineDeadError, observed 2026-07-09 04:08:34) the gateway retries the same body \u2014 still naming the Qwen model \u2014 against the Gemma backend, which correctly 404s, and '4xx -> commit to this backend, no failover' relays that terminal 404 to the client. The primary container's logs show the symmetric case at 04:34:01 and 04:39:01: 'The model coolthor/gemma-4-12B-it-NVFP4A16 does not exist.' logged BY the Qwen container.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h4", + "text": "With the cortex backend forced to 5xx or made unreachable, a request naming the cortex model NEVER reaches the Gemma backend, and the client receives 503 + Retry-After \u2014 reproducible in a unit test with a fake fleet, not just observed in production logs.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c5", + "kind": "why_it_matters", + "text": "A single unlucky turn kills a whole agent loop. A terminal 404 tells a well-behaved client the model will NEVER exist, so colleague work items abort after N steps with status=error and no output \u2014 while the model is listed in /v1/models before and after. And a lobes-discovered senses endpoint degrades instantly (~0.002s per call) because the advertised origin is dead. This has now regressed three times across #87 -> #90 -> #92: the shape keeps breaking because no test ever dials the deployed artifact.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h5", + "text": "A multi-step client loop survives an owner-backend fault: the turn that would previously have died on a terminal 404 instead receives a retryable status, and a client with ordinary backoff completes the run.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c6", + "kind": "after_state", + "text": "Every advertised capability is proven reachable before it is advertised, and every fleet fault surfaces as a retryable status. /capabilities advertises an origin derived from configured truth (never the gateway's internal listen port) and a ready flag backed by a live backend probe. /v1/models lists only backends that actually answer. A request naming a model that /v1/models lists never receives a 404 'does not exist' from the same origin \u2014 an owner that is dead yields 503 + Retry-After. The CLI and the gateway agree on the endpoint byte-for-byte. And an executable verb dials every advertised role endpoint+path so 'advertised implies reachable' is a test, not a promise.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h6", + "text": "Every claim in the after_state is asserted by an executable check, and each of those checks FAILS when run against today's HEAD + today's deployed rig.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c7", + "kind": "before_state", + "text": "The same disease reaches the roles themselves. 'senses' advertises responsibilities=[intake, normalize_input, ...] and the catalog claims image+text and audio+text, but every probe lobes owns is text-only: roles_measure.py:183 sends 'Write a short paragraph.' to senses, and assess.py builds no multimodal content parts at all. The image/audio probes exist only in tests/test_smoke_duo.py:287-347, skipped unless LOBES_SMOKE_BASE_URL is set, with no recorded run. The image+text/audio+text evidence in catalog.py:282 belongs to the sakamakismile coder checkpoint; docs/gemma-4-12b-nvfp4.md:279 states outright that those checks were 'not independently re-run against the base [coolthor] checkpoint specifically'. coolthor became the default in 0.35.0 on MTP decode throughput \u2014 orthogonal to the image+audio precondition #74 set.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h7", + "text": "lobes' own tooling sends a real image and a real audio clip to the senses role and verifies the answer against known ground truth (not merely that the response was 200 with non-empty content), and the recorded evidence names the coolthor checkpoint.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c8", + "kind": "boundary", + "text": "NOT a service mesh, a load balancer, or a health-check-per-request proxy: the gateway stays a stdlib reverse proxy with a bounded, cached readiness signal \u2014 no per-request upstream probe on the hot path. NOT a claim about answer quality: lobes keeps emitting runtime metrics only, never task-quality claims (the #81 h15 honesty condition still holds). NOT a rewrite of the role vocabulary, the tier aliases, or the compose topology. NOT an attempt to keep cortex alive through EngineDeadError \u2014 recovering the vLLM engine is out of scope; only the gateway's HONESTY about a dead engine is in scope.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h8", + "text": "The gateway's POST hot path opens no additional upstream probe connection per request: the readiness signal is read from a cache, exactly as pressure already is.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c9", + "kind": "non_goal", + "text": "Not adding retry/backoff logic inside the gateway on the client's behalf. The gateway's job is to return an honest, correctly-classified status (503 + Retry-After) and let the client's own backoff policy decide. Queuing a request until a backend finishes warming (issue #91's option 2) is explicitly NOT chosen.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c10", + "kind": "success_signal", + "text": "Five falsifiable checks, each runnable: (1) curl GET :8001/capabilities | jq '.[].endpoint' returns the SAME origin as 'lobes capabilities --json' for the same deployment, and that origin answers 200 on its advertised path. (2) With the cortex container stopped, POST /v1/chat/completions model= returns 503 + Retry-After \u2014 never 404, never a Gemma answer. (3) GET /v1/models lists exactly the models whose backends answer. (4) 'lobes doctor' (or the new verify verb) exits non-zero when the deployed gateway's lobes version differs from the CLI's. (5) The live smoke layer posts a real image and a real audio clip at coolthor and records the pass in docs, closing #74 and #69's criterion 7 with evidence rather than assertion.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h9", + "text": "Each of the five success checks is a command a reviewer can paste and watch fail on main, then pass on the branch.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c11", + "kind": "requirement", + "text": "The gateway advertises an origin from CONFIGURED TRUTH, never inference: the fleet compose injects GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) into the gateway container, and reachable_origin prefers it over the request Host header. When neither configured truth nor a Host header is available the gateway MUST NOT fabricate an absolute URL from its internal listen port.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h10", + "text": "With GATEWAY_PUBLIC_URL unset AND the request carrying no Host header, the gateway emits an endpoint that is either absent/empty or explicitly relative \u2014 never an absolute URL built from GATEWAY_PORT.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c12", + "kind": "requirement", + "text": "A model id returned by GET /v1/models never receives a 404 'model does not exist' from POST /v1/chat/completions on the same origin. This is the single invariant issue #91 asks for, stated as a test.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h11", + "text": "A test enumerates GET /v1/models and, for each id, POSTs a minimal completion and asserts the status is never 404 \u2014 run against a fake fleet where the owner backend is dead.", + "status": "rejected" + }, + { + "id": "h23", + "text": "The listed-never-404 invariant is tested at the RACE, not at rest: a fake fleet lists model M, the owner is then killed, and a completion naming M returns 503 + Retry-After \u2014 never 404. A second test asserts the converse: an id that was NEVER in /v1/models does not silently get served by the default backend under a different model's weights.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c13", + "kind": "requirement", + "text": "Failover never forwards a request to a backend that does not serve the resolved model. The failover equivalence class is the SERVED MODEL, not the task family. Cross-backend retry stays legal only when the caller addressed a capability TIER ALIAS (main/minor/multimodal/cheap/normal/hard), in which case the body is re-resolved and re-rewritten per attempt rather than once before the loop.", + "origin": "llm", + "status": "rejected", + "honesty_conditions": [ + { + "id": "h12", + "text": "order_backends(table, ) returns at most the backends that serve that exact model; the existing test_order_backends_generate_still_failovers_between_generate_backends is INVERTED to assert the new contract, and a tier-alias request re-rewrites the body per attempt.", + "status": "rejected" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c14", + "kind": "requirement", + "text": "When the owner backend of a requested model is dead, unreachable, or warming, and no legal failover exists, the gateway returns 503 + Retry-After with a distinguishable error type (e.g. type=backend_unavailable), never 404 and never 502-as-terminal.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h13", + "text": "A dead owner yields status 503 with a Retry-After header and an error type distinguishable from both 'model unknown' and 'all backends down', asserted in a unit test.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c15", + "kind": "requirement", + "text": "GET /v1/models and GET /capabilities reflect a bounded, cached LIVE readiness signal, not a config fact. RoleInfo.ready stops being an alias of loaded for the four gateway-fronted roles, exactly as it already stopped being one for stt/tts in #89. Phantom backends (wired by *_SERVED_NAME with no running container) never appear as ready.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h14", + "text": "A backend wired in config but with no listening container reports ready=false and is absent from GET /v1/models, without the gateway probing on the request hot path.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c16", + "kind": "requirement", + "text": "'advertised implies reachable' becomes executable: a read-only verb dials every role's advertised endpoint+path and reports pass/fail per role, and it is the thing CI and the operator run. It also compares the deployed gateway container's lobes.__version__ against the CLI's and fails on skew \u2014 the defect that made #92 look like a code regression when it was a stale 0.36.0 image.", + "origin": "llm", + "status": "rejected", + "honesty_conditions": [ + { + "id": "h15", + "text": "The verb exits non-zero on the CURRENT live rig (stale 0.36.0 gateway, :8000 endpoint, 404ing audio path) and exits zero after the fix is deployed.", + "status": "rejected" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c17", + "kind": "requirement", + "text": "The multimodal role is proven multimodal by lobes' own tooling: the live smoke layer (tests/test_smoke_duo.py) is executed against coolthor on the DGX Spark and its image+text and audio+text results are recorded in docs/gemma-4-12b-nvfp4.md, replacing the standing admission at line 279 that those checks were never re-run against the base checkpoint. This closes #74's unchecked box and #69's criterion 7 with evidence instead of assertion.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h16", + "text": "docs/gemma-4-12b-nvfp4.md no longer contains the admission that the content-correctness checks 'were not independently re-run against the base checkpoint specifically', because they were re-run and recorded.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c18", + "kind": "before_state", + "text": "NEITHER surface is authoritative, and they are wrong in OPPOSITE directions. For the generate roles the gateway JSON is wrong (endpoint = its internal :8000) and the CLI is right (:8001 from .env VLLM_PORT). For the audio roles the CLI is wrong (ready=true, loaded=true, endpoint=:8001/v1/audio/speech \u2014 read from .env AUDIO_URL) and the gateway is right (ready=false, loaded=false, endpoint=''). Dialing the CLI-advertised tts path returns 404 'audio endpoints are not configured on this deployment'. Root cause: AUDIO_URL reaches the gateway ONLY via docker-compose.audio.yml:144, so whether stt/tts work depends on whether the operator remembered the overlay -f flag \u2014 while the CLI reads the merged .env and assumes wired. This is #92's exact shape (ready=true on a 404 path) for the audio roles, it is present in main and in the packaged template, and no issue tracks it.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h19", + "text": "A test asserts that 'lobes capabilities' and GET /capabilities agree on ready/loaded/endpoint for ALL SIX roles \u2014 including stt/tts on a deployment where the audio overlay is not composed in \u2014 and that the audio roles never report ready=true while their advertised path returns 404.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c19", + "kind": "before_state", + "text": "GET /v1/models advertises phantom backends. _optional_backend wires a backend when *_BASE_URL OR *_SERVED_NAME is set, and falls back to a default_url naming a container that need not exist. On the live rig that yields 6 advertised models against 4 running vLLM containers: 'multimodal-coder' (http://vllm-multimodal-coder:8000) and 'middle' (http://vllm-middle:8000) have no container at all, yet they are listed to clients AND sit in the generate failover chain, which for the cortex model is [primary, multimodal, multimodal-coder, middle].", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h20", + "text": "On a deployment where MULTIMODAL_CODER_SERVED_NAME and MIDDLE_SERVED_NAME are set but their containers are absent, GET /v1/models returns exactly the ids whose backends answer, and those absent backends never appear in any failover chain.", + "status": "rejected" + }, + { + "id": "h24", + "text": "With MULTIMODAL_CODER_SERVED_NAME and MIDDLE_SERVED_NAME set but no BASE_URL and no container, build_config wires NEITHER backend, GET /v1/models returns exactly the four running ids, and resolve_model('nvidia/Qwen3-14B-NVFP4') does not resolve to the primary's weights.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c20", + "kind": "assumption", + "text": "The vLLM engine crash that triggers all this (vllm.v1.engine.exceptions.EngineDeadError on the cortex container, RestartCount=3) is a separate fault whose ROOT CAUSE is out of scope here. This spec makes lobes honest about a dead engine; it does not attempt to keep the engine alive. If EngineDeadError turns out to be caused by lobes' own serve flags (MTP + max_num_seqs), that is a distinct issue.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c21", + "kind": "decision", + "text": "The unifying repair is a single invariant with three enforcement points: (1) ONE origin resolver \u2014 configured truth (GATEWAY_PUBLIC_URL, injected from the published VLLM_PORT) beats Host header beats nothing-at-all; the internal listen port is never advertised. (2) ONE readiness source \u2014 a bounded cached probe, shared by /v1/models, /capabilities.ready and the failover order, so 'ready' means 'answered recently'. (3) ONE executable check \u2014 a read-only verb that dials every advertised endpoint+path and the deployed gateway's version, run by CI and the operator.", + "origin": "llm", + "status": "rejected", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c22", + "kind": "requirement", + "text": "A LOCAL live-test of the capabilities contract runs before every PR. It is not a CI job (CI has no GPU and no fleet); it is a local gate the developer runs against the running deployment, in the same live-gated seam tests/test_smoke_duo.py already uses (LOBES_SMOKE_BASE_URL). It must fail loudly rather than skip silently when the operator intended to run it, so that 'advertised implies reachable' is checked against the DEPLOYED artifact before a PR claims to have fixed it \u2014 the exact step whose absence let #87 regress into #92.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h17", + "text": "The local pre-PR live test, pointed at a deployment whose gateway is stale or whose audio overlay is unwired, FAILS \u2014 it does not skip, and it does not pass.", + "status": "confirmed" + }, + { + "id": "h18", + "text": "The gate is ONE trigger and then unattended: a single local command runs the whole capabilities live-test to a pass/fail verdict with no prompts, no manual steps, and no per-role babysitting \u2014 the developer types it, walks away, and reads the exit code.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c23", + "kind": "requirement", + "text": "NO cross-backend failover. Every request resolves to exactly ONE backend and is attempted only there. An explicit model id goes to its owner; a capability tier alias resolves once, up front, via the existing static tier_aliases upward fallback (a tier whose gear is not WIRED still maps to a higher rung at table-build time). There is no runtime retry against a second backend, so a caller who asked for cortex can never silently receive a Gemma answer. A dead/unreachable/warming owner yields 503 + Retry-After. order_backends collapses to a single-element list and rewrite_model stays outside any loop.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h21", + "text": "A unit test with a fake fleet proves that a request naming the cortex model, with the primary backend dead, NEVER opens a connection to the multimodal backend, and returns 503 + Retry-After. order_backends(table, served) returns exactly one backend for every input.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c24", + "kind": "decision", + "text": "Readiness becomes a bounded BACKGROUND cached probe, mirroring the existing PressureCache: each backend's /health is polled on an interval off the request path. The one cached signal feeds GET /v1/models, GET /capabilities .ready, and the wiring/advertisement decisions. The POST hot path opens no probe connection.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c25", + "kind": "decision", + "text": "Phantom backends are stopped by BOTH gates: (a) a backend is wired only when its *_BASE_URL is set \u2014 the 'or *_SERVED_NAME' clause is dropped, matching the MINOR_BASE_URL convention the fleet already documents; and (b) GET /v1/models is filtered by the live readiness signal, so a wired-but-dead backend is not advertised.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c26", + "kind": "decision", + "text": "Issue #69's 'disabled-by-default DSpark experiment entry' is closed as ANSWERED-NEGATIVE: no catalog entry is shipped for deepseek-ai/dspark_gemma4_12b_block7, because #75 proved Gemma4DSparkModel does not load on vLLM 0.23. The repair is documentary \u2014 docs/gemma4-mtp-draft.md still presents DSpark as 'the ONE route task t3 should wire next' with no superseded banner and must carry the invalidity finding.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c27", + "kind": "requirement", + "text": "The pre-PR gate detects DEPLOYED-ARTIFACT skew, not just source correctness: it compares the running gateway container's lobes.__version__ against the CLI's and fails on mismatch, and it dials every advertised role endpoint+path. This is the check whose absence let #87's fix ship in 0.38.0 while the rig kept running 0.36.0 \u2014 making #92 look like a code regression when the code was already correct and merely undeployed.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h22", + "text": "Run against the rig as it stands right now (gateway 0.36.0, CLI 0.39.0, endpoint :8000, /v1/audio/speech 404ing), the gate exits non-zero and names all three faults; after redeploy it exits zero.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c28", + "kind": "decision", + "text": "The unifying repair is one invariant with three enforcement points: (1) ONE origin resolver \u2014 configured truth (GATEWAY_PUBLIC_URL, injected from the published VLLM_PORT) beats the request Host header beats nothing; the internal listen port is never advertised. (2) ONE readiness source \u2014 a background cached probe shared by /v1/models and /capabilities.ready. (3) ONE executable check \u2014 a LOCAL, single-trigger, unattended pre-PR live test that dials every advertised endpoint+path and the deployed gateway's version.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + } + ], + "open_vagueness": [ + { + "id": "v1", + "text": "Should the gateway's cached readiness probe be a background thread (like PressureCache) or a lazy TTL-cached probe on the /v1/models + /capabilities read path? Background costs a thread and probes when idle; lazy adds latency to the first read after a TTL expiry. Both keep the hot POST path probe-free.", + "kind": "unknown_nonblocking", + "claim_id": null + }, + { + "id": "v2", + "text": "Whether requiring *_BASE_URL to wire a backend (dropping the 'OR *_SERVED_NAME' clause) breaks any existing deployment that set only the served-name var. The fleet template always sets both, but a hand-edited .env might not.", + "kind": "unknown_nonblocking", + "claim_id": null + }, + { + "id": "v3", + "text": "RESOLVED by user decision (see the DSpark decision claim): close #69's disabled-entry criterion as answered-negative and correct docs/gemma4-mtp-draft.md, which still presents DSpark as the next route to wire. Follow-up: revisit if a vLLM lands that registers Gemma4DSparkModel.", + "kind": "follow_up", + "claim_id": null + } + ] +} diff --git a/docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md b/docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md new file mode 100644 index 0000000..2dd91d0 --- /dev/null +++ b/docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md @@ -0,0 +1,81 @@ +# lobes never advertises a capability it cannot serve: every endpoint, model and role in the machine-readable contract is proven reachable, and a fleet fault degrades to a retryable 503 instead of a terminal 404 + +> lobes never advertises a capability it cannot serve: every endpoint, model and role in the machine-readable contract is proven reachable, and a fleet fault degrades to a retryable 503 instead of a terminal 404 + +## Audience + +- Programmatic consumers of the lobes contract — Colleague's resolve_role_base_url dialing GET /capabilities, and any long-running agent loop driving /v1/chat/completions through the gateway — plus the lobes operator who runs the fleet on the DGX Spark. + +## Before → After + +- Before: lobes advertises CONFIGURATION and calls it REACHABILITY, and nothing ever dials what it advertises. GET /capabilities returns endpoint=http://localhost:8000 with ready=true for every role while the gateway is published on :8001 and :8000 is an unrelated uvicorn service (reachy-mini-dae) answering {"detail":"Not Found"}. GET /v1/models lists 6 models while only 4 vLLM containers run. RoleInfo.ready is a config fact (==loaded), not a probe — its own docstring says so. And the deployed gateway silently runs lobes 0.36.0 while main is 0.39.0, so the #87 fix that shipped in 0.38.0 is not even in the running image. +- Before: A fleet fault is laundered into a terminal client error. handle_post rewrites the model id ONCE before the failover loop; order_backends offers every same-task generate backend as a failover candidate ([primary, multimodal, multimodal-coder, middle] for cortex). When the cortex vLLM returns >=500 (EngineDeadError, observed 2026-07-09 04:08:34) the gateway retries the same body — still naming the Qwen model — against the Gemma backend, which correctly 404s, and '4xx -> commit to this backend, no failover' relays that terminal 404 to the client. The primary container's logs show the symmetric case at 04:34:01 and 04:39:01: 'The model coolthor/gemma-4-12B-it-NVFP4A16 does not exist.' logged BY the Qwen container. +- Before: The same disease reaches the roles themselves. 'senses' advertises responsibilities=[intake, normalize_input, ...] and the catalog claims image+text and audio+text, but every probe lobes owns is text-only: roles_measure.py:183 sends 'Write a short paragraph.' to senses, and assess.py builds no multimodal content parts at all. The image/audio probes exist only in tests/test_smoke_duo.py:287-347, skipped unless LOBES_SMOKE_BASE_URL is set, with no recorded run. The image+text/audio+text evidence in catalog.py:282 belongs to the sakamakismile coder checkpoint; docs/gemma-4-12b-nvfp4.md:279 states outright that those checks were 'not independently re-run against the base [coolthor] checkpoint specifically'. coolthor became the default in 0.35.0 on MTP decode throughput — orthogonal to the image+audio precondition #74 set. +- Before: NEITHER surface is authoritative, and they are wrong in OPPOSITE directions. For the generate roles the gateway JSON is wrong (endpoint = its internal :8000) and the CLI is right (:8001 from .env VLLM_PORT). For the audio roles the CLI is wrong (ready=true, loaded=true, endpoint=:8001/v1/audio/speech — read from .env AUDIO_URL) and the gateway is right (ready=false, loaded=false, endpoint=''). Dialing the CLI-advertised tts path returns 404 'audio endpoints are not configured on this deployment'. Root cause: AUDIO_URL reaches the gateway ONLY via docker-compose.audio.yml:144, so whether stt/tts work depends on whether the operator remembered the overlay -f flag — while the CLI reads the merged .env and assumes wired. This is #92's exact shape (ready=true on a 404 path) for the audio roles, it is present in main and in the packaged template, and no issue tracks it. +- Before: GET /v1/models advertises phantom backends. _optional_backend wires a backend when *_BASE_URL OR *_SERVED_NAME is set, and falls back to a default_url naming a container that need not exist. On the live rig that yields 6 advertised models against 4 running vLLM containers: 'multimodal-coder' (http://vllm-multimodal-coder:8000) and 'middle' (http://vllm-middle:8000) have no container at all, yet they are listed to clients AND sit in the generate failover chain, which for the cortex model is [primary, multimodal, multimodal-coder, middle]. +- After: Every advertised capability is proven reachable before it is advertised, and every fleet fault surfaces as a retryable status. /capabilities advertises an origin derived from configured truth (never the gateway's internal listen port) and a ready flag backed by a live backend probe. /v1/models lists only backends that actually answer. A request naming a model that /v1/models lists never receives a 404 'does not exist' from the same origin — an owner that is dead yields 503 + Retry-After. The CLI and the gateway agree on the endpoint byte-for-byte. And an executable verb dials every advertised role endpoint+path so 'advertised implies reachable' is a test, not a promise. + +## Why it matters + +- A single unlucky turn kills a whole agent loop. A terminal 404 tells a well-behaved client the model will NEVER exist, so colleague work items abort after N steps with status=error and no output — while the model is listed in /v1/models before and after. And a lobes-discovered senses endpoint degrades instantly (~0.002s per call) because the advertised origin is dead. This has now regressed three times across #87 -> #90 -> #92: the shape keeps breaking because no test ever dials the deployed artifact. + +## Requirements + +- The gateway advertises an origin from CONFIGURED TRUTH, never inference: the fleet compose injects GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) into the gateway container, and reachable_origin prefers it over the request Host header. When neither configured truth nor a Host header is available the gateway MUST NOT fabricate an absolute URL from its internal listen port. + - honesty: With GATEWAY_PUBLIC_URL unset AND the request carrying no Host header, the gateway emits an endpoint that is either absent/empty or explicitly relative — never an absolute URL built from GATEWAY_PORT. +- A model id returned by GET /v1/models never receives a 404 'model does not exist' from POST /v1/chat/completions on the same origin. This is the single invariant issue #91 asks for, stated as a test. + - honesty: The listed-never-404 invariant is tested at the RACE, not at rest: a fake fleet lists model M, the owner is then killed, and a completion naming M returns 503 + Retry-After — never 404. A second test asserts the converse: an id that was NEVER in /v1/models does not silently get served by the default backend under a different model's weights. +- When the owner backend of a requested model is dead, unreachable, or warming, and no legal failover exists, the gateway returns 503 + Retry-After with a distinguishable error type (e.g. type=backend_unavailable), never 404 and never 502-as-terminal. + - honesty: A dead owner yields status 503 with a Retry-After header and an error type distinguishable from both 'model unknown' and 'all backends down', asserted in a unit test. +- GET /v1/models and GET /capabilities reflect a bounded, cached LIVE readiness signal, not a config fact. RoleInfo.ready stops being an alias of loaded for the four gateway-fronted roles, exactly as it already stopped being one for stt/tts in #89. Phantom backends (wired by *_SERVED_NAME with no running container) never appear as ready. + - honesty: A backend wired in config but with no listening container reports ready=false and is absent from GET /v1/models, without the gateway probing on the request hot path. +- The multimodal role is proven multimodal by lobes' own tooling: the live smoke layer (tests/test_smoke_duo.py) is executed against coolthor on the DGX Spark and its image+text and audio+text results are recorded in docs/gemma-4-12b-nvfp4.md, replacing the standing admission at line 279 that those checks were never re-run against the base checkpoint. This closes #74's unchecked box and #69's criterion 7 with evidence instead of assertion. + - honesty: docs/gemma-4-12b-nvfp4.md no longer contains the admission that the content-correctness checks 'were not independently re-run against the base checkpoint specifically', because they were re-run and recorded. +- A LOCAL live-test of the capabilities contract runs before every PR. It is not a CI job (CI has no GPU and no fleet); it is a local gate the developer runs against the running deployment, in the same live-gated seam tests/test_smoke_duo.py already uses (LOBES_SMOKE_BASE_URL). It must fail loudly rather than skip silently when the operator intended to run it, so that 'advertised implies reachable' is checked against the DEPLOYED artifact before a PR claims to have fixed it — the exact step whose absence let #87 regress into #92. + - honesty: The local pre-PR live test, pointed at a deployment whose gateway is stale or whose audio overlay is unwired, FAILS — it does not skip, and it does not pass. + - honesty: The gate is ONE trigger and then unattended: a single local command runs the whole capabilities live-test to a pass/fail verdict with no prompts, no manual steps, and no per-role babysitting — the developer types it, walks away, and reads the exit code. +- NO cross-backend failover. Every request resolves to exactly ONE backend and is attempted only there. An explicit model id goes to its owner; a capability tier alias resolves once, up front, via the existing static tier_aliases upward fallback (a tier whose gear is not WIRED still maps to a higher rung at table-build time). There is no runtime retry against a second backend, so a caller who asked for cortex can never silently receive a Gemma answer. A dead/unreachable/warming owner yields 503 + Retry-After. order_backends collapses to a single-element list and rewrite_model stays outside any loop. + - honesty: A unit test with a fake fleet proves that a request naming the cortex model, with the primary backend dead, NEVER opens a connection to the multimodal backend, and returns 503 + Retry-After. order_backends(table, served) returns exactly one backend for every input. +- The pre-PR gate detects DEPLOYED-ARTIFACT skew, not just source correctness: it compares the running gateway container's lobes.__version__ against the CLI's and fails on mismatch, and it dials every advertised role endpoint+path. This is the check whose absence let #87's fix ship in 0.38.0 while the rig kept running 0.36.0 — making #92 look like a code regression when the code was already correct and merely undeployed. + - honesty: Run against the rig as it stands right now (gateway 0.36.0, CLI 0.39.0, endpoint :8000, /v1/audio/speech 404ing), the gate exits non-zero and names all three faults; after redeploy it exits zero. + +## Honesty conditions + +- On the live rig, for EVERY role in the contract, the advertised endpoint+path answers a real request with a non-404 status; and no model listed by GET /v1/models returns a 404 'does not exist' from POST /v1/chat/completions on the same origin. +- Colleague's resolve_role_base_url, given ONLY GET /capabilities and no COLLEAGUE_*_BASE_URL override, reaches cortex and senses and gets an answer — the scenario that failed on 2026-07-06 and again on 2026-07-09. +- A test asserts that the origin the gateway advertises never equals its own internal listen port when a published port differs, and that GET /capabilities and 'lobes capabilities --json' return byte-identical endpoint values for the same deployment. +- With the cortex backend forced to 5xx or made unreachable, a request naming the cortex model NEVER reaches the Gemma backend, and the client receives 503 + Retry-After — reproducible in a unit test with a fake fleet, not just observed in production logs. +- A multi-step client loop survives an owner-backend fault: the turn that would previously have died on a terminal 404 instead receives a retryable status, and a client with ordinary backoff completes the run. +- Every claim in the after_state is asserted by an executable check, and each of those checks FAILS when run against today's HEAD + today's deployed rig. +- lobes' own tooling sends a real image and a real audio clip to the senses role and verifies the answer against known ground truth (not merely that the response was 200 with non-empty content), and the recorded evidence names the coolthor checkpoint. +- The gateway's POST hot path opens no additional upstream probe connection per request: the readiness signal is read from a cache, exactly as pressure already is. +- Each of the five success checks is a command a reviewer can paste and watch fail on main, then pass on the branch. +- A test asserts that 'lobes capabilities' and GET /capabilities agree on ready/loaded/endpoint for ALL SIX roles — including stt/tts on a deployment where the audio overlay is not composed in — and that the audio roles never report ready=true while their advertised path returns 404. +- With MULTIMODAL_CODER_SERVED_NAME and MIDDLE_SERVED_NAME set but no BASE_URL and no container, build_config wires NEITHER backend, GET /v1/models returns exactly the four running ids, and resolve_model('nvidia/Qwen3-14B-NVFP4') does not resolve to the primary's weights. + +## Success signals + +- Five falsifiable checks, each runnable: (1) curl GET :8001/capabilities | jq '.[].endpoint' returns the SAME origin as 'lobes capabilities --json' for the same deployment, and that origin answers 200 on its advertised path. (2) With the cortex container stopped, POST /v1/chat/completions model= returns 503 + Retry-After — never 404, never a Gemma answer. (3) GET /v1/models lists exactly the models whose backends answer. (4) 'lobes doctor' (or the new verify verb) exits non-zero when the deployed gateway's lobes version differs from the CLI's. (5) The live smoke layer posts a real image and a real audio clip at coolthor and records the pass in docs, closing #74 and #69's criterion 7 with evidence rather than assertion. + +## Scope / boundaries + +- NOT a service mesh, a load balancer, or a health-check-per-request proxy: the gateway stays a stdlib reverse proxy with a bounded, cached readiness signal — no per-request upstream probe on the hot path. NOT a claim about answer quality: lobes keeps emitting runtime metrics only, never task-quality claims (the #81 h15 honesty condition still holds). NOT a rewrite of the role vocabulary, the tier aliases, or the compose topology. NOT an attempt to keep cortex alive through EngineDeadError — recovering the vLLM engine is out of scope; only the gateway's HONESTY about a dead engine is in scope. + +## Non-goals + +- Not adding retry/backoff logic inside the gateway on the client's behalf. The gateway's job is to return an honest, correctly-classified status (503 + Retry-After) and let the client's own backoff policy decide. Queuing a request until a backend finishes warming (issue #91's option 2) is explicitly NOT chosen. + +## Assumptions + +- The vLLM engine crash that triggers all this (vllm.v1.engine.exceptions.EngineDeadError on the cortex container, RestartCount=3) is a separate fault whose ROOT CAUSE is out of scope here. This spec makes lobes honest about a dead engine; it does not attempt to keep the engine alive. If EngineDeadError turns out to be caused by lobes' own serve flags (MTP + max_num_seqs), that is a distinct issue. + +## Decisions + +- Readiness becomes a bounded BACKGROUND cached probe, mirroring the existing PressureCache: each backend's /health is polled on an interval off the request path. The one cached signal feeds GET /v1/models, GET /capabilities .ready, and the wiring/advertisement decisions. The POST hot path opens no probe connection. +- Phantom backends are stopped by BOTH gates: (a) a backend is wired only when its *_BASE_URL is set — the 'or *_SERVED_NAME' clause is dropped, matching the MINOR_BASE_URL convention the fleet already documents; and (b) GET /v1/models is filtered by the live readiness signal, so a wired-but-dead backend is not advertised. +- Issue #69's 'disabled-by-default DSpark experiment entry' is closed as ANSWERED-NEGATIVE: no catalog entry is shipped for deepseek-ai/dspark_gemma4_12b_block7, because #75 proved Gemma4DSparkModel does not load on vLLM 0.23. The repair is documentary — docs/gemma4-mtp-draft.md still presents DSpark as 'the ONE route task t3 should wire next' with no superseded banner and must carry the invalidity finding. +- The unifying repair is one invariant with three enforcement points: (1) ONE origin resolver — configured truth (GATEWAY_PUBLIC_URL, injected from the published VLLM_PORT) beats the request Host header beats nothing; the internal listen port is never advertised. (2) ONE readiness source — a background cached probe shared by /v1/models and /capabilities.ready. (3) ONE executable check — a LOCAL, single-trigger, unattended pre-PR live test that dials every advertised endpoint+path and the deployed gateway's version. + +## Open / follow-up + +- RESOLVED by user decision (see the DSpark decision claim): close #69's disabled-entry criterion as answered-negative and correct docs/gemma4-mtp-draft.md, which still presents DSpark as the next route to wire. Follow-up: revisit if a vLLM lands that registers Gemma4DSparkModel. From 2b733ff7277871cd4f21c50203cdf9747c9fe423 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 08:45:01 +0300 Subject: [PATCH 02/23] =?UTF-8?q?plan:=20advertised-implies-reachable=20?= =?UTF-8?q?=E2=80=94=2010=20tasks,=206=20waves,=2039/39=20targets=20(#92?= =?UTF-8?q?=20=C2=B7=20#91=20=C2=B7=20#74=20=C2=B7=20#69)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forward leg from the converged frame. Tasks are decomposed by FILE so each wave is operationally parallel, not merely formally so: wave 0 t1 _config.py (backend wiring gate) t2 _routing.py (no cross-backend failover) t3 _readiness.py (new, background probe cache) t4 fleet templates (GATEWAY_PUBLIC_URL + AUDIO_URL to the gateway) wave 1 t5 roles.py (ready decoupled from loaded; never advertise the internal port) wave 2 t6 server.py (503 + Retry-After, readiness wiring, reachable_origin) wave 3 t7 CLI (capabilities/gateway agreement, doctor version-skew) t8 senses perception probe (ground-truth image + audio) wave 4 t9 the local, single-trigger, unattended pre-PR live gate wave 5 t10 documentary repairs (gemma docs, DSpark banner, README quickstart) Risks recorded: *_BASE_URL gate may break a hand-edited .env (r1); background probe-thread lifecycle inside ThreadingHTTPServer (r2); Chatterbox's poisoned-CUDA history can flake the audio probe (r3); the live gate cannot run in CI, so nothing structurally forces it to run — exactly how #87's fix shipped in 0.38.0 while the rig kept running 0.36.0 (r4, follow-up). New issues filed from this investigation: #96 (AUDIO_URL never reaches the gateway), #97 (phantom backends in /v1/models), #98 (cortex EngineDeadError). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- .devague/current_plan | 2 +- ...dvertises-a-capability-it-cannot-serv.json | 454 ++++++++++++++++++ ...-advertises-a-capability-it-cannot-serv.md | 121 +++++ 3 files changed, 576 insertions(+), 1 deletion(-) create mode 100644 .devague/plans/lobes-never-advertises-a-capability-it-cannot-serv.json create mode 100644 docs/plans/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md diff --git a/.devague/current_plan b/.devague/current_plan index c10e082..6fb4fdf 100644 --- a/.devague/current_plan +++ b/.devague/current_plan @@ -1 +1 @@ -lobes-preserves-qwen-thinking-traces-across-multi +lobes-never-advertises-a-capability-it-cannot-serv diff --git a/.devague/plans/lobes-never-advertises-a-capability-it-cannot-serv.json b/.devague/plans/lobes-never-advertises-a-capability-it-cannot-serv.json new file mode 100644 index 0000000..adb8042 --- /dev/null +++ b/.devague/plans/lobes-never-advertises-a-capability-it-cannot-serv.json @@ -0,0 +1,454 @@ +{ + "slug": "lobes-never-advertises-a-capability-it-cannot-serv", + "title": "lobes never advertises a capability it cannot serve: every endpoint, model and role in the machine-readable contract is proven reachable, and a fleet fault degrades to a retryable 503 instead of a terminal 404", + "frame_slug": "lobes-never-advertises-a-capability-it-cannot-serv", + "schema_version": 1, + "status": "exported", + "created": "2026-07-09T05:37:28Z", + "updated": "2026-07-09T05:44:47Z", + "targets": [ + { + "id": "c1", + "kind": "announcement", + "text": "lobes never advertises a capability it cannot serve: every endpoint, model and role in the machine-readable contract is proven reachable, and a fleet fault degrades to a retryable 503 instead of a terminal 404" + }, + { + "id": "h1", + "kind": "honesty", + "text": "On the live rig, for EVERY role in the contract, the advertised endpoint+path answers a real request with a non-404 status; and no model listed by GET /v1/models returns a 404 'does not exist' from POST /v1/chat/completions on the same origin." + }, + { + "id": "c2", + "kind": "audience", + "text": "Programmatic consumers of the lobes contract \u2014 Colleague's resolve_role_base_url dialing GET /capabilities, and any long-running agent loop driving /v1/chat/completions through the gateway \u2014 plus the lobes operator who runs the fleet on the DGX Spark." + }, + { + "id": "h2", + "kind": "honesty", + "text": "Colleague's resolve_role_base_url, given ONLY GET /capabilities and no COLLEAGUE_*_BASE_URL override, reaches cortex and senses and gets an answer \u2014 the scenario that failed on 2026-07-06 and again on 2026-07-09." + }, + { + "id": "c3", + "kind": "before_state", + "text": "lobes advertises CONFIGURATION and calls it REACHABILITY, and nothing ever dials what it advertises. GET /capabilities returns endpoint=http://localhost:8000 with ready=true for every role while the gateway is published on :8001 and :8000 is an unrelated uvicorn service (reachy-mini-dae) answering {\"detail\":\"Not Found\"}. GET /v1/models lists 6 models while only 4 vLLM containers run. RoleInfo.ready is a config fact (==loaded), not a probe \u2014 its own docstring says so. And the deployed gateway silently runs lobes 0.36.0 while main is 0.39.0, so the #87 fix that shipped in 0.38.0 is not even in the running image." + }, + { + "id": "h3", + "kind": "honesty", + "text": "A test asserts that the origin the gateway advertises never equals its own internal listen port when a published port differs, and that GET /capabilities and 'lobes capabilities --json' return byte-identical endpoint values for the same deployment." + }, + { + "id": "c4", + "kind": "before_state", + "text": "A fleet fault is laundered into a terminal client error. handle_post rewrites the model id ONCE before the failover loop; order_backends offers every same-task generate backend as a failover candidate ([primary, multimodal, multimodal-coder, middle] for cortex). When the cortex vLLM returns >=500 (EngineDeadError, observed 2026-07-09 04:08:34) the gateway retries the same body \u2014 still naming the Qwen model \u2014 against the Gemma backend, which correctly 404s, and '4xx -> commit to this backend, no failover' relays that terminal 404 to the client. The primary container's logs show the symmetric case at 04:34:01 and 04:39:01: 'The model coolthor/gemma-4-12B-it-NVFP4A16 does not exist.' logged BY the Qwen container." + }, + { + "id": "h4", + "kind": "honesty", + "text": "With the cortex backend forced to 5xx or made unreachable, a request naming the cortex model NEVER reaches the Gemma backend, and the client receives 503 + Retry-After \u2014 reproducible in a unit test with a fake fleet, not just observed in production logs." + }, + { + "id": "c5", + "kind": "why_it_matters", + "text": "A single unlucky turn kills a whole agent loop. A terminal 404 tells a well-behaved client the model will NEVER exist, so colleague work items abort after N steps with status=error and no output \u2014 while the model is listed in /v1/models before and after. And a lobes-discovered senses endpoint degrades instantly (~0.002s per call) because the advertised origin is dead. This has now regressed three times across #87 -> #90 -> #92: the shape keeps breaking because no test ever dials the deployed artifact." + }, + { + "id": "h5", + "kind": "honesty", + "text": "A multi-step client loop survives an owner-backend fault: the turn that would previously have died on a terminal 404 instead receives a retryable status, and a client with ordinary backoff completes the run." + }, + { + "id": "c6", + "kind": "after_state", + "text": "Every advertised capability is proven reachable before it is advertised, and every fleet fault surfaces as a retryable status. /capabilities advertises an origin derived from configured truth (never the gateway's internal listen port) and a ready flag backed by a live backend probe. /v1/models lists only backends that actually answer. A request naming a model that /v1/models lists never receives a 404 'does not exist' from the same origin \u2014 an owner that is dead yields 503 + Retry-After. The CLI and the gateway agree on the endpoint byte-for-byte. And an executable verb dials every advertised role endpoint+path so 'advertised implies reachable' is a test, not a promise." + }, + { + "id": "h6", + "kind": "honesty", + "text": "Every claim in the after_state is asserted by an executable check, and each of those checks FAILS when run against today's HEAD + today's deployed rig." + }, + { + "id": "c7", + "kind": "before_state", + "text": "The same disease reaches the roles themselves. 'senses' advertises responsibilities=[intake, normalize_input, ...] and the catalog claims image+text and audio+text, but every probe lobes owns is text-only: roles_measure.py:183 sends 'Write a short paragraph.' to senses, and assess.py builds no multimodal content parts at all. The image/audio probes exist only in tests/test_smoke_duo.py:287-347, skipped unless LOBES_SMOKE_BASE_URL is set, with no recorded run. The image+text/audio+text evidence in catalog.py:282 belongs to the sakamakismile coder checkpoint; docs/gemma-4-12b-nvfp4.md:279 states outright that those checks were 'not independently re-run against the base [coolthor] checkpoint specifically'. coolthor became the default in 0.35.0 on MTP decode throughput \u2014 orthogonal to the image+audio precondition #74 set." + }, + { + "id": "h7", + "kind": "honesty", + "text": "lobes' own tooling sends a real image and a real audio clip to the senses role and verifies the answer against known ground truth (not merely that the response was 200 with non-empty content), and the recorded evidence names the coolthor checkpoint." + }, + { + "id": "c8", + "kind": "boundary", + "text": "NOT a service mesh, a load balancer, or a health-check-per-request proxy: the gateway stays a stdlib reverse proxy with a bounded, cached readiness signal \u2014 no per-request upstream probe on the hot path. NOT a claim about answer quality: lobes keeps emitting runtime metrics only, never task-quality claims (the #81 h15 honesty condition still holds). NOT a rewrite of the role vocabulary, the tier aliases, or the compose topology. NOT an attempt to keep cortex alive through EngineDeadError \u2014 recovering the vLLM engine is out of scope; only the gateway's HONESTY about a dead engine is in scope." + }, + { + "id": "h8", + "kind": "honesty", + "text": "The gateway's POST hot path opens no additional upstream probe connection per request: the readiness signal is read from a cache, exactly as pressure already is." + }, + { + "id": "c10", + "kind": "success_signal", + "text": "Five falsifiable checks, each runnable: (1) curl GET :8001/capabilities | jq '.[].endpoint' returns the SAME origin as 'lobes capabilities --json' for the same deployment, and that origin answers 200 on its advertised path. (2) With the cortex container stopped, POST /v1/chat/completions model= returns 503 + Retry-After \u2014 never 404, never a Gemma answer. (3) GET /v1/models lists exactly the models whose backends answer. (4) 'lobes doctor' (or the new verify verb) exits non-zero when the deployed gateway's lobes version differs from the CLI's. (5) The live smoke layer posts a real image and a real audio clip at coolthor and records the pass in docs, closing #74 and #69's criterion 7 with evidence rather than assertion." + }, + { + "id": "h9", + "kind": "honesty", + "text": "Each of the five success checks is a command a reviewer can paste and watch fail on main, then pass on the branch." + }, + { + "id": "c11", + "kind": "requirement", + "text": "The gateway advertises an origin from CONFIGURED TRUTH, never inference: the fleet compose injects GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) into the gateway container, and reachable_origin prefers it over the request Host header. When neither configured truth nor a Host header is available the gateway MUST NOT fabricate an absolute URL from its internal listen port." + }, + { + "id": "h10", + "kind": "honesty", + "text": "With GATEWAY_PUBLIC_URL unset AND the request carrying no Host header, the gateway emits an endpoint that is either absent/empty or explicitly relative \u2014 never an absolute URL built from GATEWAY_PORT." + }, + { + "id": "c12", + "kind": "requirement", + "text": "A model id returned by GET /v1/models never receives a 404 'model does not exist' from POST /v1/chat/completions on the same origin. This is the single invariant issue #91 asks for, stated as a test." + }, + { + "id": "h23", + "kind": "honesty", + "text": "The listed-never-404 invariant is tested at the RACE, not at rest: a fake fleet lists model M, the owner is then killed, and a completion naming M returns 503 + Retry-After \u2014 never 404. A second test asserts the converse: an id that was NEVER in /v1/models does not silently get served by the default backend under a different model's weights." + }, + { + "id": "c14", + "kind": "requirement", + "text": "When the owner backend of a requested model is dead, unreachable, or warming, and no legal failover exists, the gateway returns 503 + Retry-After with a distinguishable error type (e.g. type=backend_unavailable), never 404 and never 502-as-terminal." + }, + { + "id": "h13", + "kind": "honesty", + "text": "A dead owner yields status 503 with a Retry-After header and an error type distinguishable from both 'model unknown' and 'all backends down', asserted in a unit test." + }, + { + "id": "c15", + "kind": "requirement", + "text": "GET /v1/models and GET /capabilities reflect a bounded, cached LIVE readiness signal, not a config fact. RoleInfo.ready stops being an alias of loaded for the four gateway-fronted roles, exactly as it already stopped being one for stt/tts in #89. Phantom backends (wired by *_SERVED_NAME with no running container) never appear as ready." + }, + { + "id": "h14", + "kind": "honesty", + "text": "A backend wired in config but with no listening container reports ready=false and is absent from GET /v1/models, without the gateway probing on the request hot path." + }, + { + "id": "c17", + "kind": "requirement", + "text": "The multimodal role is proven multimodal by lobes' own tooling: the live smoke layer (tests/test_smoke_duo.py) is executed against coolthor on the DGX Spark and its image+text and audio+text results are recorded in docs/gemma-4-12b-nvfp4.md, replacing the standing admission at line 279 that those checks were never re-run against the base checkpoint. This closes #74's unchecked box and #69's criterion 7 with evidence instead of assertion." + }, + { + "id": "h16", + "kind": "honesty", + "text": "docs/gemma-4-12b-nvfp4.md no longer contains the admission that the content-correctness checks 'were not independently re-run against the base checkpoint specifically', because they were re-run and recorded." + }, + { + "id": "c18", + "kind": "before_state", + "text": "NEITHER surface is authoritative, and they are wrong in OPPOSITE directions. For the generate roles the gateway JSON is wrong (endpoint = its internal :8000) and the CLI is right (:8001 from .env VLLM_PORT). For the audio roles the CLI is wrong (ready=true, loaded=true, endpoint=:8001/v1/audio/speech \u2014 read from .env AUDIO_URL) and the gateway is right (ready=false, loaded=false, endpoint=''). Dialing the CLI-advertised tts path returns 404 'audio endpoints are not configured on this deployment'. Root cause: AUDIO_URL reaches the gateway ONLY via docker-compose.audio.yml:144, so whether stt/tts work depends on whether the operator remembered the overlay -f flag \u2014 while the CLI reads the merged .env and assumes wired. This is #92's exact shape (ready=true on a 404 path) for the audio roles, it is present in main and in the packaged template, and no issue tracks it." + }, + { + "id": "h19", + "kind": "honesty", + "text": "A test asserts that 'lobes capabilities' and GET /capabilities agree on ready/loaded/endpoint for ALL SIX roles \u2014 including stt/tts on a deployment where the audio overlay is not composed in \u2014 and that the audio roles never report ready=true while their advertised path returns 404." + }, + { + "id": "c19", + "kind": "before_state", + "text": "GET /v1/models advertises phantom backends. _optional_backend wires a backend when *_BASE_URL OR *_SERVED_NAME is set, and falls back to a default_url naming a container that need not exist. On the live rig that yields 6 advertised models against 4 running vLLM containers: 'multimodal-coder' (http://vllm-multimodal-coder:8000) and 'middle' (http://vllm-middle:8000) have no container at all, yet they are listed to clients AND sit in the generate failover chain, which for the cortex model is [primary, multimodal, multimodal-coder, middle]." + }, + { + "id": "h24", + "kind": "honesty", + "text": "With MULTIMODAL_CODER_SERVED_NAME and MIDDLE_SERVED_NAME set but no BASE_URL and no container, build_config wires NEITHER backend, GET /v1/models returns exactly the four running ids, and resolve_model('nvidia/Qwen3-14B-NVFP4') does not resolve to the primary's weights." + }, + { + "id": "c22", + "kind": "requirement", + "text": "A LOCAL live-test of the capabilities contract runs before every PR. It is not a CI job (CI has no GPU and no fleet); it is a local gate the developer runs against the running deployment, in the same live-gated seam tests/test_smoke_duo.py already uses (LOBES_SMOKE_BASE_URL). It must fail loudly rather than skip silently when the operator intended to run it, so that 'advertised implies reachable' is checked against the DEPLOYED artifact before a PR claims to have fixed it \u2014 the exact step whose absence let #87 regress into #92." + }, + { + "id": "h17", + "kind": "honesty", + "text": "The local pre-PR live test, pointed at a deployment whose gateway is stale or whose audio overlay is unwired, FAILS \u2014 it does not skip, and it does not pass." + }, + { + "id": "h18", + "kind": "honesty", + "text": "The gate is ONE trigger and then unattended: a single local command runs the whole capabilities live-test to a pass/fail verdict with no prompts, no manual steps, and no per-role babysitting \u2014 the developer types it, walks away, and reads the exit code." + }, + { + "id": "c23", + "kind": "requirement", + "text": "NO cross-backend failover. Every request resolves to exactly ONE backend and is attempted only there. An explicit model id goes to its owner; a capability tier alias resolves once, up front, via the existing static tier_aliases upward fallback (a tier whose gear is not WIRED still maps to a higher rung at table-build time). There is no runtime retry against a second backend, so a caller who asked for cortex can never silently receive a Gemma answer. A dead/unreachable/warming owner yields 503 + Retry-After. order_backends collapses to a single-element list and rewrite_model stays outside any loop." + }, + { + "id": "h21", + "kind": "honesty", + "text": "A unit test with a fake fleet proves that a request naming the cortex model, with the primary backend dead, NEVER opens a connection to the multimodal backend, and returns 503 + Retry-After. order_backends(table, served) returns exactly one backend for every input." + }, + { + "id": "c27", + "kind": "requirement", + "text": "The pre-PR gate detects DEPLOYED-ARTIFACT skew, not just source correctness: it compares the running gateway container's lobes.__version__ against the CLI's and fails on mismatch, and it dials every advertised role endpoint+path. This is the check whose absence let #87's fix ship in 0.38.0 while the rig kept running 0.36.0 \u2014 making #92 look like a code regression when the code was already correct and merely undeployed." + }, + { + "id": "h22", + "kind": "honesty", + "text": "Run against the rig as it stands right now (gateway 0.36.0, CLI 0.39.0, endpoint :8000, /v1/audio/speech 404ing), the gate exits non-zero and names all three faults; after redeploy it exits zero." + } + ], + "tasks": [ + { + "id": "t1", + "summary": "Backend wiring gate: _optional_backend wires a backend ONLY when its *_BASE_URL is set (drop the 'or *_SERVED_NAME' clause), so no phantom backend is ever invented from a default_url", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "With MULTIMODAL_CODER_SERVED_NAME and MIDDLE_SERVED_NAME set but their *_BASE_URL empty and no container, build_config wires NEITHER backend", + "resolve_model('nvidia/Qwen3-14B-NVFP4') on such a deployment does not resolve to the primary's weights", + "The packaged fleet env.example (which sets both vars when a profile is on) is unaffected: enabling COMPOSE_PROFILES=middle still wires the middle backend", + "Files touched: lobes/gateway/_config.py, tests/test_gateway_config_wiring.py (new)" + ], + "deps": [], + "covers": [ + "c19", + "h24" + ] + }, + { + "id": "t2", + "summary": "No cross-backend failover: order_backends returns exactly ONE backend for every input, and the test that currently asserts cross-model failover is inverted", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "order_backends(table, served) returns a list of length <= 1 for every input, including tier-alias-resolved names", + "tests/test_gateway_routing.py::test_order_backends_generate_still_failovers_between_generate_backends is INVERTED to assert the new contract, not deleted", + "The static tier_aliases upward fallback (an unwired tier maps to a higher rung at table-build time) is preserved and still tested", + "Files touched: lobes/gateway/_routing.py, tests/test_gateway_routing.py" + ], + "deps": [], + "covers": [ + "c23", + "h21" + ] + }, + { + "id": "t3", + "summary": "Backend readiness cache: a bounded BACKGROUND probe of each backend's /health, mirroring PressureCache, exposing a cached tri-state per backend and never probing on the request hot path", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "A new lobes/gateway/_readiness.py exposes a cache with .current() returning per-backend readiness, refreshed off the request path on an interval", + "A unit test proves .current() opens no socket (injected probe callable, call count asserted zero across N reads)", + "The probe degrades to 'unknown' (never raises) on OSError, http.client.HTTPException and ValueError \u2014 the malformed-URL lesson from PR #90", + "The background thread is a daemon and stops cleanly on server shutdown", + "Files touched: lobes/gateway/_readiness.py (new), tests/test_gateway_readiness.py (new)" + ], + "deps": [], + "covers": [ + "c8", + "h8" + ] + }, + { + "id": "t4", + "summary": "Fleet template truth: inject GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) and AUDIO_URL into the gateway container from the BASE fleet compose, so the advertised origin is configured truth and stt/tts stop advertising a 404 path (issue #96)", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "lobes/templates/fleet/docker-compose.yml gateway service passes GATEWAY_PUBLIC_URL defaulted from VLLM_PORT, and AUDIO_URL, without requiring the -f docker-compose.audio.yml overlay", + "A test parses the packaged fleet compose and asserts the gateway environment contains both keys", + "env.example documents GATEWAY_PUBLIC_URL as the tunnel/proxy override and notes it defaults to the published port", + "An audio-less deployment still yields audio_url unset in the gateway (AUDIO_URL empty), so stt/tts report loaded=false rather than a 404ing ready=true", + "Files touched: lobes/templates/fleet/docker-compose.yml, lobes/templates/fleet/env.example, tests/test_fleet_template_gateway_env.py (new)" + ], + "deps": [], + "covers": [ + "c11", + "c18" + ] + }, + { + "id": "t5", + "summary": "roles.py: RoleInfo.ready stops being an alias of loaded for the four gateway-fronted roles, and the endpoint is never built from the gateway's internal listen port", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "build_role_registry accepts a per-backend readiness signal; ready reflects it while loaded stays the config fact \u2014 the stt/tts separation from #89, generalised to cortex/senses/embedder/reranker", + "_gateway_base_url no longer returns an absolute URL when only an internal listen port is known: with no gateway_url and no public_url the endpoint is empty, never http://localhost:", + "An unconfigured/unready role is still returned (never omitted, never raises), matching the existing six-roles-always-present contract", + "Files touched: lobes/roles.py, tests/test_roles.py" + ], + "deps": [ + "t3" + ], + "covers": [ + "c15", + "c3" + ] + }, + { + "id": "t6", + "summary": "Gateway core: dead owner yields a retryable 503, the readiness cache feeds /v1/models + /capabilities.ready, and reachable_origin prefers configured truth over Host-header inference", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "handle_post: an owner that refuses, times out, or returns >=500 yields HTTP 503 with a Retry-After header and an OpenAI-shaped error whose type (e.g. backend_unavailable) is distinguishable from both 'model unknown' and the all-backends-down 502", + "A fake-fleet test proves a request naming the cortex model with the primary dead NEVER opens a connection to the multimodal backend (upstream-opener call sites asserted)", + "Race test: a fake fleet lists model M, the owner is then killed, and a completion naming M returns 503 + Retry-After \u2014 never 404", + "Converse test: an id that was never in /v1/models is not silently served by the default backend under a different model's weights", + "GET /v1/models is filtered by the cached readiness signal; GET /capabilities .ready reflects it for all six roles", + "reachable_origin(None, None) never fabricates an absolute URL from GATEWAY_PORT; GATEWAY_PUBLIC_URL > Host header > empty", + "The POST hot path opens no probe connection (asserted by call count against an injected opener)", + "Files touched: lobes/gateway/server.py, tests/test_gateway_server.py, tests/test_gateway_capabilities.py" + ], + "deps": [ + "t1", + "t2", + "t3", + "t5" + ], + "covers": [ + "c4", + "h4", + "c5", + "h5", + "c12", + "h23", + "c14", + "h13", + "h10", + "h14", + "h19" + ] + }, + { + "id": "t7", + "summary": "CLI truth: lobes capabilities agrees with GET /capabilities byte-for-byte, and lobes doctor detects deployed-gateway version skew", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "A test asserts GET /capabilities and 'lobes capabilities --json' return identical endpoint/ready/loaded for all six roles on the same deployment config", + "The CLI never reports stt/tts ready=true purely because AUDIO_URL is a string in .env", + "lobes doctor gains a check comparing the running gateway container's lobes.__version__ against the CLI wheel's, failing with severity=error on mismatch and remediation naming the rebuild command", + "Run against the rig as it stands (gateway 0.36.0, CLI 0.39.0) the version-skew check FAILS; after a rebuild it passes", + "Files touched: lobes/cli/_commands/capabilities.py, lobes/cli/_commands/doctor.py, tests/test_cli_capabilities.py, tests/test_doctor.py" + ], + "deps": [ + "t5", + "t6" + ], + "covers": [ + "h3", + "c27", + "h22" + ] + }, + { + "id": "t8", + "summary": "The senses perception probe: prove coolthor actually PERCEIVES, not merely that the wire accepts content-parts \u2014 ground-truth image and ground-truth audio", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "The image probe generates a solid-colour PNG in-process and asserts the model NAMES that colour (red -> 'red', blue -> 'blue'), not merely that content is non-empty", + "The audio probe synthesizes a known word via the rig's own /v1/audio/speech and asserts the transcription contains it \u2014 this only passes once t4 wires AUDIO_URL (issue #96)", + "The existing 1x1-placeholder assertions are replaced or explicitly relabelled as wire-liveness checks, so 'image+text confirmed' never again means 'HTTP 200 with non-empty content'", + "Files touched: tests/test_smoke_duo.py" + ], + "deps": [ + "t4", + "t6" + ], + "covers": [ + "c7", + "h7", + "c17" + ] + }, + { + "id": "t9", + "summary": "The pre-PR live gate: ONE local command, unattended, that dials every advertised role endpoint+path and the deployed gateway's version, and FAILS rather than skips", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "A single command runs the whole capabilities live-test to a pass/fail exit code with no prompts and no manual steps", + "When the operator asked for the live gate (the env var / flag is set) an unreachable deployment FAILS the run; it never degrades to pytest-skip", + "For every role in GET /capabilities the gate dials endpoint+path and asserts a non-404 status; for every id in GET /v1/models it asserts a completion never returns 404", + "It reproduces Colleague's discovery path: given ONLY the gateway origin and no COLLEAGUE_*_BASE_URL override, resolve cortex and senses from the contract and get an answer", + "It fails on deployed-gateway version skew", + "Run against today's rig it exits non-zero naming the :8000 endpoint, the 404ing audio path, and the 0.36.0-vs-0.39.0 skew; after redeploy it exits zero", + "The repo's pre-PR convention (CLAUDE.md / the run-tests skill) names this command so it is not silently never run", + "Files touched: tests/test_live_capabilities.py (new), scripts or Makefile target for the single trigger, CLAUDE.md" + ], + "deps": [ + "t4", + "t6", + "t7" + ], + "covers": [ + "c1", + "h1", + "c2", + "h2", + "c6", + "h6", + "c10", + "h9", + "c22", + "h17", + "h18" + ] + }, + { + "id": "t10", + "summary": "Documentary repairs: record the perception evidence, retire the DSpark route, and stop README claiming lobes init is single-model", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "docs/gemma-4-12b-nvfp4.md no longer contains the line-279 admission that content-correctness checks were 'not independently re-run against the base checkpoint specifically' \u2014 because they were, and the coolthor results are recorded there", + "docs/gemma4-mtp-draft.md carries a superseded banner: DSpark (deepseek-ai/dspark_gemma4_12b_block7) does NOT load on vLLM 0.23 (Gemma4DSparkModel unsupported), per #75 \u2014 it must no longer read as 'the ONE route task t3 should wire next'", + "README.md quickstart no longer documents 'lobes init' as scaffolding the single-model deployment on :8000; the duo is the default and --single opts out", + "Files touched: docs/gemma-4-12b-nvfp4.md, docs/gemma4-mtp-draft.md, README.md" + ], + "deps": [ + "t9", + "t8" + ], + "covers": [ + "h16" + ] + } + ], + "risks": [ + { + "id": "r1", + "text": "Requiring *_BASE_URL to wire a backend could break a hand-edited .env that set only *_SERVED_NAME. The packaged fleet template always sets both when a profile is enabled, but an operator's local file may not \u2014 needs a release note and possibly a warning path.", + "kind": "unknown_nonblocking", + "task_id": "t1" + }, + { + "id": "r2", + "text": "The background readiness thread lives inside a stdlib ThreadingHTTPServer. Daemon-thread lifecycle, clean shutdown, and behaviour under 'docker compose down' need care \u2014 a probe thread that outlives the server or blocks shutdown is a regression.", + "kind": "unknown_nonblocking", + "task_id": "t3" + }, + { + "id": "r3", + "text": "The audio perception probe depends on Chatterbox TTS, which has a recorded history of a poisoned CUDA context (500s cleared only by restarting the container). The probe may need a readiness precondition or a retry, or it will flake for reasons unrelated to senses.", + "kind": "unknown_nonblocking", + "task_id": "t9" + }, + { + "id": "r4", + "text": "The live gate cannot run in CI (no GPU, no fleet), so nothing structurally forces it to run. If it is only a convention it will be skipped exactly when it matters \u2014 which is precisely how #87's fix shipped in 0.38.0 while the rig ran 0.36.0. Consider a pre-push hook or a PR-template checkbox.", + "kind": "follow_up", + "task_id": "t8" + } + ] +} diff --git a/docs/plans/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md b/docs/plans/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md new file mode 100644 index 0000000..54ecc99 --- /dev/null +++ b/docs/plans/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md @@ -0,0 +1,121 @@ +# Build Plan — lobes never advertises a capability it cannot serve: every endpoint, model and role in the machine-readable contract is proven reachable, and a fleet fault degrades to a retryable 503 instead of a terminal 404 + +slug: `lobes-never-advertises-a-capability-it-cannot-serv` · status: `exported` · from frame: `lobes-never-advertises-a-capability-it-cannot-serv` + +> lobes never advertises a capability it cannot serve: every endpoint, model and role in the machine-readable contract is proven reachable, and a fleet fault degrades to a retryable 503 instead of a terminal 404 + +## Tasks + +### t1 — Backend wiring gate: _optional_backend wires a backend ONLY when its *_BASE_URL is set (drop the 'or *_SERVED_NAME' clause), so no phantom backend is ever invented from a default_url + +- covers: c19, h24 +- acceptance: + - With MULTIMODAL_CODER_SERVED_NAME and MIDDLE_SERVED_NAME set but their *_BASE_URL empty and no container, build_config wires NEITHER backend + - resolve_model('nvidia/Qwen3-14B-NVFP4') on such a deployment does not resolve to the primary's weights + - The packaged fleet env.example (which sets both vars when a profile is on) is unaffected: enabling COMPOSE_PROFILES=middle still wires the middle backend + - Files touched: lobes/gateway/_config.py, tests/test_gateway_config_wiring.py (new) + +### t2 — No cross-backend failover: order_backends returns exactly ONE backend for every input, and the test that currently asserts cross-model failover is inverted + +- covers: c23, h21 +- acceptance: + - order_backends(table, served) returns a list of length <= 1 for every input, including tier-alias-resolved names + - tests/test_gateway_routing.py::test_order_backends_generate_still_failovers_between_generate_backends is INVERTED to assert the new contract, not deleted + - The static tier_aliases upward fallback (an unwired tier maps to a higher rung at table-build time) is preserved and still tested + - Files touched: lobes/gateway/_routing.py, tests/test_gateway_routing.py + +### t3 — Backend readiness cache: a bounded BACKGROUND probe of each backend's /health, mirroring PressureCache, exposing a cached tri-state per backend and never probing on the request hot path + +- covers: c8, h8 +- acceptance: + - A new lobes/gateway/_readiness.py exposes a cache with .current() returning per-backend readiness, refreshed off the request path on an interval + - A unit test proves .current() opens no socket (injected probe callable, call count asserted zero across N reads) + - The probe degrades to 'unknown' (never raises) on OSError, http.client.HTTPException and ValueError — the malformed-URL lesson from PR #90 + - The background thread is a daemon and stops cleanly on server shutdown + - Files touched: lobes/gateway/_readiness.py (new), tests/test_gateway_readiness.py (new) + +### t4 — Fleet template truth: inject GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) and AUDIO_URL into the gateway container from the BASE fleet compose, so the advertised origin is configured truth and stt/tts stop advertising a 404 path (issue #96) + +- covers: c11, c18 +- acceptance: + - lobes/templates/fleet/docker-compose.yml gateway service passes GATEWAY_PUBLIC_URL defaulted from VLLM_PORT, and AUDIO_URL, without requiring the -f docker-compose.audio.yml overlay + - A test parses the packaged fleet compose and asserts the gateway environment contains both keys + - env.example documents GATEWAY_PUBLIC_URL as the tunnel/proxy override and notes it defaults to the published port + - An audio-less deployment still yields audio_url unset in the gateway (AUDIO_URL empty), so stt/tts report loaded=false rather than a 404ing ready=true + - Files touched: lobes/templates/fleet/docker-compose.yml, lobes/templates/fleet/env.example, tests/test_fleet_template_gateway_env.py (new) + +### t5 — roles.py: RoleInfo.ready stops being an alias of loaded for the four gateway-fronted roles, and the endpoint is never built from the gateway's internal listen port + +- depends on: t3 +- covers: c15, c3 +- acceptance: + - build_role_registry accepts a per-backend readiness signal; ready reflects it while loaded stays the config fact — the stt/tts separation from #89, generalised to cortex/senses/embedder/reranker + - _gateway_base_url no longer returns an absolute URL when only an internal listen port is known: with no gateway_url and no public_url the endpoint is empty, never http://localhost: + - An unconfigured/unready role is still returned (never omitted, never raises), matching the existing six-roles-always-present contract + - Files touched: lobes/roles.py, tests/test_roles.py + +### t6 — Gateway core: dead owner yields a retryable 503, the readiness cache feeds /v1/models + /capabilities.ready, and reachable_origin prefers configured truth over Host-header inference + +- depends on: t1, t2, t3, t5 +- covers: c4, h4, c5, h5, c12, h23, c14, h13, h10, h14, h19 +- acceptance: + - handle_post: an owner that refuses, times out, or returns >=500 yields HTTP 503 with a Retry-After header and an OpenAI-shaped error whose type (e.g. backend_unavailable) is distinguishable from both 'model unknown' and the all-backends-down 502 + - A fake-fleet test proves a request naming the cortex model with the primary dead NEVER opens a connection to the multimodal backend (upstream-opener call sites asserted) + - Race test: a fake fleet lists model M, the owner is then killed, and a completion naming M returns 503 + Retry-After — never 404 + - Converse test: an id that was never in /v1/models is not silently served by the default backend under a different model's weights + - GET /v1/models is filtered by the cached readiness signal; GET /capabilities .ready reflects it for all six roles + - reachable_origin(None, None) never fabricates an absolute URL from GATEWAY_PORT; GATEWAY_PUBLIC_URL > Host header > empty + - The POST hot path opens no probe connection (asserted by call count against an injected opener) + - Files touched: lobes/gateway/server.py, tests/test_gateway_server.py, tests/test_gateway_capabilities.py + +### t7 — CLI truth: lobes capabilities agrees with GET /capabilities byte-for-byte, and lobes doctor detects deployed-gateway version skew + +- depends on: t5, t6 +- covers: h3, c27, h22 +- acceptance: + - A test asserts GET /capabilities and 'lobes capabilities --json' return identical endpoint/ready/loaded for all six roles on the same deployment config + - The CLI never reports stt/tts ready=true purely because AUDIO_URL is a string in .env + - lobes doctor gains a check comparing the running gateway container's lobes.__version__ against the CLI wheel's, failing with severity=error on mismatch and remediation naming the rebuild command + - Run against the rig as it stands (gateway 0.36.0, CLI 0.39.0) the version-skew check FAILS; after a rebuild it passes + - Files touched: lobes/cli/_commands/capabilities.py, lobes/cli/_commands/doctor.py, tests/test_cli_capabilities.py, tests/test_doctor.py + +### t8 — The senses perception probe: prove coolthor actually PERCEIVES, not merely that the wire accepts content-parts — ground-truth image and ground-truth audio + +- depends on: t4, t6 +- covers: c7, h7, c17 +- acceptance: + - The image probe generates a solid-colour PNG in-process and asserts the model NAMES that colour (red -> 'red', blue -> 'blue'), not merely that content is non-empty + - The audio probe synthesizes a known word via the rig's own /v1/audio/speech and asserts the transcription contains it — this only passes once t4 wires AUDIO_URL (issue #96) + - The existing 1x1-placeholder assertions are replaced or explicitly relabelled as wire-liveness checks, so 'image+text confirmed' never again means 'HTTP 200 with non-empty content' + - Files touched: tests/test_smoke_duo.py + +### t9 — The pre-PR live gate: ONE local command, unattended, that dials every advertised role endpoint+path and the deployed gateway's version, and FAILS rather than skips + +- depends on: t4, t6, t7 +- covers: c1, h1, c2, h2, c6, h6, c10, h9, c22, h17, h18 +- acceptance: + - A single command runs the whole capabilities live-test to a pass/fail exit code with no prompts and no manual steps + - When the operator asked for the live gate (the env var / flag is set) an unreachable deployment FAILS the run; it never degrades to pytest-skip + - For every role in GET /capabilities the gate dials endpoint+path and asserts a non-404 status; for every id in GET /v1/models it asserts a completion never returns 404 + - It reproduces Colleague's discovery path: given ONLY the gateway origin and no COLLEAGUE_*_BASE_URL override, resolve cortex and senses from the contract and get an answer + - It fails on deployed-gateway version skew + - Run against today's rig it exits non-zero naming the :8000 endpoint, the 404ing audio path, and the 0.36.0-vs-0.39.0 skew; after redeploy it exits zero + - The repo's pre-PR convention (CLAUDE.md / the run-tests skill) names this command so it is not silently never run + - Files touched: tests/test_live_capabilities.py (new), scripts or Makefile target for the single trigger, CLAUDE.md + +### t10 — Documentary repairs: record the perception evidence, retire the DSpark route, and stop README claiming lobes init is single-model + +- depends on: t9, t8 +- covers: h16 +- acceptance: + - docs/gemma-4-12b-nvfp4.md no longer contains the line-279 admission that content-correctness checks were 'not independently re-run against the base checkpoint specifically' — because they were, and the coolthor results are recorded there + - docs/gemma4-mtp-draft.md carries a superseded banner: DSpark (deepseek-ai/dspark_gemma4_12b_block7) does NOT load on vLLM 0.23 (Gemma4DSparkModel unsupported), per #75 — it must no longer read as 'the ONE route task t3 should wire next' + - README.md quickstart no longer documents 'lobes init' as scaffolding the single-model deployment on :8000; the duo is the default and --single opts out + - Files touched: docs/gemma-4-12b-nvfp4.md, docs/gemma4-mtp-draft.md, README.md + +## Risks + +- [unknown_nonblocking] Requiring *_BASE_URL to wire a backend could break a hand-edited .env that set only *_SERVED_NAME. The packaged fleet template always sets both when a profile is enabled, but an operator's local file may not — needs a release note and possibly a warning path. (task t1) +- [unknown_nonblocking] The background readiness thread lives inside a stdlib ThreadingHTTPServer. Daemon-thread lifecycle, clean shutdown, and behaviour under 'docker compose down' need care — a probe thread that outlives the server or blocks shutdown is a regression. (task t3) +- [unknown_nonblocking] The audio perception probe depends on Chatterbox TTS, which has a recorded history of a poisoned CUDA context (500s cleared only by restarting the container). The probe may need a readiness precondition or a retry, or it will flake for reasons unrelated to senses. (task t9) +- [follow_up] The live gate cannot run in CI (no GPU, no fleet), so nothing structurally forces it to run. If it is only a convention it will be skipped exactly when it matters — which is precisely how #87's fix shipped in 0.38.0 while the rig ran 0.36.0. Consider a pre-push hook or a PR-template checkbox. (task t8) From 972b47fc6508e10917ef9d7165095827550abfbd Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 09:57:20 +0300 Subject: [PATCH 03/23] fix: base fleet compose passes GATEWAY_PUBLIC_URL and AUDIO_URL to the gateway (#96) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two env vars never reached the gateway container from the base fleet compose, both causing it to advertise things it cannot serve: - GATEWAY_PUBLIC_URL defaulted to empty, so the advertised /capabilities origin fell back to Host-header inference — and, absent a Host header, fabricated a URL from the gateway's internal listen port rather than the published host port. It now defaults to http://localhost:${VLLM_PORT:-8000}, the same port the gateway is actually published on, while staying operator-overridable for a tunnel / Host-rewriting reverse proxy. - AUDIO_URL only reached the gateway via the --audio overlay (docker-compose.audio.yml). A base-only deployment left ServerConfig.audio_url unset in code but 'lobes capabilities' (reading the merged .env) still reported stt/tts as ready=true, and POST /v1/audio/speech 404'd. AUDIO_URL is now declared on the base template too, defaulted to empty so a base-only deployment resolves it to unset (loaded=false) instead of pointing at a realtime container that was never started; the audio overlay's override still supplies the real value when present. Verified the nested ${GATEWAY_PUBLIC_URL:-http://localhost:${VLLM_PORT:-8000}} interpolation with `docker compose config` against a scratch copy of the template, both with and without the audio overlay layered on top. tests/test_fleet_template_gateway_env.py (new) parses the packaged fleet compose and locks in both keys/defaults. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/templates/fleet/docker-compose.yml | 14 ++- lobes/templates/fleet/env.example | 12 +- tests/test_fleet_template_gateway_env.py | 133 +++++++++++++++++++++++ 3 files changed, 154 insertions(+), 5 deletions(-) create mode 100644 tests/test_fleet_template_gateway_env.py diff --git a/lobes/templates/fleet/docker-compose.yml b/lobes/templates/fleet/docker-compose.yml index 45a3b79..11bc4de 100644 --- a/lobes/templates/fleet/docker-compose.yml +++ b/lobes/templates/fleet/docker-compose.yml @@ -618,7 +618,19 @@ services: - vllm-rerank environment: - GATEWAY_PORT=8000 - - GATEWAY_PUBLIC_URL=${GATEWAY_PUBLIC_URL:-} + # Defaults to the PUBLISHED host port (VLLM_PORT, below), not the + # internal container port — configured truth, not Host-header inference + # (issue #96). Set GATEWAY_PUBLIC_URL explicitly to override for a + # tunnel / Host-rewriting reverse proxy (still wins over this default). + - GATEWAY_PUBLIC_URL=${GATEWAY_PUBLIC_URL:-http://localhost:${VLLM_PORT:-8000}} + # Audio/realtime backend for /v1/audio/*. Empty by default — this base + # template has no realtime container; the --audio overlay + # (docker-compose.audio.yml) supplies the real http://realtime:8080 + # value via its gateway environment: override (issue #96). Declaring the + # key here (rather than leaving it entirely absent) means a base-only + # deployment resolves audio_url to unset (stt/tts report loaded=false), + # instead of a 404ing endpoint /capabilities still calls ready=true. + - AUDIO_URL=${AUDIO_URL:-} - PRIMARY_URL=http://vllm-primary:8000 - PRIMARY_SERVED_NAME=${PRIMARY_SERVED_NAME:-sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP} # To add a fallback: set FALLBACK_URL + FALLBACK_SERVED_NAME here (the diff --git a/lobes/templates/fleet/env.example b/lobes/templates/fleet/env.example index 891550d..ff0f1c9 100644 --- a/lobes/templates/fleet/env.example +++ b/lobes/templates/fleet/env.example @@ -225,10 +225,14 @@ MULTIMODAL_CODER_CUDAGRAPH_ESTIMATE=0 # Host port acp/culture.yaml expects → mapped to the gateway. The primary backend # is reachable only on the internal compose network (vllm-primary:8000). VLLM_PORT=8000 -# GATEWAY_PUBLIC_URL — OPTIONAL. The client-reachable origin the gateway advertises -# for every role in GET /capabilities (e.g. a cloudflared tunnel URL like -# https://foo.example). Leave empty to use the incoming request Host header -# (correct for a normal published host port). +# GATEWAY_PUBLIC_URL — OPTIONAL override for a tunnel / Host-rewriting reverse +# proxy (e.g. a cloudflared tunnel URL like https://foo.example). Set it when +# clients dial a different origin than the one you published below. +# Leave unset/empty and it defaults to http://localhost:${VLLM_PORT} — the +# published host port (issue #96; previously an empty default let the +# advertised /capabilities origin fall back to inferring from the incoming +# request Host header, and when no Host header was present, fabricated a URL +# from the gateway's INTERNAL container port instead of the published one). # GATEWAY_PUBLIC_URL= GATEWAY_DEFAULT_MODEL=sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP # Extra name -> served-name routes, comma-separated (e.g. diff --git a/tests/test_fleet_template_gateway_env.py b/tests/test_fleet_template_gateway_env.py new file mode 100644 index 0000000..ac6b4db --- /dev/null +++ b/tests/test_fleet_template_gateway_env.py @@ -0,0 +1,133 @@ +"""The base fleet compose must pass GATEWAY_PUBLIC_URL and AUDIO_URL to the +gateway container WITHOUT the ``-f docker-compose.audio.yml`` overlay (issue +#96). + +Two bugs this guards against: + + (a) GATEWAY_PUBLIC_URL defaulted to empty (``${GATEWAY_PUBLIC_URL:-}``), so + the advertised /capabilities origin fell back to inferring from the + request Host header — and when no Host header is present, fabricated an + absolute URL from the gateway's INTERNAL listen port + (GATEWAY_PORT=8000), not the published host port (VLLM_PORT, default + 8001 on the reference rig). The default must instead be built from the + published ``VLLM_PORT`` mapping (``"${VLLM_PORT:-8000}:8000"``), so the + advertised origin is configured truth. + + (b) AUDIO_URL only reached the gateway via docker-compose.audio.yml's + override block. On a base-only deployment (no --audio overlay), + ServerConfig.audio_url is empty and POST /v1/audio/speech 404s, while + `lobes capabilities` (reading the merged .env) reports stt/tts as + ready=true. AUDIO_URL must be present in the base template's gateway + environment, defaulted to EMPTY (``${AUDIO_URL:-}``) — the audio overlay + is what supplies the real ``http://realtime:8080`` value; a base-only + deployment must resolve audio_url to unset, not a URL of a container + that was never started. +""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +_TEMPLATES = Path(__file__).resolve().parents[1] / "lobes" / "templates" +_FLEET_COMPOSE = _TEMPLATES / "fleet" / "docker-compose.yml" + + +def _load_fleet() -> dict: + return yaml.safe_load(_FLEET_COMPOSE.read_text(encoding="utf-8")) + + +def _gateway_env_map() -> dict[str, str]: + """The gateway's ``environment:`` list, as a ``{KEY: raw-value}`` dict. + + Values keep their raw ``${...}`` interpolation text (PyYAML expands + nothing) — exactly what a caller composing WITHOUT the audio overlay + would see baked into the base template. + """ + compose = _load_fleet() + env: list[str] = compose["services"]["gateway"]["environment"] + out: dict[str, str] = {} + for entry in env: + assert "=" in entry, f"non key=value gateway environment entry: {entry!r}" + key, _, value = entry.partition("=") + out[key] = value + return out + + +class TestGatewayEnvKeysPresent: + """Both vars must reach the gateway from the BASE template alone.""" + + def test_gateway_public_url_key_present(self) -> None: + env = _gateway_env_map() + assert "GATEWAY_PUBLIC_URL" in env + + def test_audio_url_key_present(self) -> None: + env = _gateway_env_map() + assert "AUDIO_URL" in env + + +class TestGatewayPublicUrlDefaultsFromVllmPort: + """The GATEWAY_PUBLIC_URL default must be built from the published + VLLM_PORT mapping (``"${VLLM_PORT:-8000}:8000"``), not left empty.""" + + def test_default_is_not_empty(self) -> None: + env = _gateway_env_map() + value = env["GATEWAY_PUBLIC_URL"] + # The old, buggy form: an empty default that falls through to + # Host-header inference (and, absent a Host header, the internal port). + assert value != "${GATEWAY_PUBLIC_URL:-}", ( + "GATEWAY_PUBLIC_URL must not default to empty — an empty default " + "lets the advertised origin fall back to the gateway's internal " + "listen port when no Host header is present (issue #96)" + ) + + def test_default_references_vllm_port(self) -> None: + env = _gateway_env_map() + value = env["GATEWAY_PUBLIC_URL"] + assert "${VLLM_PORT" in value, ( + "GATEWAY_PUBLIC_URL's default must be derived from ${VLLM_PORT:-...} " + f"(the published host port) — got {value!r}" + ) + + def test_still_overridable_by_operator(self) -> None: + # The var name itself must still gate on GATEWAY_PUBLIC_URL, so an + # operator-set GATEWAY_PUBLIC_URL in .env continues to win (a tunnel / + # Host-rewriting reverse proxy). + env = _gateway_env_map() + value = env["GATEWAY_PUBLIC_URL"] + assert value.startswith( + "${GATEWAY_PUBLIC_URL:-" + ), f"GATEWAY_PUBLIC_URL must remain operator-overridable — got {value!r}" + + def test_ports_mapping_is_the_vllm_port_default_this_test_assumes(self) -> None: + # Guard the assumption this whole class is built on: the published + # mapping is "${VLLM_PORT:-8000}:8000". If that ever changes, this + # test (and the GATEWAY_PUBLIC_URL default it drives) must be revisited. + compose = _load_fleet() + ports = compose["services"]["gateway"]["ports"] + assert "${VLLM_PORT:-8000}:8000" in ports + + def test_rendered_default_matches_published_port_shape(self) -> None: + # Exact string check on the template as documented in the task: the + # nested-default form composing GATEWAY_PUBLIC_URL from VLLM_PORT. + env = _gateway_env_map() + assert ( + env["GATEWAY_PUBLIC_URL"] + == "${GATEWAY_PUBLIC_URL:-http://localhost:${VLLM_PORT:-8000}}" + ) + + +class TestAudioUrlDefaultsEmpty: + """AUDIO_URL must default to EMPTY on the base template — the audio + overlay (docker-compose.audio.yml) supplies the real + http://realtime:8080 value; a base-only deployment has no realtime + container to point at.""" + + def test_default_is_empty_not_a_url(self) -> None: + env = _gateway_env_map() + assert env["AUDIO_URL"] == "${AUDIO_URL:-}", ( + "AUDIO_URL must default to empty on the base template — a non-empty " + "default (e.g. http://realtime:8080) would advertise a realtime " + "container that a base-only (no --audio) deployment never starts" + ) From 8d679fbee047003c25f49167d6d6d1881e5548bf Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 09:57:32 +0300 Subject: [PATCH 04/23] Wire a fleet backend only when its *_BASE_URL is set (t1) _optional_backend previously wired a backend when EITHER its *_BASE_URL OR its *_SERVED_NAME env var was set, falling back to a hardcoded default_url naming a compose service that need not exist. On the reference rig this invented two phantom backends -- multimodal-coder and middle -- that GET /v1/models advertised while no container served them. Drop the `or name_key` clause: a served name with no URL describes a model, not a reachable backend. This matches the contract the fleet already documents for MINOR_BASE_URL ("empty => minor silently unwired"). Verified on the live rig this unwires exactly multimodal-coder and middle, keeping primary/multimodal/embed/rerank. Repairs the tests that asserted the old permissive behaviour (test_gateway_routing, test_fleet_minor, test_gateway_tiers, and the test_gateway_server _cfg fixture, which relied on FALLBACK_SERVED_NAME alone wiring the fallback backend for its failover tests) and adds tests/test_gateway_config_wiring.py covering the new contract. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/gateway/_config.py | 18 ++-- tests/test_fleet_minor.py | 19 +++- tests/test_gateway_config_wiring.py | 133 ++++++++++++++++++++++++++++ tests/test_gateway_routing.py | 23 +++-- tests/test_gateway_server.py | 7 +- tests/test_gateway_tiers.py | 41 +++++++-- 6 files changed, 220 insertions(+), 21 deletions(-) create mode 100644 tests/test_gateway_config_wiring.py diff --git a/lobes/gateway/_config.py b/lobes/gateway/_config.py index 765ae35..a88b705 100644 --- a/lobes/gateway/_config.py +++ b/lobes/gateway/_config.py @@ -112,13 +112,21 @@ def _optional_backend( default_name: str, task: str = "generate", ) -> Backend | None: - """A fleet backend wired only when its env (``url_key`` or ``name_key``) is set. + """A fleet backend wired only when its ``url_key`` env var is non-empty. - Returns ``None`` when neither is present — so the default gateway serves the - primary alone, and each extra backend (fallback / embed / rerank) opts in - independently via its own env pair. + ``name_key`` alone is NOT enough — a served name with no URL describes a + model, not a reachable backend, and wiring one anyway invents a "phantom" + backend whose ``base_url`` falls back to a hardcoded ``default_url`` + naming a compose service that need not exist (advertised on + ``GET /v1/models`` yet unreachable: every request to it fails to + connect). This mirrors the contract the fleet already documents for + ``MINOR_BASE_URL`` — empty ⇒ silently unwired. + + Returns ``None`` when ``url_key`` is absent/empty — so the default + gateway serves the primary alone, and each extra backend (fallback / + embed / rerank / …) opts in independently via its own ``*_BASE_URL``. """ - if not (env.get(url_key) or env.get(name_key)): + if not env.get(url_key): return None return Backend( name=name, diff --git a/tests/test_fleet_minor.py b/tests/test_fleet_minor.py index cd2dd19..ef3f569 100644 --- a/tests/test_fleet_minor.py +++ b/tests/test_fleet_minor.py @@ -45,13 +45,26 @@ def test_minor_backend_added_when_minor_url_set() -> None: assert minor.task == "generate" -def test_minor_backend_added_when_minor_served_name_set() -> None: - """MINOR_SERVED_NAME alone triggers the minor backend with the default URL.""" +def test_minor_backend_not_added_when_only_minor_served_name_set() -> None: + """MINOR_SERVED_NAME alone does NOT wire the minor backend — a served name + with no URL describes a model, not a reachable backend.""" table, _ = build_config({"MINOR_SERVED_NAME": _MINOR_SERVED}) names = [b.name for b in table.backends] + assert "minor" not in names + assert names == ["primary"] + + +def test_minor_backend_added_when_both_minor_url_and_served_name_set() -> None: + """MINOR_BASE_URL wires the backend; MINOR_SERVED_NAME alongside it just + customises the served name (mirrors the default-name case above).""" + table, _ = build_config( + {"MINOR_BASE_URL": "http://vllm-minor:8000", "MINOR_SERVED_NAME": _MINOR_SERVED} + ) + names = [b.name for b in table.backends] assert "minor" in names minor = next(b for b in table.backends if b.name == "minor") - assert minor.base_url == "http://vllm-minor:8000" # falls back to default + assert minor.base_url == "http://vllm-minor:8000" + assert minor.served_name == _MINOR_SERVED def test_minor_backend_url_stripped() -> None: diff --git a/tests/test_gateway_config_wiring.py b/tests/test_gateway_config_wiring.py new file mode 100644 index 0000000..9d4ccd3 --- /dev/null +++ b/tests/test_gateway_config_wiring.py @@ -0,0 +1,133 @@ +"""``_optional_backend`` wiring contract: a backend is real only when its +``*_BASE_URL`` is set (t1, "advertised implies reachable"). + +Before this change, ``_optional_backend`` wired a backend when EITHER its +``*_BASE_URL`` OR its ``*_SERVED_NAME`` env var was set, falling back to a +hardcoded ``default_url`` naming a compose service that need not actually +exist. On the reference rig this invented two phantom backends — +``multimodal-coder`` and ``middle`` — whose containers were never started: +``GET /v1/models`` advertised them, but every request to them would fail to +connect. The fix: wire a backend only when its ``*_BASE_URL`` is non-empty, +matching the contract the fleet already documents for ``MINOR_BASE_URL`` +("empty ⇒ minor silently unwired"). +""" + +from __future__ import annotations + +from lobes.gateway._config import ( + _DEFAULT_MIDDLE, + _DEFAULT_MULTIMODAL_CODER, + _DEFAULT_PRIMARY, + build_config, +) +from lobes.gateway._routing import resolve_model + +_MIDDLE_ID = "nvidia/Qwen3-14B-NVFP4" + + +# --- criterion 1: SERVED_NAME alone (no BASE_URL) wires NEITHER backend ----- + + +def test_served_name_alone_wires_neither_middle_nor_multimodal_coder() -> None: + table, _ = build_config( + { + "MULTIMODAL_CODER_SERVED_NAME": _DEFAULT_MULTIMODAL_CODER, + "MIDDLE_SERVED_NAME": _MIDDLE_ID, + } + ) + names = [b.name for b in table.backends] + assert "middle" not in names + assert "multimodal-coder" not in names + assert names == ["primary"] # no phantom backends — primary alone + + +# --- criterion 2: a caller naming the 14B id falls back to default_model, -- +# --- never routed to a "middle" Backend object ------------------------------ + + +def test_resolve_model_for_unwired_middle_id_falls_back_to_default_model() -> None: + table, _ = build_config({"MIDDLE_SERVED_NAME": _MIDDLE_ID}) + # No backend is actually serving the 14B weights. + assert not any(b.name == "middle" for b in table.backends) + assert not any(b.served_name == _MIDDLE_ID for b in table.backends) + # The caller's request resolves to default_model (the primary's served + # name) — NOT to the 14B id, and NOT to a middle Backend object (there is + # none to route to). + assert table.default_model == _DEFAULT_PRIMARY + assert resolve_model(table, _MIDDLE_ID) == table.default_model + assert resolve_model(table, _MIDDLE_ID) == _DEFAULT_PRIMARY + + +# --- criterion 3: a *_BASE_URL that IS set still wires the backend ---------- + + +def test_middle_base_url_alone_still_wires_the_backend() -> None: + table, _ = build_config({"MIDDLE_BASE_URL": "http://vllm-middle:8000"}) + mid = next(b for b in table.backends if b.name == "middle") + assert mid.served_name == _DEFAULT_MIDDLE + assert mid.base_url == "http://vllm-middle:8000" + assert resolve_model(table, _DEFAULT_MIDDLE) == _DEFAULT_MIDDLE + + +def test_multimodal_coder_base_url_alone_still_wires_the_backend() -> None: + table, _ = build_config({"MULTIMODAL_CODER_BASE_URL": "http://vllm-multimodal-coder:8000"}) + coder = next(b for b in table.backends if b.name == "multimodal-coder") + assert coder.served_name == _DEFAULT_MULTIMODAL_CODER + assert coder.base_url == "http://vllm-multimodal-coder:8000" + + +# --- criterion 4: EMBED_URL / RERANK_URL / PRIMARY_URL are unchanged ------- +# (these are the vars the fleet template actually sets, so this is the +# steady-state / most-common-path regression guard for this change) + + +def test_primary_url_alone_still_wires_primary() -> None: + table, _ = build_config({"PRIMARY_URL": "http://vllm-primary:8000"}) + assert table.backends[0].name == "primary" + assert table.backends[0].base_url == "http://vllm-primary:8000" + + +def test_embed_url_alone_still_wires_embed_backend() -> None: + table, _ = build_config({"EMBED_URL": "http://vllm-embed:8000"}) + embed = next(b for b in table.backends if b.name == "embed") + assert embed.base_url == "http://vllm-embed:8000" + assert embed.task == "embed" + + +def test_rerank_url_alone_still_wires_rerank_backend() -> None: + table, _ = build_config({"RERANK_URL": "http://vllm-rerank:8000"}) + rerank = next(b for b in table.backends if b.name == "rerank") + assert rerank.base_url == "http://vllm-rerank:8000" + assert rerank.task == "score" + + +def test_embed_rerank_primary_urls_together_unchanged_from_before() -> None: + # The standard fleet template shape: three *_URL vars set, no *_SERVED_NAME + # overrides. Must still produce three backends, exactly as before this change + # (this scenario never depended on the dropped `or name_key` clause). + table, _ = build_config( + { + "PRIMARY_URL": "http://vllm-primary:8000", + "EMBED_URL": "http://vllm-embed:8000", + "RERANK_URL": "http://vllm-rerank:8000", + } + ) + names = {b.name for b in table.backends} + assert names == {"primary", "embed", "rerank"} + + +# --- criterion 5: enabling a profile that sets *_BASE_URL still wires it --- + + +def test_middle_profile_with_base_url_and_served_name_both_set_wires_middle() -> None: + # Mirrors what the compose "middle" profile actually sets: both vars present, + # BASE_URL doing the wiring, SERVED_NAME customising the served id. + table, _ = build_config( + { + "MIDDLE_BASE_URL": "http://vllm-middle:8000", + "MIDDLE_SERVED_NAME": _MIDDLE_ID, + } + ) + mid = next(b for b in table.backends if b.name == "middle") + assert mid.served_name == _MIDDLE_ID + assert resolve_model(table, _MIDDLE_ID) == _MIDDLE_ID diff --git a/tests/test_gateway_routing.py b/tests/test_gateway_routing.py index f9ebfc1..13915e9 100644 --- a/tests/test_gateway_routing.py +++ b/tests/test_gateway_routing.py @@ -96,8 +96,16 @@ def test_build_config_defaults_single_backend() -> None: def test_build_config_adds_fallback_only_when_configured() -> None: # No optional-backend env → just the generate primary. assert len(build_config({})[0].backends) == 1 - # FALLBACK_SERVED_NAME alone is enough to wire a second backend. + # FALLBACK_SERVED_NAME alone is NOT enough to wire a second backend — a + # served name with no URL describes a model, not a reachable backend + # (see _optional_backend's "advertised implies reachable" contract). table, _ = build_config({"FALLBACK_SERVED_NAME": "beta"}) + assert [b.name for b in table.backends] == ["primary"] + # FALLBACK_URL is what actually wires the backend; FALLBACK_SERVED_NAME + # alongside it only customises the served name. + table, _ = build_config( + {"FALLBACK_URL": "http://vllm-fallback:8000", "FALLBACK_SERVED_NAME": "beta"} + ) assert [b.name for b in table.backends] == ["primary", "fallback"] assert table.backends[1].served_name == "beta" @@ -235,12 +243,17 @@ def test_build_config_embed_url_alone_triggers_embed_backend() -> None: assert embed_backend.served_name == "Qwen/Qwen3-Embedding-0.6B" -def test_build_config_rerank_served_name_alone_triggers_rerank_backend() -> None: - # RERANK_SERVED_NAME alone → rerank backend added with default URL. +def test_build_config_rerank_served_name_alone_does_not_wire_rerank_backend() -> None: + # RERANK_SERVED_NAME alone → no rerank backend (no URL ⇒ nothing reachable). table, _ = build_config({"RERANK_SERVED_NAME": "custom/reranker"}) - assert any(b.name == "rerank" for b in table.backends) + assert not any(b.name == "rerank" for b in table.backends) + # RERANK_URL is what wires it; RERANK_SERVED_NAME alongside it customises + # the served name. + table, _ = build_config( + {"RERANK_URL": "http://vllm-rerank:9999", "RERANK_SERVED_NAME": "custom/reranker"} + ) rerank_backend = next(b for b in table.backends if b.name == "rerank") - assert rerank_backend.base_url == "http://vllm-rerank:8000" + assert rerank_backend.base_url == "http://vllm-rerank:9999" assert rerank_backend.served_name == "custom/reranker" diff --git a/tests/test_gateway_server.py b/tests/test_gateway_server.py index 5c74e7a..03987f5 100644 --- a/tests/test_gateway_server.py +++ b/tests/test_gateway_server.py @@ -18,7 +18,12 @@ def _cfg(**over): - env = {"PRIMARY_SERVED_NAME": "P", "FALLBACK_SERVED_NAME": "F", "GATEWAY_DEFAULT_MODEL": "P"} + env = { + "PRIMARY_SERVED_NAME": "P", + "FALLBACK_URL": "http://vllm-fallback:8000", + "FALLBACK_SERVED_NAME": "F", + "GATEWAY_DEFAULT_MODEL": "P", + } env.update(over) return build_config(env) diff --git a/tests/test_gateway_tiers.py b/tests/test_gateway_tiers.py index 05cf8be..8b96ae3 100644 --- a/tests/test_gateway_tiers.py +++ b/tests/test_gateway_tiers.py @@ -53,9 +53,22 @@ def test_multimodal_base_url_wires_generate_backend_with_default_name() -> None: assert mm.base_url == "http://vllm-multimodal:8000" # trailing slash stripped -def test_multimodal_served_name_alone_wires_backend() -> None: - # MULTIMODAL_SERVED_NAME alone wires it with the default URL (mirror minor). +def test_multimodal_served_name_alone_does_not_wire_backend() -> None: + # MULTIMODAL_SERVED_NAME alone does NOT wire it — no URL ⇒ nothing + # reachable (a served name with no URL only describes a model). table, _ = build_config({"MULTIMODAL_SERVED_NAME": "custom/gemma"}) + assert not any(b.name == "multimodal" for b in table.backends) + + +def test_multimodal_url_and_served_name_together_wire_backend() -> None: + # MULTIMODAL_BASE_URL wires it; MULTIMODAL_SERVED_NAME alongside it + # customises the served name. + table, _ = build_config( + { + "MULTIMODAL_BASE_URL": "http://vllm-multimodal:8000", + "MULTIMODAL_SERVED_NAME": "custom/gemma", + } + ) mm = next(b for b in table.backends if b.name == "multimodal") assert mm.base_url == "http://vllm-multimodal:8000" assert mm.served_name == "custom/gemma" @@ -92,8 +105,14 @@ def test_middle_base_url_wires_generate_backend_with_default_name() -> None: def test_middle_reachable_by_explicit_served_name() -> None: - # The 14B is addressable by its served name once the profile is active. - table, _ = build_config({"MIDDLE_SERVED_NAME": "nvidia/Qwen3-14B-NVFP4"}) + # The 14B is addressable by its served name once the profile is active + # (the profile sets MIDDLE_BASE_URL, which is what actually wires it). + table, _ = build_config( + { + "MIDDLE_BASE_URL": "http://vllm-middle:8000", + "MIDDLE_SERVED_NAME": "nvidia/Qwen3-14B-NVFP4", + } + ) assert resolve_model(table, "nvidia/Qwen3-14B-NVFP4") == "nvidia/Qwen3-14B-NVFP4" @@ -145,9 +164,15 @@ def test_multimodal_coder_alias_resolves_once_wired() -> None: def test_multimodal_coder_reachable_by_explicit_served_name() -> None: - # The coder is also addressable by its served name once the profile is active, - # same as any other wired backend. - table, _ = build_config({"MULTIMODAL_CODER_SERVED_NAME": "custom/gemma-coder"}) + # The coder is also addressable by its served name once the profile is + # active (the profile sets MULTIMODAL_CODER_BASE_URL, which is what + # actually wires it), same as any other wired backend. + table, _ = build_config( + { + "MULTIMODAL_CODER_BASE_URL": "http://vllm-multimodal-coder:8000", + "MULTIMODAL_CODER_SERVED_NAME": "custom/gemma-coder", + } + ) assert resolve_model(table, "custom/gemma-coder") == "custom/gemma-coder" @@ -244,7 +269,9 @@ def test_three_tier_aliases_resolve_to_their_gears_when_all_wired() -> None: def test_tier_aliases_track_custom_served_names() -> None: table, _ = build_config( { + "MINOR_BASE_URL": "http://vllm-minor:8000", "MINOR_SERVED_NAME": "my/minor", + "MULTIMODAL_BASE_URL": "http://vllm-multimodal:8000", "MULTIMODAL_SERVED_NAME": "my/multimodal", "PRIMARY_SERVED_NAME": "my/primary", } From 91a5099a018908b86702e4d9c16ceb1b8051db18 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 10:00:53 +0300 Subject: [PATCH 05/23] Add background readiness cache for fleet backends (t3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New module lobes/gateway/_readiness.py: a bounded, background-cached probe of each fleet backend's /health so the gateway can distinguish "answered recently" from "merely configured" (today RoleInfo.ready aliases loaded). Mirrors PressureCache's shape/naming/threading discipline: a single daemon thread refreshes an in-memory snapshot on an interval; .current() returns a copy without ever probing (O(1), socket-free on the request path). Readiness is tri-state, matching probe_audio_ready: True (reached, 200) / False (reached, non-200 e.g. warming) / None (unreachable/unknown) — False and None are not collapsed. The probe helper catches OSError, HTTPException AND ValueError (the non-numeric-port bug caught on PR #90) and degrades to None. The background thread is a daemon and stop()/close() joins it cleanly. Standalone + fully unit-tested (tests/test_gateway_readiness.py); a later task wires .current() into GET /v1/models and GET /capabilities. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/gateway/_readiness.py | 258 +++++++++++++++++++++++++ tests/test_gateway_readiness.py | 323 ++++++++++++++++++++++++++++++++ 2 files changed, 581 insertions(+) create mode 100644 lobes/gateway/_readiness.py create mode 100644 tests/test_gateway_readiness.py diff --git a/lobes/gateway/_readiness.py b/lobes/gateway/_readiness.py new file mode 100644 index 0000000..0e76e12 --- /dev/null +++ b/lobes/gateway/_readiness.py @@ -0,0 +1,258 @@ +"""Background readiness cache for the fleet backends — "answered recently" vs "configured". + +Why this module exists +---------------------- +``RoleInfo.ready`` is an alias of ``loaded`` — a *config* fact (is this backend +wired in this deployment) that its own docstring admits is not a probe. That is +why ``GET /capabilities`` can advertise ``ready: true`` for an endpoint that +404s: a backend can be configured yet not answering. This cache supplies the +missing signal — a bounded, **background** probe of each backend's ``/health`` +so the gateway can tell a backend that *answered recently* from one that is +*merely configured*. A later task wires :meth:`ReadinessCache.current` into +``GET /v1/models`` and ``GET /capabilities``; this module is that cache, +standalone and fully unit-tested. + +Tri-state, never two-state +-------------------------- +Readiness is **tri-state**, matching :func:`lobes.gateway.server.probe_audio_ready` +(issue #89): + +* ``True`` — reached the backend and it answered HTTP 200 → a request will + round-trip right now. +* ``False`` — reached the backend but it answered non-200 (e.g. 503 while the + engine warms up) → advertised, reachable, but not yet consumable. +* ``None`` — could not reach the backend at all (connection refused / timeout / + malformed URL) → readiness is *unknown*. + +``False`` and ``None`` are deliberately **not** collapsed: "reachable but +warming" and "cannot reach at all" are different operational states and a caller +(or a later ``/capabilities`` overlay) needs to distinguish them. + +Threading discipline — mirror of :class:`lobes.gateway._tier_request.PressureCache` +----------------------------------------------------------------------------------- +This mirrors the shape, naming and threading discipline of ``PressureCache``: a +single background **daemon** thread refreshes an in-memory snapshot on an +interval, and :meth:`current` only ever returns a copy of that snapshot — it +**never probes**, so a request reads readiness in O(1) with no socket and no +blocking. Consistency with ``PressureCache`` matters more than novelty. + +One deliberate divergence from ``PressureCache``: construction seeds every +backend to ``None`` (*unknown*) **without probing**, rather than sampling +synchronously. Probing at construction would open one socket per backend and +could block gateway startup for ``timeout × N`` if a backend is down; and +before the first probe completes, "unknown" is the honest readiness. The daemon +thread performs the first real probe immediately after it starts (off the +request path), so the snapshot populates promptly without ever blocking a +request or startup. + +Stdlib only — this gateway is deliberately dependency-free. +""" + +from __future__ import annotations + +import http.client +import threading +from collections.abc import Iterable, Mapping +from typing import Callable +from urllib.parse import urlsplit + +# Per-backend probe timeout: bounded so a slow/hung backend cannot stall the +# refresh thread indefinitely. Mirrors ``server._STATUS_PROBE_TIMEOUT``. +_READINESS_PROBE_TIMEOUT: float = 3.0 + +# How often the daemon thread re-probes every backend. Health changes on a +# human/warm-up timescale, so a few seconds is plenty; probing is off the +# request path, so this interval never affects request latency. +_DEFAULT_REFRESH_INTERVAL: float = 5.0 + +# vLLM serves ``/health`` unauthenticated and returns 200 only once the engine +# is live, so it is the correct readiness endpoint (same one ``_metrics`` GETs). +_HEALTH_PATH = "/health" + +# A probe maps a backend base URL to its tri-state readiness. Injectable so the +# cache is unit-testable without sockets (the ``.current()``-opens-no-socket and +# defensive-degradation properties are proved by driving this callable). +Probe = Callable[[str], "bool | None"] + +# The raw opener maps a full URL to an HTTP status code (an int). It is the ONLY +# thing that opens a socket; injecting it keeps :func:`probe_backend_ready` +# unit-testable offline. Mirror of ``server._default_ready_probe``'s signature. +Opener = Callable[[str, float], int] + + +def _default_ready_opener(url: str, timeout: float) -> int: # pragma: no cover - opens a socket + """GET *url* over plain HTTP and return the response status code. + + Fleet backends are internal-only (``http://vllm-primary:8000``), so this is + intentionally HTTP-only, matching ``server._default_ready_probe``. Accessing + ``urlsplit(url).port`` on a non-numeric port raises ``ValueError`` here — + caught by :func:`probe_backend_ready`, never by this opener. + """ + parts = urlsplit(url) + conn = http.client.HTTPConnection(parts.hostname, parts.port or 80, timeout=timeout) + try: + conn.request("GET", parts.path or "/") + return conn.getresponse().status + finally: + conn.close() + + +def probe_backend_ready( + base_url: str, + *, + timeout: float = _READINESS_PROBE_TIMEOUT, + opener: Opener | None = None, +) -> bool | None: + """Live-probe one backend's ``/health`` and map it to the readiness tri-state. + + * ``True`` — HTTP 200: the backend answered and is ready. + * ``False`` — reached the backend but it answered non-200 (e.g. warming). + * ``None`` — could not reach it at all, OR the URL was malformed → unknown. + + The ``opener`` is injected so this is unit-testable without sockets; the + default opens a bounded ``http.client`` GET. + + The ``except`` clause catches ``OSError``, ``http.client.HTTPException`` AND + ``ValueError`` and degrades to ``None``. ``ValueError`` is load-bearing: a + malformed ``base_url`` with a non-numeric port makes ``urlsplit(...).port`` + raise ``ValueError`` (this exact bug was caught in review on PR #90, where + ``probe_audio_ready`` originally caught only ``OSError``). A degrade-to-None + here can never crash the caller that will later fold this into + ``/capabilities``. + """ + get_status = opener or _default_ready_opener + try: + return get_status(base_url.rstrip("/") + _HEALTH_PATH, timeout) == 200 + except (OSError, http.client.HTTPException, ValueError): + return None + + +class ReadinessCache: + """A non-blocking, background readiness provider for the fleet's backends. + + Constructed with a mapping of ``backend name → base URL``. A single daemon + thread probes each backend's ``/health`` every ``interval`` seconds and + stores the tri-state verdicts; :meth:`current` returns a copy of the latest + snapshot without ever probing, so a request reads readiness in O(1) with no + socket and no blocking. + + The ``probe`` callable (``base_url → bool | None``) is injectable so tests + can drive fixed verdicts and never touch a socket; it defaults to + :func:`probe_backend_ready` bound to this cache's ``timeout``. A probe that + raises is swallowed per-backend (that backend degrades to ``None``) so a + transient failure can never kill the daemon thread or bubble out of a read. + + Mirrors :class:`lobes.gateway._tier_request.PressureCache` in shape, naming + and threading discipline; see the module docstring for the one deliberate + divergence (construction seeds ``None`` instead of probing). + """ + + def __init__( + self, + targets: Mapping[str, str], + *, + probe: Probe | None = None, + timeout: float = _READINESS_PROBE_TIMEOUT, + interval: float = _DEFAULT_REFRESH_INTERVAL, + start: bool = True, + ) -> None: + # Copy the targets so a caller mutating theirs cannot change what we probe. + self._targets: dict[str, str] = dict(targets) + self._timeout = timeout + self._probe: Probe = probe or self._default_probe + self._interval = interval + self._lock = threading.Lock() + self._stop = threading.Event() + self._thread: threading.Thread | None = None + # Seed every backend to unknown WITHOUT probing: construction opens no + # socket and never blocks startup on a down backend (see module docstring). + self._value: dict[str, bool | None] = {name: None for name in self._targets} + if start: + self.start() + + @classmethod + def from_backends(cls, backends: Iterable[object], **kwargs) -> "ReadinessCache": + """Build a cache from an iterable of backends (``.name`` + ``.base_url``). + + Convenience for the later wiring task: pass ``table.backends`` directly. + Duck-typed (reads only ``.name`` / ``.base_url``) so this module needs no + import of :class:`lobes.gateway._routing.Backend` and stays decoupled. + """ + targets = {b.name: b.base_url for b in backends} # type: ignore[attr-defined] + return cls(targets, **kwargs) + + def _default_probe(self, base_url: str) -> bool | None: + """The default probe: :func:`probe_backend_ready` bound to our timeout.""" + return probe_backend_ready(base_url, timeout=self._timeout) + + def _read(self) -> dict[str, bool | None]: + """Probe every backend once, degrading a raising probe to ``None``. + + Per-backend ``try`` so one misbehaving probe cannot abort the whole pass + or crash the daemon thread — the offending backend simply reads unknown. + Runs on the background thread only, never the request path. + """ + result: dict[str, bool | None] = {} + for name, base_url in self._targets.items(): + try: + result[name] = self._probe(base_url) + except Exception: # nosec B110 — readiness is best-effort; never crash the daemon + result[name] = None + return result + + def _refresh_once(self) -> None: + value = self._read() + with self._lock: + self._value = value + + def current(self) -> dict[str, bool | None]: + """Return a copy of the latest readiness snapshot. Never probes, never blocks. + + Values are ``True`` / ``False`` / ``None`` (the tri-state). The returned + dict is a fresh copy, so a caller mutating it cannot corrupt the cache. + """ + with self._lock: + return dict(self._value) + + def start(self) -> None: + """Start the background refresh thread (idempotent).""" + if self._thread is not None: + return + # Clear the stop flag so a cache restarted after stop() runs again. + self._stop.clear() + self._thread = threading.Thread( + target=self._loop, name="lobes-readiness-cache", daemon=True + ) + self._thread.start() + + def _loop(self) -> None: + # Probe once immediately so the snapshot populates promptly after start() + # (off the request path), then refresh every interval. Event.wait(interval) + # returns True only when stop() is set, so it both paces the refresh and + # exits promptly on shutdown. + self._refresh_once() + while not self._stop.wait(self._interval): + self._refresh_once() + + def stop(self) -> None: + """Signal the daemon thread to exit and join it (idempotent, clean shutdown). + + Safe to call before :meth:`start` (no thread yet). Joins with a bounded + timeout so a caller (e.g. server shutdown) gets deterministic termination + without hanging on a probe in flight. + """ + self._stop.set() + thread = self._thread + if thread is not None: + # Bound the join so shutdown cannot hang on a probe still in flight; + # the thread is a daemon, so a (pathological) straggler never blocks + # interpreter exit anyway. + thread.join(timeout=self._timeout + 1.0) + self._thread = None + + # Explicit alias — server shutdown code reads more naturally as close(). + close = stop + + def is_alive(self) -> bool: + """True while the background refresh thread is running.""" + return self._thread is not None and self._thread.is_alive() diff --git a/tests/test_gateway_readiness.py b/tests/test_gateway_readiness.py new file mode 100644 index 0000000..45fbbe9 --- /dev/null +++ b/tests/test_gateway_readiness.py @@ -0,0 +1,323 @@ +"""Tests for the gateway's background readiness cache (:mod:`lobes.gateway._readiness`). + +Task t3 of "advertised implies reachable": prove that ``ReadinessCache`` can tell +"this backend answered recently" from "this backend is merely configured", WITHOUT +ever probing on the request path. + +Two properties are the most likely to be silently violated, so each has a test +that would actually catch a regression: + +* ``.current()`` opens **no socket** — the probe callable is injected and its call + count is asserted to stay flat across many reads (and zero at construction). +* the probe helper degrades a **non-numeric port** (``urlsplit(...).port`` raises + ``ValueError``) to ``None`` instead of crashing — the exact bug caught in review + on PR #90 for ``probe_audio_ready``. + +Stdlib only, mirroring the gateway's dependency-free discipline. +""" + +from __future__ import annotations + +import http.client +import threading +import time + +from lobes.gateway import _readiness as R + +# --- probe_backend_ready: the tri-state helper ------------------------------ + + +def test_probe_backend_ready_tristate() -> None: + # 200 → True (reached, ready); non-200 → False (reached, warming); + # OSError → None (could not reach at all). None must NOT collapse into False. + assert R.probe_backend_ready("http://primary:8000", opener=lambda u, t: 200) is True + assert R.probe_backend_ready("http://primary:8000", opener=lambda u, t: 503) is False + + def boom(_u, _t): + raise OSError("connection refused") + + assert R.probe_backend_ready("http://primary:8000", opener=boom) is None + + +def test_probe_backend_ready_swallows_valueerror_and_httpexception() -> None: + """A malformed base_url (``urlsplit(...).port`` raises ``ValueError``) or a + broken HTTP exchange (``HTTPException``) must degrade to unknown (``None``), + never bubble out — this is the PR #90 regression the ValueError guard exists + for. ``probe_audio_ready`` originally caught only ``OSError``; do not repeat it. + """ + + def value_boom(_u, _t): + raise ValueError("invalid literal for int() with base 10: 'abc'") + + def http_boom(_u, _t): + raise http.client.BadStatusLine("garbage") + + assert R.probe_backend_ready("http://primary:8000", opener=value_boom) is None + assert R.probe_backend_ready("http://primary:8000", opener=http_boom) is None + + +def test_probe_backend_ready_malformed_url_no_crash_no_socket() -> None: + """The DEFAULT opener path: a non-numeric port makes ``urlsplit(...).port`` + raise ``ValueError`` *before any socket opens*, so the probe returns ``None`` + offline instead of crashing. This exercises the real opener, not an injected + one, so a regression that dropped the ``ValueError`` guard would surface here. + """ + assert R.probe_backend_ready("http://backend:notaport/") is None + + +def test_probe_backend_ready_hits_health_path() -> None: + # The probe targets the vLLM ``/health`` endpoint, with base_url trailing + # slash stripped so we never emit a double slash. + seen = {} + + def opener(url, _timeout): + seen["url"] = url + return 200 + + R.probe_backend_ready("http://primary:8000/", opener=opener) + assert seen["url"] == "http://primary:8000/health" + + +# --- ReadinessCache: seed + socket-free reads ------------------------------- + + +def test_seed_is_all_unknown_and_construction_opens_no_socket() -> None: + # Before any probe completes, readiness is honestly UNKNOWN (None) for every + # backend — and construction itself must not probe (no blocking N-socket + # fan-out at gateway startup). + calls = {"n": 0} + + def probe(_url): + calls["n"] += 1 + return True + + cache = R.ReadinessCache( + {"primary": "http://primary:8000", "minor": "http://minor:8000"}, + probe=probe, + interval=1000, + start=False, + ) + try: + assert cache.current() == {"primary": None, "minor": None} + assert calls["n"] == 0 # construction probed nothing → opened no socket + finally: + cache.stop() + + +def test_current_probe_call_count_is_zero_across_reads() -> None: + # The load-bearing property: reading the cache must never sample. Inject a + # counting probe, read N times, assert the count does not move. + calls = {"n": 0} + + def probe(_url): + calls["n"] += 1 + return True + + cache = R.ReadinessCache( + {"a": "http://a:8000", "b": "http://b:8000"}, + probe=probe, + interval=1000, + start=False, + ) + try: + before = calls["n"] + for _ in range(2000): + cache.current() + assert calls["n"] == before # reads add ZERO probe calls + assert before == 0 # and the seed itself opened no socket + finally: + cache.stop() + + +def test_current_read_path_probe_never_invoked() -> None: + # A second, independent guard: a probe that RAISES if ever called. With the + # thread stopped, neither construction nor any read may invoke it. + def probe(_url): + raise AssertionError("current() must not probe on the read path") + + cache = R.ReadinessCache({"a": "http://a:8000"}, probe=probe, interval=1000, start=False) + try: + for _ in range(2000): + assert cache.current() == {"a": None} + finally: + cache.stop() + + +def test_current_returns_a_copy_isolated_from_caller_mutation() -> None: + cache = R.ReadinessCache({"a": "http://a:8000"}, probe=lambda u: True, start=False) + try: + snapshot = cache.current() + snapshot["a"] = "corrupted" + snapshot["injected"] = True + assert cache.current() == {"a": None} + finally: + cache.stop() + + +def test_empty_targets_current_is_empty_dict() -> None: + cache = R.ReadinessCache({}, probe=lambda u: True, start=False) + try: + assert cache.current() == {} + finally: + cache.stop() + + +# --- ReadinessCache: the background daemon thread --------------------------- + + +def test_background_thread_probes_each_backend_tristate() -> None: + # Once the daemon thread completes a pass, .current() reports each backend's + # own tri-state verdict — True, False and None must all survive side by side. + targets = { + "primary": "http://primary:8000", + "minor": "http://minor:8000", + "senses": "http://senses:8000", + } + verdicts = { + "http://primary:8000": True, + "http://minor:8000": False, + "http://senses:8000": None, + } + cache = R.ReadinessCache(targets, probe=lambda u: verdicts[u], interval=0.01, start=True) + try: + expected = {"primary": True, "minor": False, "senses": None} + deadline = time.time() + 2.0 + while time.time() < deadline and cache.current() != expected: + time.sleep(0.005) + assert cache.current() == expected + finally: + cache.stop() + + +def test_background_thread_refreshes_repeatedly() -> None: + # The daemon re-probes every ``interval`` (tracks live health, not just a seed). + counter = {"n": 0} + twice = threading.Event() + + def probe(_url): + counter["n"] += 1 + if counter["n"] >= 2: + twice.set() + return True + + cache = R.ReadinessCache({"a": "http://a:8000"}, probe=probe, interval=0.01, start=True) + try: + assert twice.wait(2.0), "background thread never re-probed" + finally: + cache.stop() + + +def test_one_probe_raising_does_not_kill_the_refresh() -> None: + # criteria #6: a probe blowing up on one backend degrades THAT backend to + # None and must not stop the others being probed nor kill the daemon thread. + def probe(base_url): + if base_url == "http://bad:8000": + raise RuntimeError("boom") + return True + + targets = {"bad": "http://bad:8000", "good": "http://good:8000"} + cache = R.ReadinessCache(targets, probe=probe, interval=0.01, start=True) + try: + deadline = time.time() + 2.0 + while time.time() < deadline and cache.current().get("good") is not True: + time.sleep(0.005) + snap = cache.current() + assert snap["good"] is True # the pass ran past the raising probe + assert snap["bad"] is None # the raising probe degraded to unknown + assert cache.is_alive() # the daemon thread survived the exception + finally: + cache.stop() + + +def test_current_never_raises_and_thread_survives_chaotic_probe() -> None: + # Every probe raises; .current() must still return a dict, and the thread + # must keep looping (proved by a second refresh happening after the first raise). + counter = {"n": 0} + + def probe(_url): + counter["n"] += 1 + raise RuntimeError("chaos") + + cache = R.ReadinessCache({"a": "http://a:8000"}, probe=probe, interval=0.01, start=True) + try: + deadline = time.time() + 2.0 + while counter["n"] < 2 and time.time() < deadline: + time.sleep(0.005) + assert counter["n"] >= 2, "thread died after the first raising probe" + result = cache.current() + assert isinstance(result, dict) + assert result.get("a") is None + assert cache.is_alive() + finally: + cache.stop() + + +def test_refresh_thread_is_a_daemon() -> None: + cache = R.ReadinessCache( + {"primary": "http://primary:8000"}, probe=lambda u: True, interval=1000, start=True + ) + try: + assert cache.is_alive() + # A daemon thread never blocks interpreter shutdown. + assert cache._thread is not None and cache._thread.daemon is True + finally: + cache.stop() + + +# --- ReadinessCache: clean shutdown ----------------------------------------- + + +def test_stop_terminates_the_thread() -> None: + cache = R.ReadinessCache( + {"primary": "http://primary:8000"}, probe=lambda u: True, interval=0.01, start=True + ) + assert cache.is_alive() + cache.stop() + assert not cache.is_alive() # stop() joined the thread → it is gone + + +def test_close_is_an_alias_for_stop() -> None: + cache = R.ReadinessCache( + {"primary": "http://primary:8000"}, probe=lambda u: True, interval=0.01, start=True + ) + assert cache.is_alive() + cache.close() + assert not cache.is_alive() + + +def test_stop_is_idempotent_and_safe_before_start() -> None: + cache = R.ReadinessCache({"a": "http://a:8000"}, probe=lambda u: True, start=False) + cache.stop() # no thread yet — must not raise + cache.stop() # idempotent + assert not cache.is_alive() + + +# --- ReadinessCache.from_backends: ergonomic constructor -------------------- + + +def test_from_backends_builds_targets_from_name_and_base_url() -> None: + class _B: + def __init__(self, name, base_url): + self.name = name + self.base_url = base_url + + backends = [_B("primary", "http://primary:8000"), _B("minor", "http://minor:8000")] + cache = R.ReadinessCache.from_backends(backends, probe=lambda u: True, start=False) + try: + assert cache.current() == {"primary": None, "minor": None} + finally: + cache.stop() + + +def test_from_backends_accepts_routing_backend() -> None: + from lobes.gateway._routing import Backend + + backends = [ + Backend(name="primary", base_url="http://primary:8000", served_name="qwen"), + Backend(name="embed", base_url="http://embed:8000", served_name="emb", task="embed"), + ] + cache = R.ReadinessCache.from_backends(backends, probe=lambda u: True, start=False) + try: + assert set(cache.current()) == {"primary", "embed"} + finally: + cache.stop() From b92e234d0ab2a33e0c04e3087ae238a19429e514 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 10:04:23 +0300 Subject: [PATCH 06/23] =?UTF-8?q?spec:=20amend=20c11=20->=20c29=20?= =?UTF-8?q?=E2=80=94=20a=20defaulted=20GATEWAY=5FPUBLIC=5FURL=20reintroduc?= =?UTF-8?q?es=20#92=20for=20remote=20clients?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit c11 required the fleet compose to inject GATEWAY_PUBLIC_URL derived from the published VLLM_PORT, and for reachable_origin to prefer it over the request Host header. That is wrong: a defaulted public_url is ALWAYS set, so the Host header is never consulted, and a LAN or tunnel client GETting /capabilities is told to dial http://localhost:8001 — which on that client's machine is a different service. The fix for "the advertised endpoint points at a foreign daemon" would have reintroduced exactly that defect one rung out. c29 replaces it: explicit operator override (GATEWAY_PUBLIC_URL, for a tunnel or Host-rewriting proxy) > the origin the client actually dialed (Host header) > nothing (an empty endpoint). Never an absolute URL built from the internal GATEWAY_PORT, and never a defaulted localhost public_url. Each caller receives an origin correct for itself. The AUDIO_URL half of the compose change (issue #96) is unaffected and stands. Plan coverage repointed: c29/h25 -> t4 (compose) + t6 (origin resolver). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- ...dvertises-a-capability-it-cannot-serv.json | 22 ++++++++++++-- ...dvertises-a-capability-it-cannot-serv.json | 29 ++++++++++--------- ...-advertises-a-capability-it-cannot-serv.md | 4 +-- ...-advertises-a-capability-it-cannot-serv.md | 4 +-- 4 files changed, 39 insertions(+), 20 deletions(-) diff --git a/.devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json b/.devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json index 8fb56c4..a337d11 100644 --- a/.devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json +++ b/.devague/frames/lobes-never-advertises-a-capability-it-cannot-serv.json @@ -4,7 +4,7 @@ "schema_version": 1, "status": "exported", "created": "2026-07-09T05:14:35Z", - "updated": "2026-07-09T05:33:40Z", + "updated": "2026-07-09T07:02:42Z", "claims": [ { "id": "c1", @@ -165,12 +165,12 @@ "kind": "requirement", "text": "The gateway advertises an origin from CONFIGURED TRUTH, never inference: the fleet compose injects GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) into the gateway container, and reachable_origin prefers it over the request Host header. When neither configured truth nor a Host header is available the gateway MUST NOT fabricate an absolute URL from its internal listen port.", "origin": "llm", - "status": "confirmed", + "status": "rejected", "honesty_conditions": [ { "id": "h10", "text": "With GATEWAY_PUBLIC_URL unset AND the request carrying no Host header, the gateway emits an endpoint that is either absent/empty or explicitly relative \u2014 never an absolute URL built from GATEWAY_PORT.", - "status": "confirmed" + "status": "rejected" } ], "hard_questions": [], @@ -426,6 +426,22 @@ "honesty_conditions": [], "hard_questions": [], "links": [] + }, + { + "id": "c29", + "kind": "requirement", + "text": "The gateway resolves its advertised origin as: an explicit operator override (GATEWAY_PUBLIC_URL \u2014 for a tunnel or a Host-rewriting reverse proxy) FIRST; else the origin the client actually dialed, echoed from the request Host header; else NOTHING \u2014 an empty endpoint. It must never fabricate an absolute URL from its internal listen port (GATEWAY_PORT), and it must never default GATEWAY_PUBLIC_URL to a localhost URL, because a defaulted public_url outranks the Host header and would tell every LAN/tunnel client to dial its own loopback \u2014 reintroducing the #92 defect in a new place. Each caller therefore receives an origin correct for itself. The base fleet compose still passes AUDIO_URL (empty by default; the audio overlay supplies the real value), which is the #96 half of this change.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h25", + "text": "With GATEWAY_PUBLIC_URL unset and no Host header, the gateway emits an empty endpoint \u2014 never an absolute URL built from GATEWAY_PORT. With GATEWAY_PUBLIC_URL unset and Host='spark.local:8001', it advertises http://spark.local:8001, not http://localhost:8001. With GATEWAY_PUBLIC_URL set, it wins over any Host header.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] } ], "open_vagueness": [ diff --git a/.devague/plans/lobes-never-advertises-a-capability-it-cannot-serv.json b/.devague/plans/lobes-never-advertises-a-capability-it-cannot-serv.json index adb8042..dd4730a 100644 --- a/.devague/plans/lobes-never-advertises-a-capability-it-cannot-serv.json +++ b/.devague/plans/lobes-never-advertises-a-capability-it-cannot-serv.json @@ -5,7 +5,7 @@ "schema_version": 1, "status": "exported", "created": "2026-07-09T05:37:28Z", - "updated": "2026-07-09T05:44:47Z", + "updated": "2026-07-09T07:03:39Z", "targets": [ { "id": "c1", @@ -97,16 +97,6 @@ "kind": "honesty", "text": "Each of the five success checks is a command a reviewer can paste and watch fail on main, then pass on the branch." }, - { - "id": "c11", - "kind": "requirement", - "text": "The gateway advertises an origin from CONFIGURED TRUTH, never inference: the fleet compose injects GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) into the gateway container, and reachable_origin prefers it over the request Host header. When neither configured truth nor a Host header is available the gateway MUST NOT fabricate an absolute URL from its internal listen port." - }, - { - "id": "h10", - "kind": "honesty", - "text": "With GATEWAY_PUBLIC_URL unset AND the request carrying no Host header, the gateway emits an endpoint that is either absent/empty or explicitly relative \u2014 never an absolute URL built from GATEWAY_PORT." - }, { "id": "c12", "kind": "requirement", @@ -201,6 +191,16 @@ "id": "h22", "kind": "honesty", "text": "Run against the rig as it stands right now (gateway 0.36.0, CLI 0.39.0, endpoint :8000, /v1/audio/speech 404ing), the gate exits non-zero and names all three faults; after redeploy it exits zero." + }, + { + "id": "c29", + "kind": "requirement", + "text": "The gateway resolves its advertised origin as: an explicit operator override (GATEWAY_PUBLIC_URL \u2014 for a tunnel or a Host-rewriting reverse proxy) FIRST; else the origin the client actually dialed, echoed from the request Host header; else NOTHING \u2014 an empty endpoint. It must never fabricate an absolute URL from its internal listen port (GATEWAY_PORT), and it must never default GATEWAY_PUBLIC_URL to a localhost URL, because a defaulted public_url outranks the Host header and would tell every LAN/tunnel client to dial its own loopback \u2014 reintroducing the #92 defect in a new place. Each caller therefore receives an origin correct for itself. The base fleet compose still passes AUDIO_URL (empty by default; the audio overlay supplies the real value), which is the #96 half of this change." + }, + { + "id": "h25", + "kind": "honesty", + "text": "With GATEWAY_PUBLIC_URL unset and no Host header, the gateway emits an empty endpoint \u2014 never an absolute URL built from GATEWAY_PORT. With GATEWAY_PUBLIC_URL unset and Host='spark.local:8001', it advertises http://spark.local:8001, not http://localhost:8001. With GATEWAY_PUBLIC_URL set, it wins over any Host header." } ], "tasks": [ @@ -271,7 +271,8 @@ "deps": [], "covers": [ "c11", - "c18" + "c18", + "c29" ] }, { @@ -325,7 +326,9 @@ "h13", "h10", "h14", - "h19" + "h19", + "c29", + "h25" ] }, { diff --git a/docs/plans/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md b/docs/plans/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md index 54ecc99..6f4fdbc 100644 --- a/docs/plans/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md +++ b/docs/plans/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md @@ -36,7 +36,7 @@ slug: `lobes-never-advertises-a-capability-it-cannot-serv` · status: `exported` ### t4 — Fleet template truth: inject GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) and AUDIO_URL into the gateway container from the BASE fleet compose, so the advertised origin is configured truth and stt/tts stop advertising a 404 path (issue #96) -- covers: c11, c18 +- covers: c11, c18, c29 - acceptance: - lobes/templates/fleet/docker-compose.yml gateway service passes GATEWAY_PUBLIC_URL defaulted from VLLM_PORT, and AUDIO_URL, without requiring the -f docker-compose.audio.yml overlay - A test parses the packaged fleet compose and asserts the gateway environment contains both keys @@ -57,7 +57,7 @@ slug: `lobes-never-advertises-a-capability-it-cannot-serv` · status: `exported` ### t6 — Gateway core: dead owner yields a retryable 503, the readiness cache feeds /v1/models + /capabilities.ready, and reachable_origin prefers configured truth over Host-header inference - depends on: t1, t2, t3, t5 -- covers: c4, h4, c5, h5, c12, h23, c14, h13, h10, h14, h19 +- covers: c4, h4, c5, h5, c12, h23, c14, h13, h10, h14, h19, c29, h25 - acceptance: - handle_post: an owner that refuses, times out, or returns >=500 yields HTTP 503 with a Retry-After header and an OpenAI-shaped error whose type (e.g. backend_unavailable) is distinguishable from both 'model unknown' and the all-backends-down 502 - A fake-fleet test proves a request naming the cortex model with the primary dead NEVER opens a connection to the multimodal backend (upstream-opener call sites asserted) diff --git a/docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md b/docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md index 2dd91d0..c7c523e 100644 --- a/docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md +++ b/docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md @@ -21,8 +21,6 @@ ## Requirements -- The gateway advertises an origin from CONFIGURED TRUTH, never inference: the fleet compose injects GATEWAY_PUBLIC_URL (derived from the published VLLM_PORT) into the gateway container, and reachable_origin prefers it over the request Host header. When neither configured truth nor a Host header is available the gateway MUST NOT fabricate an absolute URL from its internal listen port. - - honesty: With GATEWAY_PUBLIC_URL unset AND the request carrying no Host header, the gateway emits an endpoint that is either absent/empty or explicitly relative — never an absolute URL built from GATEWAY_PORT. - A model id returned by GET /v1/models never receives a 404 'model does not exist' from POST /v1/chat/completions on the same origin. This is the single invariant issue #91 asks for, stated as a test. - honesty: The listed-never-404 invariant is tested at the RACE, not at rest: a fake fleet lists model M, the owner is then killed, and a completion naming M returns 503 + Retry-After — never 404. A second test asserts the converse: an id that was NEVER in /v1/models does not silently get served by the default backend under a different model's weights. - When the owner backend of a requested model is dead, unreachable, or warming, and no legal failover exists, the gateway returns 503 + Retry-After with a distinguishable error type (e.g. type=backend_unavailable), never 404 and never 502-as-terminal. @@ -38,6 +36,8 @@ - honesty: A unit test with a fake fleet proves that a request naming the cortex model, with the primary backend dead, NEVER opens a connection to the multimodal backend, and returns 503 + Retry-After. order_backends(table, served) returns exactly one backend for every input. - The pre-PR gate detects DEPLOYED-ARTIFACT skew, not just source correctness: it compares the running gateway container's lobes.__version__ against the CLI's and fails on mismatch, and it dials every advertised role endpoint+path. This is the check whose absence let #87's fix ship in 0.38.0 while the rig kept running 0.36.0 — making #92 look like a code regression when the code was already correct and merely undeployed. - honesty: Run against the rig as it stands right now (gateway 0.36.0, CLI 0.39.0, endpoint :8000, /v1/audio/speech 404ing), the gate exits non-zero and names all three faults; after redeploy it exits zero. +- The gateway resolves its advertised origin as: an explicit operator override (GATEWAY_PUBLIC_URL — for a tunnel or a Host-rewriting reverse proxy) FIRST; else the origin the client actually dialed, echoed from the request Host header; else NOTHING — an empty endpoint. It must never fabricate an absolute URL from its internal listen port (GATEWAY_PORT), and it must never default GATEWAY_PUBLIC_URL to a localhost URL, because a defaulted public_url outranks the Host header and would tell every LAN/tunnel client to dial its own loopback — reintroducing the #92 defect in a new place. Each caller therefore receives an origin correct for itself. The base fleet compose still passes AUDIO_URL (empty by default; the audio overlay supplies the real value), which is the #96 half of this change. + - honesty: With GATEWAY_PUBLIC_URL unset and no Host header, the gateway emits an empty endpoint — never an absolute URL built from GATEWAY_PORT. With GATEWAY_PUBLIC_URL unset and Host='spark.local:8001', it advertises http://spark.local:8001, not http://localhost:8001. With GATEWAY_PUBLIC_URL set, it wins over any Host header. ## Honesty conditions From a1839d80eb18d96899527bf25b92625bba220fae Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 10:09:03 +0300 Subject: [PATCH 07/23] =?UTF-8?q?fix:=20GATEWAY=5FPUBLIC=5FURL=20stays=20e?= =?UTF-8?q?mpty=20by=20default=20(amended=20c29=20=E2=80=94=20#92=20/=20#9?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Amends the previous commit's GATEWAY_PUBLIC_URL half. The c11 requirement it implemented (default the origin from the published VLLM_PORT) was defective: a defaulted public_url reintroduces the #92 defect. reachable_origin prefers a set public_url OVER the request Host header, so any localhost/published-port default advertises loopback to every remote client — a LAN/tunnel caller dialing spark.local:8001 is told to use http://localhost:8001, a foreign service on their machine. Amended requirement (c29): origin precedence is explicit operator override > Host header > empty. GATEWAY_PUBLIC_URL exists ONLY as an operator override for a tunnel / Host-rewriting reverse proxy; it must not be defaulted to a localhost URL. So: - docker-compose.yml: GATEWAY_PUBLIC_URL back to ${GATEWAY_PUBLIC_URL:-}, with a load-bearing comment explaining WHY it must stay empty (a defaulted public_url outranks the Host header) so it is not "helpfully" restored later. - env.example: document GATEWAY_PUBLIC_URL as the tunnel / proxy override only; empty makes the gateway echo the origin the client dialed. No published-port default claim. - test_fleet_template_gateway_env.py: invert the ${VLLM_PORT}-default assertions. Now asserts the default is exactly ${GATEWAY_PUBLIC_URL:-} AND the negative regression guard — the default contains neither 'localhost' nor 'VLLM_PORT'. Kept the ports: mapping test (documents the published-vs-internal port distinction that is the root cause). The AUDIO_URL=${AUDIO_URL:-} half (the #96 fix) is unchanged. Re-verified with `docker compose config`: GATEWAY_PUBLIC_URL stays empty even with VLLM_PORT=8001 set, operator overrides win, and the --audio overlay still supplies AUDIO_URL while leaving GATEWAY_PUBLIC_URL empty. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/templates/fleet/docker-compose.yml | 16 ++- lobes/templates/fleet/env.example | 14 +-- tests/test_fleet_template_gateway_env.py | 122 ++++++++++++++--------- 3 files changed, 91 insertions(+), 61 deletions(-) diff --git a/lobes/templates/fleet/docker-compose.yml b/lobes/templates/fleet/docker-compose.yml index 11bc4de..2324018 100644 --- a/lobes/templates/fleet/docker-compose.yml +++ b/lobes/templates/fleet/docker-compose.yml @@ -618,11 +618,17 @@ services: - vllm-rerank environment: - GATEWAY_PORT=8000 - # Defaults to the PUBLISHED host port (VLLM_PORT, below), not the - # internal container port — configured truth, not Host-header inference - # (issue #96). Set GATEWAY_PUBLIC_URL explicitly to override for a - # tunnel / Host-rewriting reverse proxy (still wins over this default). - - GATEWAY_PUBLIC_URL=${GATEWAY_PUBLIC_URL:-http://localhost:${VLLM_PORT:-8000}} + # MUST default to EMPTY (issue #92 / #96). GATEWAY_PUBLIC_URL is an + # operator override ONLY — a tunnel URL or a Host-rewriting reverse + # proxy. Do NOT default it to a localhost/published-port URL: reachable_origin + # prefers a set public_url OVER the request Host header, so ANY default + # here (e.g. http://localhost:${VLLM_PORT}) would advertise loopback to + # every remote client in GET /capabilities — a LAN/tunnel caller dialing + # spark.local:8001 would be told to use http://localhost:8001, a foreign + # service on THEIR machine (the #92 defect). Empty → the gateway echoes + # the origin the client actually dialed (its Host header); with neither, + # it emits an empty endpoint, never one built from the internal GATEWAY_PORT. + - GATEWAY_PUBLIC_URL=${GATEWAY_PUBLIC_URL:-} # Audio/realtime backend for /v1/audio/*. Empty by default — this base # template has no realtime container; the --audio overlay # (docker-compose.audio.yml) supplies the real http://realtime:8080 diff --git a/lobes/templates/fleet/env.example b/lobes/templates/fleet/env.example index ff0f1c9..f0e8f6b 100644 --- a/lobes/templates/fleet/env.example +++ b/lobes/templates/fleet/env.example @@ -226,13 +226,13 @@ MULTIMODAL_CODER_CUDAGRAPH_ESTIMATE=0 # is reachable only on the internal compose network (vllm-primary:8000). VLLM_PORT=8000 # GATEWAY_PUBLIC_URL — OPTIONAL override for a tunnel / Host-rewriting reverse -# proxy (e.g. a cloudflared tunnel URL like https://foo.example). Set it when -# clients dial a different origin than the one you published below. -# Leave unset/empty and it defaults to http://localhost:${VLLM_PORT} — the -# published host port (issue #96; previously an empty default let the -# advertised /capabilities origin fall back to inferring from the incoming -# request Host header, and when no Host header was present, fabricated a URL -# from the gateway's INTERNAL container port instead of the published one). +# proxy ONLY (e.g. a cloudflared tunnel URL like https://foo.example). Set it +# only when clients dial a different origin than the one they connect to. +# Leave it EMPTY (the default) for a normal published host port: the gateway +# then echoes the origin the client actually dialed (its request Host header) +# in GET /capabilities, so the advertised endpoint is reachable as-is. Do NOT +# set it to a localhost/published-port URL — a set value outranks the Host +# header and would advertise loopback to every remote client (issue #92 / #96). # GATEWAY_PUBLIC_URL= GATEWAY_DEFAULT_MODEL=sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP # Extra name -> served-name routes, comma-separated (e.g. diff --git a/tests/test_fleet_template_gateway_env.py b/tests/test_fleet_template_gateway_env.py index ac6b4db..1c7837b 100644 --- a/tests/test_fleet_template_gateway_env.py +++ b/tests/test_fleet_template_gateway_env.py @@ -1,19 +1,10 @@ """The base fleet compose must pass GATEWAY_PUBLIC_URL and AUDIO_URL to the -gateway container WITHOUT the ``-f docker-compose.audio.yml`` overlay (issue -#96). +gateway container WITHOUT the ``-f docker-compose.audio.yml`` overlay +(issues #92 / #96). Two bugs this guards against: - (a) GATEWAY_PUBLIC_URL defaulted to empty (``${GATEWAY_PUBLIC_URL:-}``), so - the advertised /capabilities origin fell back to inferring from the - request Host header — and when no Host header is present, fabricated an - absolute URL from the gateway's INTERNAL listen port - (GATEWAY_PORT=8000), not the published host port (VLLM_PORT, default - 8001 on the reference rig). The default must instead be built from the - published ``VLLM_PORT`` mapping (``"${VLLM_PORT:-8000}:8000"``), so the - advertised origin is configured truth. - - (b) AUDIO_URL only reached the gateway via docker-compose.audio.yml's + (a) AUDIO_URL only reached the gateway via docker-compose.audio.yml's override block. On a base-only deployment (no --audio overlay), ServerConfig.audio_url is empty and POST /v1/audio/speech 404s, while `lobes capabilities` (reading the merged .env) reports stt/tts as @@ -21,7 +12,27 @@ environment, defaulted to EMPTY (``${AUDIO_URL:-}``) — the audio overlay is what supplies the real ``http://realtime:8080`` value; a base-only deployment must resolve audio_url to unset, not a URL of a container - that was never started. + that was never started (issue #96). + + (b) GATEWAY_PUBLIC_URL must default to EMPTY. It is an operator override + ONLY — a tunnel URL or a Host-rewriting reverse proxy. It must NOT be + defaulted to a localhost/published-port URL, because + ``lobes.gateway.server.reachable_origin`` prefers a set ``public_url`` + OVER the request Host header:: + + if public_url: return public_url.rstrip("/") # a default here wins + if host_header: return f"{scheme}://{host_header}" + return None + + So ANY default (e.g. ``http://localhost:${VLLM_PORT}``) would advertise + loopback to every remote client in GET /capabilities: a LAN/tunnel + caller dialing ``spark.local:8001`` would be told to dial + ``http://localhost:8001`` — a foreign service on THEIR machine. That is + the #92 defect (advertised endpoint points at a foreign daemon). The + correct precedence is: explicit operator override > request Host header + > empty (never an internal-port-derived URL). The compose side owns + keeping the default empty; the resolver side is enforced elsewhere + (plan task t6). """ from __future__ import annotations @@ -67,62 +78,52 @@ def test_audio_url_key_present(self) -> None: assert "AUDIO_URL" in env -class TestGatewayPublicUrlDefaultsFromVllmPort: - """The GATEWAY_PUBLIC_URL default must be built from the published - VLLM_PORT mapping (``"${VLLM_PORT:-8000}:8000"``), not left empty.""" +class TestGatewayPublicUrlDefaultsEmpty: + """GATEWAY_PUBLIC_URL must default to EMPTY — it is an operator override + only (a tunnel / Host-rewriting reverse proxy). A defaulted value would + outrank the request Host header and advertise loopback to remote clients + (the #92 defect).""" - def test_default_is_not_empty(self) -> None: + def test_default_is_exactly_empty(self) -> None: env = _gateway_env_map() - value = env["GATEWAY_PUBLIC_URL"] - # The old, buggy form: an empty default that falls through to - # Host-header inference (and, absent a Host header, the internal port). - assert value != "${GATEWAY_PUBLIC_URL:-}", ( - "GATEWAY_PUBLIC_URL must not default to empty — an empty default " - "lets the advertised origin fall back to the gateway's internal " - "listen port when no Host header is present (issue #96)" + assert env["GATEWAY_PUBLIC_URL"] == "${GATEWAY_PUBLIC_URL:-}", ( + "GATEWAY_PUBLIC_URL must default to empty (${GATEWAY_PUBLIC_URL:-}) " + "— any non-empty default outranks the request Host header in " + "reachable_origin and advertises the wrong origin to remote " + "clients (issue #92)" ) - def test_default_references_vllm_port(self) -> None: + def test_default_has_no_localhost_or_vllm_port(self) -> None: + # THE regression guard: the amended requirement (c29) forbids a + # localhost/published-port default. If someone "helpfully" restores a + # ${VLLM_PORT}-derived default, this fails. Neither token may appear. env = _gateway_env_map() value = env["GATEWAY_PUBLIC_URL"] - assert "${VLLM_PORT" in value, ( - "GATEWAY_PUBLIC_URL's default must be derived from ${VLLM_PORT:-...} " - f"(the published host port) — got {value!r}" + assert "localhost" not in value, ( + "GATEWAY_PUBLIC_URL default must not contain 'localhost' — a " + "loopback default is advertised to remote clients (issue #92)" + ) + assert "VLLM_PORT" not in value, ( + "GATEWAY_PUBLIC_URL default must not be derived from VLLM_PORT — " + "it is an operator override only, defaulted empty (issue #92)" ) - def test_still_overridable_by_operator(self) -> None: + def test_still_operator_overridable(self) -> None: # The var name itself must still gate on GATEWAY_PUBLIC_URL, so an - # operator-set GATEWAY_PUBLIC_URL in .env continues to win (a tunnel / - # Host-rewriting reverse proxy). + # operator-set GATEWAY_PUBLIC_URL in .env still wins (a tunnel / + # Host-rewriting reverse proxy) — the override path is preserved. env = _gateway_env_map() value = env["GATEWAY_PUBLIC_URL"] assert value.startswith( "${GATEWAY_PUBLIC_URL:-" ), f"GATEWAY_PUBLIC_URL must remain operator-overridable — got {value!r}" - def test_ports_mapping_is_the_vllm_port_default_this_test_assumes(self) -> None: - # Guard the assumption this whole class is built on: the published - # mapping is "${VLLM_PORT:-8000}:8000". If that ever changes, this - # test (and the GATEWAY_PUBLIC_URL default it drives) must be revisited. - compose = _load_fleet() - ports = compose["services"]["gateway"]["ports"] - assert "${VLLM_PORT:-8000}:8000" in ports - - def test_rendered_default_matches_published_port_shape(self) -> None: - # Exact string check on the template as documented in the task: the - # nested-default form composing GATEWAY_PUBLIC_URL from VLLM_PORT. - env = _gateway_env_map() - assert ( - env["GATEWAY_PUBLIC_URL"] - == "${GATEWAY_PUBLIC_URL:-http://localhost:${VLLM_PORT:-8000}}" - ) - class TestAudioUrlDefaultsEmpty: """AUDIO_URL must default to EMPTY on the base template — the audio overlay (docker-compose.audio.yml) supplies the real http://realtime:8080 value; a base-only deployment has no realtime - container to point at.""" + container to point at (issue #96).""" def test_default_is_empty_not_a_url(self) -> None: env = _gateway_env_map() @@ -131,3 +132,26 @@ def test_default_is_empty_not_a_url(self) -> None: "default (e.g. http://realtime:8080) would advertise a realtime " "container that a base-only (no --audio) deployment never starts" ) + + def test_still_operator_overridable(self) -> None: + # The audio overlay overrides AUDIO_URL by re-declaring the key; the + # base default must still gate on ${AUDIO_URL:-...} so an operator or + # the overlay can supply the real value. + env = _gateway_env_map() + assert env["AUDIO_URL"].startswith("${AUDIO_URL:-") + + +class TestPortsMappingDistinguishesPublishedFromInternal: + """The published host port (VLLM_PORT) differs from the gateway's internal + container port (GATEWAY_PORT=8000). This distinction is what makes a + localhost:GATEWAY_PORT default wrong on a rig that publishes elsewhere + (e.g. :8001) — kept as documentation of the root cause.""" + + def test_ports_mapping_uses_vllm_port_published_to_internal_8000(self) -> None: + compose = _load_fleet() + ports = compose["services"]["gateway"]["ports"] + assert "${VLLM_PORT:-8000}:8000" in ports, ( + "gateway must publish ${VLLM_PORT:-8000} -> internal 8000; the " + "published port can differ from the internal one, which is exactly " + "why GATEWAY_PUBLIC_URL must not be built from the internal port" + ) From 6d0bce7d448b7f470953d0f67f85607ade33cb7d Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 10:11:30 +0300 Subject: [PATCH 08/23] Remove cross-backend failover from order_backends (t2, closes #91) order_backends() previously walked every same-task generate backend as a failover chain (e.g. cortex -> multimodal). A dead cortex vLLM engine got silently retried against the Gemma backend with a body still naming the Qwen model id, producing a terminal 404 that killed long-running agent loops -- or worse, a real answer from the wrong model, violating the final_authority role contract (#81). order_backends(table, served) now always returns a list of length <= 1: the served name's owning backend, or the default_model's owner for an unknown name. No runtime cross-backend retry, ever. The static tier-alias upward fallback in tier_aliases() is unrelated and preserved -- that resolves an unwired capability tier at table-build time, before order_backends runs. Inverted tests to the new contract (renamed, not deleted, to keep the bug's history legible): - test_gateway_routing.py: test_order_backends_generate_still_failovers_ between_generate_backends -> test_order_backends_generate_never_failovers_ across_models; test_order_backends_owner_first_then_failover -> test_order_backends_owner_only_no_failover; added test_order_backends_always_returns_at_most_one_backend, test_order_backends_known_served_returns_its_own_owner, test_order_backends_unknown_served_returns_default_owner. - test_fleet_minor.py: test_order_backends_minor_is_owner_with_primary_ failover -> test_order_backends_minor_is_owner_with_no_failover; the "primary failover includes minor" test -> test_primary_never_failovers_ to_minor_when_minor_present. - test_gateway_server.py: test_failover_on_connection_refused -> test_no_failover_on_connection_refused; test_failover_on_5xx -> test_no_failover_on_5xx; test_explicit_fallback_routes_to_fallback_first -> test_explicit_fallback_routes_to_fallback_only; updated test_all_backends_down_returns_502 for the single-attempt attempts list. All still assert the existing 502 upstream_unavailable (handle_post's 502->503+Retry-After conversion is t6's scope, not touched here). lobes/gateway/server.py, _config.py, _readiness.py, and roles.py are untouched -- handle_post's existing for-loop over order_backends() already degrades correctly to a single attempt. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/gateway/_routing.py | 45 ++++++++++++++------ tests/test_fleet_minor.py | 23 ++++++----- tests/test_gateway_routing.py | 78 ++++++++++++++++++++++++++++++----- tests/test_gateway_server.py | 41 +++++++++++++----- 4 files changed, 145 insertions(+), 42 deletions(-) diff --git a/lobes/gateway/_routing.py b/lobes/gateway/_routing.py index bdf8ff6..ad4d50a 100644 --- a/lobes/gateway/_routing.py +++ b/lobes/gateway/_routing.py @@ -120,24 +120,45 @@ def _backend_for(table: RoutingTable, served_name: str) -> Backend | None: def order_backends(table: RoutingTable, served_name: str) -> list[Backend]: - """Attempt order for ``served_name``: its owner first, then same-task failovers. - - The owner is tried first; failover candidates are restricted to backends - with the same ``task`` as the owner. This prevents an embed request from - falling over to a generate backend (which would return a confusing 400 for - ``/v1/embeddings``). An unmatched ``served_name`` falls back to the default - model's owner (a generate backend) with same-task (generate) failovers. + """Resolve ``served_name`` to its single owning backend — never a failover chain. + + Returns a list of length 0 or 1. **No cross-backend failover, ever** (issue + #91, "advertised implies reachable"): a request that resolves to one model + is attempted at that model's owner only, never retried against a different + backend serving a different model. + + This used to walk every other backend that shared the owner's ``task`` as a + failover chain — e.g. cortex (primary) falling over to the multimodal + (Gemma) backend when the vLLM engine died. That is unsound: the retry still + carries the *original* body, which still names the original model (cortex's + Qwen id). A backend that does not serve that model has exactly one honest + answer — an OpenAI-shaped 404 ``model does not exist`` — and that 404 is + **indistinguishable to the caller** from "this model id was never valid". + ``handle_post``'s own rule ("2xx or 4xx → commit to this backend; 4xx is a + client error, no failover") then relays that 404 as terminal, silently + killing the request instead of surfacing the real problem (the owner's + engine crashed). Worse, if the other backend's model *did* happen to exist, + the caller would get a real answer from the wrong model — a `final_authority` + role-contract violation (issue #81): a caller who asked for cortex must never + silently receive a Gemma answer. + So: one served name resolves to exactly one backend, tried once. If that + backend is unreachable or errors, the caller gets an honest failure instead + of an answer from a model they did not ask for. (The *static* tier-alias + upward fallback in :func:`tier_aliases` is unrelated and unaffected — that + resolves an unwired capability tier to a different served name at + table-build time, before ``order_backends`` ever runs; it is config-time + resolution, not a runtime retry against a mismatched body.) + + An unmatched ``served_name`` still falls back to the ``default_model``'s + owner (preserves the existing "unknown model routes to default" behaviour) + — that remains a single backend, not a chain. """ owner = _backend_for(table, served_name) or _backend_for(table, table.default_model) - ordered: list[Backend] = [] # Invariant: a built table always has a primary backend and default_model # resolves to it, so owner is non-None in practice. We degrade gracefully (an # empty list → handle_post returns a 502) rather than assert, so a malformed # table can never crash the long-lived gateway process. - if owner is not None: - ordered.append(owner) - ordered.extend(b for b in table.backends if b is not owner and b.task == owner.task) - return ordered + return [owner] if owner is not None else [] def list_models_payload(table: RoutingTable) -> dict: diff --git a/tests/test_fleet_minor.py b/tests/test_fleet_minor.py index ef3f569..34e3cc7 100644 --- a/tests/test_fleet_minor.py +++ b/tests/test_fleet_minor.py @@ -86,24 +86,27 @@ def test_resolve_model_primary_unaffected_when_minor_present() -> None: assert resolve_model(table, _PRIMARY_SERVED) == _PRIMARY_SERVED -def test_order_backends_minor_is_owner_with_primary_failover() -> None: - """order_backends: minor is tried first; primary is the generate failover.""" +def test_order_backends_minor_is_owner_with_no_failover() -> None: + """order_backends: minor is the sole entry for its own served name — INVERTED + for issue #91 ("advertised implies reachable"): minor no longer fails over + to primary (or anywhere else). A request naming the minor served name is + attempted at minor, once, full stop.""" table, _ = build_config({"MINOR_BASE_URL": "http://vllm-minor:8000"}) result = order_backends(table, _MINOR_SERVED) names = [b.name for b in result] - assert names[0] == "minor" - assert "primary" in names # same-task (generate) failover - # All failover backends must be generate tasks. - assert all(b.task == "generate" for b in result) + assert names == ["minor"] + assert "primary" not in names # no cross-backend failover, ever -def test_primary_failover_includes_minor_when_minor_present() -> None: - """If minor is configured, order_backends for primary includes minor as failover.""" +def test_primary_never_failovers_to_minor_when_minor_present() -> None: + """INVERTED for issue #91: even with minor wired, order_backends for the + primary served name returns primary alone — minor is never a failover + candidate for a request that named the primary model.""" table, _ = build_config({"MINOR_BASE_URL": "http://vllm-minor:8000"}) result = order_backends(table, _PRIMARY_SERVED) names = [b.name for b in result] - assert names[0] == "primary" - assert "minor" in names # minor is also generate → in the failover pool + assert names == ["primary"] + assert "minor" not in names def test_minor_does_not_pollute_embed_failover_chain() -> None: diff --git a/tests/test_gateway_routing.py b/tests/test_gateway_routing.py index 13915e9..be5d178 100644 --- a/tests/test_gateway_routing.py +++ b/tests/test_gateway_routing.py @@ -42,12 +42,65 @@ def test_resolve_model_exact_alias_default() -> None: # --- order_backends ------------------------------------------------------- -def test_order_backends_owner_first_then_failover() -> None: +def test_order_backends_owner_only_no_failover() -> None: + # No cross-backend failover (issue #91): each served name resolves to + # exactly its own owner, never to another backend serving a different model. t = _table() - assert [b.name for b in order_backends(t, "P")] == ["primary", "fallback"] - assert [b.name for b in order_backends(t, "F")] == ["fallback", "primary"] - # an unmatched served name falls back to the default model's owner first - assert [b.name for b in order_backends(t, "nope")] == ["primary", "fallback"] + assert [b.name for b in order_backends(t, "P")] == ["primary"] + assert [b.name for b in order_backends(t, "F")] == ["fallback"] + # an unmatched served name falls back to the default model's owner — still + # a single backend, not a chain. + assert [b.name for b in order_backends(t, "nope")] == ["primary"] + + +def test_order_backends_always_returns_at_most_one_backend() -> None: + # The blanket contract (issue #91): order_backends(table, served) has + # length <= 1 for EVERY input — a known served name, an unknown one, an + # embed name, a rerank name, or a tier-alias-resolved name. No caller of + # order_backends should ever have to reason about a failover chain again. + embed_name = "Qwen/Qwen3-Embedding-0.6B" + rerank_name = "Qwen/Qwen3-Reranker-0.6B" + full = RoutingTable( + backends=( + Backend("primary", "http://vllm-primary:8000", "P"), + Backend("fallback", "http://vllm-fallback:8000", "F"), + Backend("embed", "http://vllm-embed:8000", embed_name, task="embed"), + Backend("rerank", "http://vllm-rerank:8000", rerank_name, task="score"), + ), + default_model="P", + aliases={"fast": "F", "main": "P"}, # a tier-style alias resolving to "P" + ) + for requested in ("P", "F", embed_name, rerank_name, "unknown-model", "", None): + served = resolve_model(full, requested) + result = order_backends(full, served) + assert len(result) <= 1, f"{requested!r} -> {served!r} produced {result!r}" + # A tier alias resolved first through resolve_model must also land on + # exactly one backend. + tier_served = resolve_model(full, "main") + assert len(order_backends(full, tier_served)) <= 1 + # Sanity: known names still resolve to a non-empty (length-1) result — the + # <= 1 bound above isn't vacuously satisfied by empty lists for real models. + assert len(order_backends(full, "P")) == 1 + assert len(order_backends(full, embed_name)) == 1 + assert len(order_backends(full, rerank_name)) == 1 + + +def test_order_backends_known_served_returns_its_own_owner() -> None: + # Acceptance criterion #2: for a known served name, order_backends returns + # exactly that model's owning backend (not some other backend). + t = _table() + assert [b.name for b in order_backends(t, "P")] == ["primary"] + assert [b.name for b in order_backends(t, "F")] == ["fallback"] + + +def test_order_backends_unknown_served_returns_default_owner() -> None: + # Acceptance criterion #3: an unknown served name routes to the + # default_model's owner (preserving today's "unknown model -> default" + # behaviour), still a single element. + t = _table() + result = order_backends(t, "totally-unknown-model-id") + assert [b.name for b in result] == ["primary"] + assert len(result) == 1 def test_list_models_payload_shape() -> None: @@ -305,14 +358,19 @@ def test_order_backends_rerank_returns_only_rerank_backend() -> None: assert all(b.task == "score" for b in result) -def test_order_backends_generate_still_failovers_between_generate_backends() -> None: - # The generate failover contract must not regress: primary owns "P", then - # falls over to fallback (also generate), but NOT to embed or rerank. +def test_order_backends_generate_never_failovers_across_models() -> None: + # INVERTED for issue #91 ("advertised implies reachable"): order_backends + # used to fail a generate request over to every other same-task backend + # (e.g. cortex -> multimodal), which meant a dead cortex engine got silently + # retried against the Gemma backend with a body still naming the Qwen model + # id -> a terminal, confusing 404 (or worse, a real answer from the wrong + # model). Now: "P" resolves to primary and ONLY primary, full stop. Fallback + # and embed/rerank are never attempted for a "P" request. t = _full_table() result = order_backends(t, "P") names = [b.name for b in result] - assert names == ["primary", "fallback"] - # Embed and rerank must be absent from the generate failover chain. + assert names == ["primary"] + assert "fallback" not in names assert "embed" not in names assert "rerank" not in names diff --git a/tests/test_gateway_server.py b/tests/test_gateway_server.py index 03987f5..947edbb 100644 --- a/tests/test_gateway_server.py +++ b/tests/test_gateway_server.py @@ -68,20 +68,32 @@ def opener(backend, path, body, headers, *, connect_timeout, read_timeout): # --- handle_post: failover / default / rewrite (no sockets) --------------- -def test_failover_on_connection_refused() -> None: +def test_no_failover_on_connection_refused() -> None: + # INVERTED for issue #91 ("advertised implies reachable"): a dead primary + # must NOT retry against fallback (which serves a different model — the + # forwarded body still names "P", and fallback would either 404 on an + # unknown id or, worse, silently answer as the wrong model). order_backends + # now yields only the owner, so handle_post attempts primary once and stops. + # Interim: this surfaces as the existing 502 upstream_unavailable path (t6 / + # issue #91 converts that to 503 + Retry-After with a distinguishable error + # type — not this task's file, lobes/gateway/server.py is out of scope here). table, cfg = _cfg() opener, calls = _opener({"primary": S.UpstreamError("refused"), "fallback": 200}) resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) - assert [c[0] for c in calls] == ["primary", "fallback"] - assert resp.status == 200 and resp.upstream is not None + assert [c[0] for c in calls] == ["primary"] # fallback is never dialed + assert resp.status == 502 and resp.upstream is None -def test_failover_on_5xx() -> None: +def test_no_failover_on_5xx() -> None: + # INVERTED for issue #91: a same-task 5xx (e.g. EngineDeadError) must not + # fail over to a backend serving a different model. See + # test_no_failover_on_connection_refused for the full rationale; same t6 + # note re: 502 -> 503+Retry-After being out of this task's scope. table, cfg = _cfg() opener, calls = _opener({"primary": 503, "fallback": 200}) resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) - assert [c[0] for c in calls] == ["primary", "fallback"] - assert resp.status == 200 + assert [c[0] for c in calls] == ["primary"] # fallback is never dialed + assert resp.status == 502 def test_no_failover_on_4xx() -> None: @@ -93,11 +105,16 @@ def test_no_failover_on_4xx() -> None: def test_all_backends_down_returns_502() -> None: + # With no cross-backend failover (issue #91), only the owner is ever + # attempted — fallback's outcome is irrelevant to a "P" request and is + # never dialed. attempts therefore has a single entry, not one per backend. + # (t6 converts this 502 to 503 + Retry-After; out of scope for this task.) table, cfg = _cfg() - opener, _ = _opener({"primary": S.UpstreamError("x"), "fallback": S.UpstreamError("y")}) + opener, calls = _opener({"primary": S.UpstreamError("x"), "fallback": S.UpstreamError("y")}) resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) + assert [c[0] for c in calls] == ["primary"] # fallback never dialed assert resp.status == 502 and resp.upstream is None - assert json.loads(resp.body)["error"]["attempts"] == ["x", "y"] + assert json.loads(resp.body)["error"]["attempts"] == ["x"] def test_missing_model_routes_to_default() -> None: @@ -108,11 +125,15 @@ def test_missing_model_routes_to_default() -> None: assert resp.status == 200 -def test_explicit_fallback_routes_to_fallback_first() -> None: +def test_explicit_fallback_routes_to_fallback_only() -> None: + # RENAMED from test_explicit_fallback_routes_to_fallback_first (issue #91): + # "first" implied there could be a second (a failover to primary) — there + # no longer is. An explicit "F" request is attempted at fallback and + # nowhere else, even though primary is healthy and configured in the table. table, cfg = _cfg() opener, calls = _opener({"primary": 200, "fallback": 200}) resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"F"}', opener) - assert calls[0][0] == "fallback" + assert [c[0] for c in calls] == ["fallback"] # primary is never dialed assert resp.status == 200 From e5ad5dc40c1a434ac36ac40168c1aed614178230 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 10:20:00 +0300 Subject: [PATCH 09/23] Split RoleInfo.ready from loaded for all six roles; never fabricate a gateway endpoint from internal host:port (t5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs let GET /capabilities advertise ready: true for a role whose endpoint 404s (advertised-implies-reachable violation): 1. RoleInfo.ready was a bare alias of loaded (a config fact) for cortex/senses/embedder/reranker, unlike stt/tts which already split the two (issue #89/#90). build_role_registry now accepts an optional backend_ready: Mapping[str, bool | None] keyed by ROLE_BACKEND name (the exact shape lobes.gateway._readiness.ReadinessCache.current() returns), mirroring audio_ready's shape/defaulting. ready is structurally clamped to False whenever a role's backend isn't wired or its endpoint is empty, so no caller-supplied signal can fabricate ready=True — generalising the #89/#90 clamp to all six roles. 2. _gateway_base_url(server) fabricated an absolute URL from the gateway's INTERNAL container listen port (GATEWAY_HOST/GATEWAY_PORT), which need not match the published host port a caller can actually dial. It now only ever returns server.public_url (an operator- declared GATEWAY_PUBLIC_URL) or "" — never a value derived from host:port. An empty endpoint is covered by the same ready clamp. Repairs the resulting blast radius: tests/test_roles.py (new backend_ready + empty-endpoint coverage, replacing the now-obsolete GATEWAY_HOST bracket/normalize tests), tests/test_gateway_capabilities.py (explicit gateway_url added where a test needs a dialable endpoint), and tests/test_colleague_contract.py (the fake fleet now passes its own real loopback origin as gateway_url, matching what the production HTTP route does via reachable_origin()). lobes/gateway/server.py is untouched — wiring ReadinessCache into the live gateway route is a follow-up task. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/roles.py | 182 +++++++++++++++++++++-------- tests/test_colleague_contract.py | 19 ++- tests/test_gateway_capabilities.py | 39 ++++++- tests/test_roles.py | 171 +++++++++++++++++++++++---- 4 files changed, 333 insertions(+), 78 deletions(-) diff --git a/lobes/roles.py b/lobes/roles.py index b32d888..cdca695 100644 --- a/lobes/roles.py +++ b/lobes/roles.py @@ -164,11 +164,21 @@ class RoleInfo: mtp: bool # speculative decoding (MTP draft head) active for this model responsibilities: tuple[str, ...] forbidden_responsibilities: tuple[str, ...] - # Coarse "configured/wired" readiness — always == `loaded` today. This is - # the same proxy the gateway's GET /capabilities uses (issue #81), so the - # CLI and gateway agree on one boolean. It is NOT a live health probe; - # true liveness (did the backend answer a request just now) is - # `lobes measure`'s job (t8), not this dataclass. + # Runtime readiness — a caller-supplied LIVE signal, folded in by + # build_role_registry: `backend_ready` (keyed by the ROLE_BACKEND name) + # for the four gateway-fronted roles, `audio_ready` for stt/tts (issue + # #89). Generalised from the stt/tts-only split (issue #89/#90) to all six + # roles (issue #81 t5) — `ready` is no longer a bare alias of `loaded`. + # When a caller supplies no signal (the parameter is `None`, the default), + # `ready` falls back to the coarse `loaded` "configured/wired" proxy — the + # original t4 behaviour, still exercised by every non-HTTP caller (the + # CLI's non-live paths, most of this module's own test suite). + # Structurally CLAMPED either way: a role whose backend is not wired + # (`loaded is False`) or whose `endpoint` is empty can never report + # `ready=True`, no matter what signal a caller passes in. This mirrors — + # and is enforced by the same code path as — the stt/tts clamp on + # `audio_configured` (issue #89/#90 review finding), now applied to all + # six roles by build_role_registry itself, not left to caller discipline. ready: bool = False # Is this role's backend/service wired/present in THIS deployment? An # unconfigured/opt-in role is still returned, with loaded=False. @@ -186,25 +196,33 @@ def _catalog_by_role_hint(role_hint: str) -> SupportedModel | None: def _gateway_base_url(server: ServerConfig) -> str: - """The gateway's caller-facing base URL derived from its listen config. - - ``ServerConfig.host`` defaults to the wildcard bind ``0.0.0.0`` (usable for - binding, not as a client target), so it is normalized to ``localhost``. - Callers that know the real reachable address (a published host port, a tunnel - URL) should pass an explicit ``gateway_url`` to :func:`build_role_registry`. - - An IPv6 literal host (e.g. ``GATEWAY_HOST=::1``) is bracketed per RFC 3986 - (``http://[::1]:8000``) — an unbracketed IPv6 literal ahead of ``:`` - is not a valid URL authority (the address's own colons collide with the - port separator). IPv4 literals and hostnames carry no colon and pass - through unchanged. + """The gateway's caller-facing base URL — NEVER fabricated from host:port. + + ``ServerConfig.host``/``.port`` (``GATEWAY_HOST``/``GATEWAY_PORT``) are the + gateway process's own INTERNAL listen config — where it binds inside its + container — not necessarily where a caller can reach it from outside. On + the reference rig the gateway listens on internal container port 8000 but + is PUBLISHED on host port 8001, and host port 8000 belongs to a wholly + unrelated daemon (a stray uvicorn service). A URL built from + ``host:port`` would therefore silently advertise that foreign daemon as + if it were the gateway — a caller dialing it gets whatever happens to be + listening there, not a 404 from the gateway, which is worse than an + honest "unknown". This function must never do that. + + Returns ``server.public_url`` (the operator-declared, caller-reachable + origin — ``GATEWAY_PUBLIC_URL``), rstripped of a trailing slash, when it + is set; otherwise ``""``. An empty return here is not a degraded case to + special-case downstream — :func:`build_role_registry` already treats an + empty ``endpoint`` as a hard "never advertise ready=True" signal, so a + caller either gets a real, dialable endpoint or an honest absence of one. + + Callers that know the real reachable address from elsewhere (a published + host port, a tunnel URL, or — as the gateway's own HTTP route does, + issue #87 — the request's own ``Host`` header) must pass it explicitly as + ``gateway_url`` to :func:`build_role_registry`; that explicit value always + wins over this fallback. """ - host = server.host - if host in ("0.0.0.0", "::", ""): # nosec B104 — a comparison, not a bind - host = "localhost" - if ":" in host and not host.startswith("["): - host = f"[{host}]" - return f"http://{host}:{server.port}" + return (server.public_url or "").rstrip("/") def _served_context(role: str, env: Mapping[str, str], native: int) -> int: @@ -234,8 +252,27 @@ def _gateway_role( table: RoutingTable, gateway: str, env: Mapping[str, str], + ready_signal: bool | None, ) -> RoleInfo: - """Resolve a gateway-fronted role (cortex/senses/embedder/reranker).""" + """Resolve a gateway-fronted role (cortex/senses/embedder/reranker). + + ``ready_signal`` is the caller's live-readiness tri-state for THIS role's + backend (the value :func:`build_role_registry` looked up in its + ``backend_ready`` mapping by :data:`ROLE_BACKEND` name) — ``True``/ + ``False`` reflect an actual probe result; ``None`` means no live signal is + available (no ``backend_ready`` mapping was supplied, or it had no entry + for this backend), in which case ``ready`` falls back to the coarse + ``loaded`` proxy, matching the original t4 behaviour. + + Either way, ``ready`` is CLAMPED to ``False`` whenever the backend is not + wired (``loaded is False``) or the resolved ``endpoint`` is empty (see + :func:`_gateway_base_url`) — enforced HERE, structurally, so a caller + passing a stale/wrong ``ready_signal`` for an unwired or undialable role + can never fabricate ``ready=True``. This generalises, to all four + gateway-fronted roles, the same clamp issue #89/#90 established for + stt/tts (a caller-supplied signal can never override "nothing is wired" + or "nothing to dial"). + """ backend = next((b for b in table.backends if b.name == ROLE_BACKEND[role]), None) loaded = backend is not None if backend is not None: @@ -248,18 +285,23 @@ def _gateway_role( # canonical entry when the operator serves a non-catalog name. entry = _catalog_by_id(model_id) or _catalog_by_role_hint(ROLE_ROLE_HINT[role]) native_context = entry.native_max_model_len if entry else 0 + endpoint = gateway + if loaded and endpoint: + ready = ready_signal if ready_signal is not None else loaded + else: + ready = False return RoleInfo( role=role, model=model_id, runtime=_VLLM_RUNTIME, - endpoint=gateway, + endpoint=endpoint, path=ROLE_PATH[role], context=_served_context(role, env, native_context), quant=entry.quantization if entry else "", mtp=bool(entry.speculative_config) if entry else False, responsibilities=ROLE_RESPONSIBILITIES[role], forbidden_responsibilities=ROLE_FORBIDDEN[role], - ready=loaded, + ready=ready, loaded=loaded, ) @@ -299,6 +341,7 @@ def build_role_registry( env: Mapping[str, str] | None = None, gateway_url: str | None = None, audio_ready: bool | None = None, + backend_ready: Mapping[str, bool | None] | None = None, ) -> dict[str, RoleInfo]: """Resolve the six first-class roles to live metadata — the #81 contract. @@ -310,8 +353,9 @@ def build_role_registry( :param table: the gateway routing table — its wired :class:`Backend` objects tell us which roles are ``loaded`` and each role's served model id. :param server: the gateway server config — supplies the audio overlay URL - (``audio_url``) for stt/tts and, absent ``gateway_url``, the derived - gateway base URL for the four gateway-fronted roles. + (``audio_url``) for stt/tts and, absent ``gateway_url``, the (very + narrow) ``public_url`` fallback for the four gateway-fronted roles — + see :func:`_gateway_base_url`. :param env: the deployment's environment mapping, consulted ONLY for the served ``--max-model-len`` overlay (:data:`ROLE_MAX_MODEL_LEN_ENV`) — so ``RoleInfo.context`` reports what the deployment actually SERVES @@ -321,33 +365,63 @@ def build_role_registry( Kept separate from ``table``/``server`` (typically built from the SAME env) so a caller assembling those by hand isn't forced to also pass it. :param gateway_url: the caller-facing gateway base URL for cortex / senses / - embedder / reranker. When ``None`` it is derived from ``server`` (host - ``0.0.0.0`` → ``localhost``). Audio roles also use this origin as their - endpoint (issue #87). - :param audio_ready: optional live-readiness override for stt/tts (issue #89). + embedder / reranker. When ``None``, it falls back to + ``server.public_url`` (an operator-declared ``GATEWAY_PUBLIC_URL``) and, + failing that, to ``""`` — it is NEVER fabricated from + ``server.host``/``server.port`` (issue #81 t5; those are the gateway's + INTERNAL listen config, not a client-reachable address — see + :func:`_gateway_base_url`). Audio roles also use this origin as their + endpoint when the overlay is wired (issue #87). + :param audio_ready: optional live-readiness signal for stt/tts (issue #89). When not ``None`` it sets the audio roles' ``ready`` (the runtime signal) — ``loaded`` stays the config fact ``bool(audio_url)``. When ``None``, ``ready`` falls back to ``bool(audio_url)`` (the CLI/back-compat path). + :param backend_ready: optional live-readiness signal for the four + gateway-fronted roles (issue #81 t5), keyed by the internal + :class:`~lobes.gateway._routing.Backend` name (:data:`ROLE_BACKEND`'s + values — ``"primary"``/``"multimodal"``/``"embed"``/``"rerank"``), one + tri-state value (``True``/``False``/``None``) per backend — exactly + the shape :meth:`lobes.gateway._readiness.ReadinessCache.current` + returns. Mirrors ``audio_ready``'s shape and defaulting: when a role's + backend has an entry, that entry sets ``ready`` (the runtime signal); + ``loaded`` stays the config fact "is this backend wired". When + ``backend_ready`` is ``None`` (the default) — OR a role's backend has + no entry in it — ``ready`` falls back to ``loaded``, the original t4 + behaviour, so every existing non-HTTP caller (the CLI, this module's + own offline test suite) is unchanged. ``roles.py`` itself never probes + anything to produce this signal — it is computed elsewhere (t3's + :class:`~lobes.gateway._readiness.ReadinessCache`, socket-free to read) + and handed in, exactly like ``audio_ready``. :returns: an ordered ``dict`` keyed by role name with EXACTLY the six roles. Every role is always present — an unconfigured/opt-in role (stt/tts with ``audio_url`` unset, or an unwired embed/rerank/multimodal backend) is returned with ``loaded=False``, never omitted and never raising. - Readiness (``RoleInfo.ready``) is set to the same coarse "configured/wired" - proxy as ``loaded`` — the CLI (t5) and the gateway's ``GET /capabilities`` - (t6) must agree on one boolean, so it is computed here, once, for both. - This is NOT a live health probe (it opens no socket); true liveness is a - later task's concern (``lobes measure``, t8). + Readiness (``RoleInfo.ready``) is no longer a bare alias of ``loaded`` + (issue #81 t5 — generalising the stt/tts split from issue #89/#90 to all + six roles): it reflects ``backend_ready``/``audio_ready`` when the caller + supplies a live signal, else it falls back to the coarse "configured/wired" + ``loaded`` proxy. Either way it is CLAMPED, here, to ``False`` whenever a + role's backend is not wired OR its resolved ``endpoint`` is empty — a + caller can never fabricate ``ready=True`` for a role with nothing to dial, + regardless of what signal it passes in. ``roles.py`` stays pure/offline + either way — it opens no socket to produce or consume this signal; true + liveness is probed elsewhere (t3's ``ReadinessCache`` / + ``probe_audio_ready``, issue #89) and handed in as a plain value. """ resolved_env: Mapping[str, str] = env if env is not None else {} gateway = (gateway_url or _gateway_base_url(server)).rstrip("/") registry: dict[str, RoleInfo] = {} for role in ("cortex", "senses", "embedder", "reranker"): - registry[role] = _gateway_role(role, table, gateway, resolved_env) + signal = backend_ready.get(ROLE_BACKEND[role]) if backend_ready is not None else None + registry[role] = _gateway_role(role, table, gateway, resolved_env, signal) audio_url = (server.audio_url or "").rstrip("/") audio_configured = bool(audio_url) + # Audio roles use the gateway origin when the overlay is wired (issue #87), + # but fall back to empty endpoint when it is not wired. + audio_endpoint = gateway if audio_configured else "" # `loaded` is a config fact — is the audio overlay wired in THIS deployment — # kept SEPARATE from `ready`, the runtime signal. `ready` is the gateway's # live probe (`audio_ready`) when it supplied one, else it falls back to the @@ -356,14 +430,18 @@ def build_role_registry( # masquerading as not-deployed, and an unconfigured overlay never reports a # ready role with an empty endpoint. # - # Clamp on `audio_configured` so that last invariant holds STRUCTURALLY, not - # merely by caller discipline: an unconfigured overlay is never ready, no - # matter what `audio_ready` a caller passes. When configured, use the live - # probe signal if one was supplied, else fall back to the configured fact. - audio_ready_signal = audio_configured and (audio_ready if audio_ready is not None else True) - # Audio roles use the gateway origin when the overlay is wired (issue #87), - # but fall back to empty endpoint when it is not wired. - audio_endpoint = gateway if audio_configured else "" + # Clamp on `audio_configured` AND `audio_endpoint` so that last invariant + # holds STRUCTURALLY, not merely by caller discipline: an unconfigured + # overlay, or one whose endpoint came back empty because no gateway_url/ + # public_url was known (issue #81 t5, criterion 3), is never ready, no + # matter what `audio_ready` a caller passes. When configured AND dialable, + # use the live probe signal if one was supplied, else fall back to the + # configured fact. + audio_ready_signal = ( + audio_configured + and bool(audio_endpoint) + and (audio_ready if audio_ready is not None else True) + ) registry["stt"] = _audio_role( "stt", _STT_MODEL, _STT_RUNTIME, audio_endpoint, audio_configured, ready=audio_ready_signal ) @@ -378,6 +456,7 @@ def role_registry_from_env( *, gateway_url: str | None = None, audio_ready: bool | None = None, + backend_ready: Mapping[str, bool | None] | None = None, ) -> dict[str, RoleInfo]: """Build the role registry straight from an env mapping. @@ -389,9 +468,18 @@ def role_registry_from_env( build the registry from a deployment's ``.env`` without assembling a ``RoutingTable``/``ServerConfig`` pair by hand. ``env`` defaults to ``os.environ`` when omitted (matching :func:`build_config`'s default). + ``audio_ready``/``backend_ready`` pass straight through to + :func:`build_role_registry` (both default ``None`` — this offline + convenience never probes anything itself; a caller with a live signal in + hand supplies it here exactly as it would to the canonical builder). """ resolved_env = os.environ if env is None else env table, server = build_config(resolved_env) return build_role_registry( - table, server, env=resolved_env, gateway_url=gateway_url, audio_ready=audio_ready + table, + server, + env=resolved_env, + gateway_url=gateway_url, + audio_ready=audio_ready, + backend_ready=backend_ready, ) diff --git a/tests/test_colleague_contract.py b/tests/test_colleague_contract.py index 8b24ffc..cdd32a5 100644 --- a/tests/test_colleague_contract.py +++ b/tests/test_colleague_contract.py @@ -131,8 +131,17 @@ def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API route = self.path.split("?", 1)[0] if route == "/capabilities": # THE REAL production builder — the contract under test is the - # actual shipped one, not a hand-rolled fixture. - self._send_json(200, S.capabilities_payload(self.table, self.cfg, env=self.env)) + # actual shipped one, not a hand-rolled fixture. capabilities_payload + # never fabricates an endpoint from GATEWAY_HOST/GATEWAY_PORT + # (issue #81 t5, criterion 3), so — exactly like the real gateway's + # do_GET route derives `origin` via `reachable_origin` and passes it + # explicitly — this fake fleet passes its own real, dialable + # loopback origin (the address this httpd is actually bound to). + gateway_url = f"http://{self.cfg.host}:{self.cfg.port}" + self._send_json( + 200, + S.capabilities_payload(self.table, self.cfg, env=self.env, gateway_url=gateway_url), + ) elif route == "/health": self._send_json(200, {"status": "ok"}) elif route == "/metrics": @@ -366,7 +375,11 @@ def test_capabilities_contract_is_runtime_descriptor_only(fake_fleet) -> None: def test_measure_registry_emits_only_allowed_runtime_metric_keys(fake_fleet) -> None: base_url, table, cfg, env = fake_fleet - registry = build_role_registry(table, cfg, env=env) + # gateway_url=base_url: the fake fleet's own real, dialable loopback origin + # — the honest fix for issue #81 t5 criterion 3 (build_role_registry never + # fabricates an endpoint from GATEWAY_HOST/GATEWAY_PORT), mirroring what the + # production HTTP route passes via reachable_origin(...). + registry = build_role_registry(table, cfg, env=env, gateway_url=base_url) measured = measure_registry(registry, timeout=3.0) assert set(measured) == set(ROLES) for role, result in measured.items(): diff --git a/tests/test_gateway_capabilities.py b/tests/test_gateway_capabilities.py index b8695bd..ec0f4ad 100644 --- a/tests/test_gateway_capabilities.py +++ b/tests/test_gateway_capabilities.py @@ -33,6 +33,16 @@ _EMBED_ID = "Qwen/Qwen3-Embedding-0.6B" _RERANK_ID = "Qwen/Qwen3-Reranker-0.6B" +# capabilities_payload/build_role_registry NEVER fabricate an endpoint from +# GATEWAY_HOST/GATEWAY_PORT (issue #81 t5, criterion 3 — see +# lobes.roles._gateway_base_url): those are the gateway's own INTERNAL listen +# config, not necessarily a client-reachable address. Every real caller (the +# HTTP route via reachable_origin, the CLI) always supplies an explicit, +# known-dialable gateway_url — this constant mirrors that for the tests below +# that exercise capabilities_payload directly (bypassing the HTTP route) and +# aren't themselves testing the "no gateway_url known" path. +_GATEWAY_URL = "http://localhost:8000" + def _full_env(**over) -> dict[str, str]: """A fully-wired six-role generate+pooling fleet (audio deliberately unset).""" @@ -61,9 +71,9 @@ def test_capabilities_payload_matches_cli_shape() -> None: # (the CLI leaves it None too; the gateway fills it from `loaded`). env = _full_env() table, cfg = build_config(env) - payload = S.capabilities_payload(table, cfg, env=env) + payload = S.capabilities_payload(table, cfg, env=env, gateway_url=_GATEWAY_URL) assert set(payload) == set(ROLES) - registry = build_role_registry(table, cfg, env=env) + registry = build_role_registry(table, cfg, env=env, gateway_url=_GATEWAY_URL) for role in ROLES: expected = dataclasses.asdict(registry[role]) expected["ready"] = registry[role].loaded @@ -106,7 +116,7 @@ def test_capabilities_payload_unwired_roles_present_not_omitted() -> None: # A primary-only fleet: no multimodal/embed/rerank/audio wired. env = {"PRIMARY_URL": "http://vllm-primary:8000", "PRIMARY_SERVED_NAME": _PRIMARY_ID} table, cfg = build_config(env) - payload = S.capabilities_payload(table, cfg, env=env) + payload = S.capabilities_payload(table, cfg, env=env, gateway_url=_GATEWAY_URL) assert set(payload) == set(ROLES) # all six present, never a 500, never omitted for role in ("senses", "embedder", "reranker", "stt", "tts"): assert payload[role]["loaded"] is False @@ -118,11 +128,26 @@ def test_capabilities_payload_unwired_roles_present_not_omitted() -> None: def test_capabilities_payload_ready_mirrors_loaded() -> None: env = _full_env() table, cfg = build_config(env) - payload = S.capabilities_payload(table, cfg, env=env) + payload = S.capabilities_payload(table, cfg, env=env, gateway_url=_GATEWAY_URL) for role in ROLES: assert payload[role]["ready"] == payload[role]["loaded"] +def test_capabilities_payload_no_gateway_url_and_no_public_url_never_ready() -> None: + """issue #81 t5, criterion 3: absent gateway_url/GATEWAY_PUBLIC_URL, the + gateway-fronted roles get an EMPTY endpoint — never a URL fabricated from + GATEWAY_HOST/GATEWAY_PORT — and are never advertised ready, even though + `loaded` (the config fact) is still True. This is the exact bug class the + issue reports: 'ready: true for roles whose endpoint returns 404'.""" + env = _full_env() + table, cfg = build_config(env) + payload = S.capabilities_payload(table, cfg, env=env) # no gateway_url + for role in ("cortex", "senses", "embedder", "reranker"): + assert payload[role]["endpoint"] == "" + assert payload[role]["loaded"] is True + assert payload[role]["ready"] is False + + def test_capabilities_payload_defaults_env_to_os_environ(monkeypatch) -> None: # Inside the gateway container, os.environ IS the deployment env — the `env` # kwarg must default to it, not require a caller to pass one explicitly. @@ -224,11 +249,13 @@ def test_capabilities_payload_threads_audio_ready() -> None: env = _full_env(AUDIO_URL="http://realtime:8080") table, cfg = build_config(env) # audio_ready=False → stt/tts advertise ready:false even though AUDIO_URL is set. - warming = S.capabilities_payload(table, cfg, env=env, audio_ready=False) + warming = S.capabilities_payload( + table, cfg, env=env, gateway_url=_GATEWAY_URL, audio_ready=False + ) for role in ("stt", "tts"): assert warming[role]["ready"] is False # audio_ready=True → ready. - live = S.capabilities_payload(table, cfg, env=env, audio_ready=True) + live = S.capabilities_payload(table, cfg, env=env, gateway_url=_GATEWAY_URL, audio_ready=True) for role in ("stt", "tts"): assert live[role]["ready"] is True diff --git a/tests/test_roles.py b/tests/test_roles.py index c39c51f..9bab5d1 100644 --- a/tests/test_roles.py +++ b/tests/test_roles.py @@ -20,6 +20,7 @@ from lobes.catalog import SUPPORTED_MODELS, SupportedModel from lobes.gateway._config import build_config from lobes.roles import ( + ROLE_BACKEND, ROLE_FORBIDDEN, ROLE_MAX_MODEL_LEN_ENV, ROLE_RESPONSIBILITIES, @@ -40,6 +41,16 @@ _EXPECTED_ROLES = {"cortex", "senses", "embedder", "reranker", "stt", "tts"} +# The gateway-fronted roles' endpoint is NEVER fabricated from GATEWAY_HOST/ +# GATEWAY_PORT (issue #81 t5, criterion 3 — those are the gateway's INTERNAL +# listen config, not necessarily a client-reachable address; see +# lobes.roles._gateway_base_url). Every real caller (the CLI, the gateway's +# own HTTP route) always passes an explicit, known-reachable `gateway_url` — +# so this default mirrors that reality for the tests below that aren't +# specifically exercising the "no gateway_url known" path (which pass their +# own `gateway_url=None` override instead, see the "no fabrication" tests). +_DEFAULT_TEST_GATEWAY_URL = "http://localhost:8000" + def _full_env() -> dict[str, str]: """A fully-wired six-role fleet: primary + multimodal + embed + rerank + audio.""" @@ -75,6 +86,13 @@ def _registry(env: dict[str, str], *, audio_ready: bool | None = None, **kw) -> # against the catalog native, which is exactly the t4-vs-t5 regression this # module now guards against. kw.setdefault("env", env) + # Default to a realistic, dialable gateway_url — exactly what every real + # caller (the CLI, the gateway's own HTTP route) always supplies (t5, + # criterion 3: build_role_registry never fabricates one from GATEWAY_HOST/ + # GATEWAY_PORT). A test that specifically wants the "no gateway_url known" + # path passes `gateway_url=None` explicitly, which `setdefault` respects + # (the key is already present, so the default below is skipped). + kw.setdefault("gateway_url", _DEFAULT_TEST_GATEWAY_URL) return build_role_registry(table, server, audio_ready=audio_ready, **kw) @@ -257,35 +275,56 @@ def test_static_responsibility_maps_cover_all_six_roles() -> None: # --------------------------------------------------------------------------- -def test_gateway_roles_endpoint_defaults_to_derived_gateway_url() -> None: - """host 0.0.0.0 (bind wildcard) is normalized to a caller-usable localhost.""" - registry = _registry(_full_env()) +def test_no_gateway_url_and_no_public_url_yields_empty_endpoint_never_fabricated() -> None: + """issue #81 t5, criterion 3 — the core fix. GATEWAY_HOST/GATEWAY_PORT are + the gateway's own INTERNAL listen config (where it binds inside its + container), not necessarily what a caller outside can dial: on the + reference rig the gateway listens on internal container port 8000 but is + PUBLISHED on host port 8001, and host port 8000 belongs to an unrelated + daemon. Advertising a URL fabricated from GATEWAY_HOST/GATEWAY_PORT would + silently point a Colleague client at that foreign service, so absent an + explicit gateway_url or an operator-declared GATEWAY_PUBLIC_URL, the + endpoint must be empty — never http://:.""" + env = _full_env() + env["GATEWAY_HOST"] = "gateway-internal" # would have been the old fabrication input + env["GATEWAY_PORT"] = "8000" # the internal-only listen port + registry = _registry(env, gateway_url=None) # explicitly opt out of the test default for name in ("cortex", "senses", "embedder", "reranker"): - assert registry[name].endpoint == "http://localhost:8000" + info = registry[name] + assert info.endpoint == "" + assert "gateway-internal" not in info.endpoint + assert "8000" not in info.endpoint + # loaded stays the config truth even though nothing is dialable... + assert info.loaded is True + # ...but an empty endpoint can never be advertised ready (criterion 3). + assert info.ready is False -def test_gateway_roles_endpoint_brackets_ipv6_host() -> None: - """An IPv6 literal GATEWAY_HOST must be bracketed per RFC 3986 — an - unbracketed 'http://::1:8000' is not a valid/parseable URL authority - (the address's own colons collide with the ':' separator).""" +def test_gateway_public_url_is_the_one_implicit_endpoint_source() -> None: + """GATEWAY_PUBLIC_URL is an operator's explicit, caller-reachable origin + declaration — read verbatim (trailing slash trimmed), never derived from + GATEWAY_HOST/GATEWAY_PORT. It is the only source of an endpoint besides an + explicit gateway_url.""" env = _full_env() - env["GATEWAY_HOST"] = "::1" - registry = _registry(env) + env["GATEWAY_PUBLIC_URL"] = "https://tunnel.example/" + registry = _registry(env, gateway_url=None) for name in ("cortex", "senses", "embedder", "reranker"): - assert registry[name].endpoint == "http://[::1]:8000" + info = registry[name] + assert info.endpoint == "https://tunnel.example" + assert info.loaded is True + # Non-empty endpoint + loaded + no live signal supplied → falls back + # to the coarse loaded proxy, the t4 behaviour. + assert info.ready is True -def test_gateway_roles_endpoint_leaves_ipv4_and_hostnames_unbracketed() -> None: - """IPv4 literals and hostnames carry no colon — bracketing must be scoped - to IPv6 only, never applied to these.""" - for host, expected in ( - ("127.0.0.1", "http://127.0.0.1:8000"), - ("gateway.internal", "http://gateway.internal:8000"), - ): - env = _full_env() - env["GATEWAY_HOST"] = host - registry = _registry(env) - assert registry["cortex"].endpoint == expected +def test_explicit_gateway_url_wins_over_public_url() -> None: + """An explicit gateway_url (the caller's own reachable origin, issue #87) + always takes precedence over a configured GATEWAY_PUBLIC_URL.""" + env = _full_env() + env["GATEWAY_PUBLIC_URL"] = "https://tunnel.example/" + registry = _registry(env, gateway_url="https://actual-caller-origin.example") + for name in ("cortex", "senses", "embedder", "reranker"): + assert registry[name].endpoint == "https://actual-caller-origin.example" def test_explicit_gateway_url_applies_to_all_roles_including_audio() -> None: @@ -465,6 +504,94 @@ def test_audio_ready_none_falls_back_to_audio_url() -> None: assert registry[name].ready is False +# --------------------------------------------------------------------------- +# Issue #81 t5 — backend_ready: generalising the stt/tts ready/loaded split +# (issue #89/#90) to the four gateway-fronted roles. `RoleInfo.ready` is no +# longer a bare alias of `loaded` for cortex/senses/embedder/reranker either. +# --------------------------------------------------------------------------- + + +def test_backend_ready_sets_ready_independent_of_loaded() -> None: + """A live signal in `backend_ready` (keyed by ROLE_BACKEND name) sets a + gateway-fronted role's `ready` — `loaded` stays the config fact.""" + registry = _registry( + _full_env(), + backend_ready={"primary": False, "multimodal": True, "embed": False, "rerank": True}, + ) + assert registry["cortex"].loaded is True + assert registry["cortex"].ready is False # wired, but the live probe said no + assert registry["senses"].loaded is True + assert registry["senses"].ready is True + assert registry["embedder"].loaded is True + assert registry["embedder"].ready is False + assert registry["reranker"].loaded is True + assert registry["reranker"].ready is True + + +def test_backend_ready_missing_entry_falls_back_to_loaded() -> None: + """A backend_ready mapping that simply has no entry for a role's backend + (e.g. a readiness cache that hasn't probed it yet) degrades to the coarse + `loaded` proxy for THAT role only — never raises, never guesses.""" + registry = _registry(_full_env(), backend_ready={"primary": True}) + assert registry["cortex"].ready is True # has a live entry + for role in ("senses", "embedder", "reranker"): + info = registry[role] + assert info.ready == info.loaded # no entry → falls back, same as backend_ready=None + + +def test_backend_ready_none_preserves_t4_behaviour() -> None: + """backend_ready omitted entirely (the default) → ready == loaded for every + gateway-fronted role, exactly the pre-t5 behaviour — every existing + non-HTTP caller is unaffected.""" + registry = _registry(_full_env()) + for role in ("cortex", "senses", "embedder", "reranker"): + assert registry[role].ready == registry[role].loaded + + +def test_backend_ready_never_fabricates_ready_when_backend_unwired() -> None: + """Clamp (Qodo #90-class finding, generalised to all four gateway-fronted + roles): a role whose backend is NOT wired in this deployment must never + report ready=True, no matter what a caller's backend_ready says. A stale + or wrong-keyed live signal can't override 'nothing is actually wired'.""" + registry = _registry( + _primary_only_env(), # senses/embedder/reranker are NOT wired here + backend_ready={"primary": True, "multimodal": True, "embed": True, "rerank": True}, + ) + assert registry["cortex"].loaded is True + assert registry["cortex"].ready is True + for role in ("senses", "embedder", "reranker"): + info = registry[role] + assert info.loaded is False + assert info.ready is False # never True, despite backend_ready saying so + assert not (info.ready and info.endpoint == "") # never ready+undialable + + +def test_backend_ready_never_fabricates_ready_when_endpoint_empty() -> None: + """Clamp, the other half (criterion 3): even a WIRED role with a live + True signal must never report ready=True when the resolved endpoint is + empty (no gateway_url, no GATEWAY_PUBLIC_URL known) — there is nothing to + dial, so 'ready' would be a lie.""" + registry = _registry( + _full_env(), + gateway_url=None, # opt out of the test default → endpoint resolves to "" + backend_ready={"primary": True, "multimodal": True, "embed": True, "rerank": True}, + ) + for role in ("cortex", "senses", "embedder", "reranker"): + info = registry[role] + assert info.endpoint == "" + assert info.loaded is True + assert info.ready is False + + +def test_role_backend_keys_match_backend_ready_vocabulary() -> None: + """ROLE_BACKEND's values are exactly the backend names a caller's + backend_ready mapping (e.g. lobes.gateway._readiness.ReadinessCache) is + keyed by — "primary"/"multimodal"/"embed"/"rerank" — so a caller can pass + ReadinessCache.current() straight through with no key translation.""" + assert set(ROLE_BACKEND.values()) == {"primary", "multimodal", "embed", "rerank"} + assert set(ROLE_BACKEND) == {"cortex", "senses", "embedder", "reranker"} + + # --------------------------------------------------------------------------- # All six roles expose the identical key set # --------------------------------------------------------------------------- From f5f5c3c206688330f35a7acf7e16fdafcd168ec4 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 10:39:15 +0300 Subject: [PATCH 10/23] =?UTF-8?q?t6:=20advertised=20implies=20reachable=20?= =?UTF-8?q?=E2=80=94=20dead=20owner=20=E2=87=92=20retryable=20503;=20readi?= =?UTF-8?q?ness-gated=20/v1/models=20&=20/capabilities=20(#91=20#92=20#14)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Integration task the "advertised implies reachable" plan converges on. Job 1 — status matrix (handle_post, #14/#91). With order_backends now single-element (no cross-backend failover), a present-but-dead owner (refusal / timeout / >=500) becomes a RETRYABLE 503 backend_unavailable + Retry-After, never a terminal 404/502. The 429 server_busy shed is untouched; a 4xx (incl. the owner's own 404 "model does not exist") is a client error relayed verbatim; 502 upstream_unavailable survives only for the degenerate empty order_backends (malformed routing table). Rewrote handle_post's + the module docstring, which documented removed failover. Job 2 — readiness wiring (#92, c15/h14). serve() constructs a ReadinessCache, does one bounded synchronous refresh() BEFORE binding (closes the startup window — the cache seeds to None), then start()s the daemon; _make_handler binds it. GET /v1/models lists only backends whose readiness is True (list_models_payload gains an additive ready filter in _routing.py). GET /capabilities folds the snapshot into role readiness. Bridged a semantic mismatch at the boundary (_ready_iff_true): the cache's None means "unreachable" but roles.py reads None as "no signal → fall back to loaded", so a raw pass-through would advertise a wired-but-dead backend ready=True — collapse None/False→False so a dead backend is advertised nowhere. The POST hot path opens no probe (reads socket-free .current()). Job 3 — origin resolution (c29/h25). Locked reachable_origin precedence (GATEWAY_PUBLIC_URL > Host > empty) with tests, incl. the regression guard that an unset public_url + Host: spark.local:8001 yields the client's own origin, never localhost/GATEWAY_PORT (host :8000 is a foreign daemon on the rig), and that None → "" endpoint end to end. Invariants: race (listed → owner killed → 503, never 404), no-Gemma (dead primary never dials the multimodal backend, asserted on opener call sites), converse (unknown model id routes to default owner — current behaviour locked as a documented choice; recommend a future 404 in _routing.py). Added public ReadinessCache.refresh() (permitted additive change) for the seed-before-bind. Tests: 1151 passed, 6 skipped. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/gateway/_readiness.py | 17 ++ lobes/gateway/_routing.py | 25 ++- lobes/gateway/server.py | 225 ++++++++++++++++++++--- tests/test_gateway_capabilities.py | 162 +++++++++++++++++ tests/test_gateway_server.py | 282 ++++++++++++++++++++++++++--- 5 files changed, 653 insertions(+), 58 deletions(-) diff --git a/lobes/gateway/_readiness.py b/lobes/gateway/_readiness.py index 0e76e12..f2f452f 100644 --- a/lobes/gateway/_readiness.py +++ b/lobes/gateway/_readiness.py @@ -205,6 +205,23 @@ def _refresh_once(self) -> None: with self._lock: self._value = value + def refresh(self) -> None: + """Probe every backend once, synchronously, and update the snapshot NOW. + + A public, blocking one-shot the gateway calls **once before it binds** so + ``GET /v1/models`` and ``GET /capabilities`` are correct on the very first + request: construction seeds every backend to ``None`` (*unknown*) without + probing, so without this a freshly-started cache would report everything + unready until the daemon's first background pass lands (up to one + ``interval``). This closes that startup window with a single bounded pass + (each backend probed once, ``timeout``-capped), then :meth:`start` hands + subsequent refreshes to the daemon thread — off the request path. It is a + thin public alias for the daemon's own :meth:`_refresh_once`; keeping the + internal name private and exposing this one keeps the seed-before-bind + intent legible at the one call site (``server.serve``) that needs it. + """ + self._refresh_once() + def current(self) -> dict[str, bool | None]: """Return a copy of the latest readiness snapshot. Never probes, never blocks. diff --git a/lobes/gateway/_routing.py b/lobes/gateway/_routing.py index ad4d50a..44a6f2c 100644 --- a/lobes/gateway/_routing.py +++ b/lobes/gateway/_routing.py @@ -161,13 +161,32 @@ def order_backends(table: RoutingTable, served_name: str) -> list[Backend]: return [owner] if owner is not None else [] -def list_models_payload(table: RoutingTable) -> dict: - """OpenAI ``/v1/models`` shape listing every backend's served model.""" +def list_models_payload( + table: RoutingTable, ready: Mapping[str, "bool | None"] | None = None +) -> dict: + """OpenAI ``/v1/models`` shape listing the fleet's served models. + + When ``ready`` is supplied — the gateway's live readiness snapshot, keyed by + backend **name** (exactly what + :meth:`lobes.gateway._readiness.ReadinessCache.current` returns) — only + backends whose signal ``is True`` are listed. This is the core of "advertised + implies reachable" (issue #92): a backend that is wired but dead/missing + (``None``) or reached-but-unhealthy (``False``) must NOT be advertised, so a + client can trust that a model id appearing here will reach a live engine. + ``None`` (*unknown*) and ``False`` are BOTH treated as not-ready — only an + affirmative ``True`` advertises; treating ``None`` as "list it anyway" is the + exact defect #92 fixes (a wired-but-dead backend probes ``None``, not + ``False``). ``ready=None`` (the default) lists every wired backend unchanged — + the offline/CLI path and any caller without a live signal. + """ + backends = table.backends + if ready is not None: + backends = tuple(b for b in backends if ready.get(b.name) is True) return { "object": "list", "data": [ {"id": backend.served_name, "object": "model", "owned_by": "lobes"} - for backend in table.backends + for backend in backends ], } diff --git a/lobes/gateway/server.py b/lobes/gateway/server.py index 44906d9..56d8861 100644 --- a/lobes/gateway/server.py +++ b/lobes/gateway/server.py @@ -1,16 +1,35 @@ """The gateway HTTP server: a stdlib reverse proxy fronting the fleet backends. ``ThreadingHTTPServer`` + ``BaseHTTPRequestHandler``; the only module that opens -sockets. Routing/failover *decisions* live in :func:`handle_post` (a seam that -takes an ``open_upstream`` callable, so it's unit-testable without sockets) and in +sockets. Routing *decisions* live in :func:`handle_post` (a seam that takes an +``open_upstream`` callable, so it's unit-testable without sockets) and in :mod:`lobes.gateway._routing` (pure). The handler just reads the request, calls :func:`handle_post`, and relays the chosen upstream response — buffered for normal JSON, re-chunked for SSE streams. -Failover is intentionally narrow: a backend is retried only when it refuses the -connection or returns a 5xx **before any response body reaches the client**. A -4xx is a client error (returned verbatim, no failover); once a 2xx body starts -streaming, there is no retry (the client already has bytes). +**No cross-backend failover** (issue #91, "advertised implies reachable"). +:func:`lobes.gateway._routing.order_backends` resolves a requested model to its +ONE owning backend; a model is never retried against a different backend serving +a different model (that would either 404 on an unknown id or, worse, silently +answer as the wrong model — a role-contract violation). Because the owner is the +only backend that can serve the model, its verdict is authoritative: + +* a **2xx / 4xx** commits and is relayed verbatim — a 4xx (e.g. the owner's own + 404 "model does not exist") is a genuine *client* error now, not a trigger to + fail over; +* a **refusal / timeout / >=500** means the owner is transiently down → a + RETRYABLE **503** ``backend_unavailable`` + ``Retry-After`` (issue #14), NOT a + terminal 404/502, so a client retries the same model instead of concluding it + is gone; +* a **429** ``server_busy`` is the separate pressure-shed path (#85), and a + **502** ``upstream_unavailable`` survives only for the degenerate malformed + routing table (``order_backends`` returned an empty list) — see + :func:`handle_post`. + +Readiness governs *advertisement*, not routing: ``GET /v1/models`` and +``GET /capabilities`` fold in the background :class:`~lobes.gateway._readiness. +ReadinessCache` so a wired-but-dead backend is not advertised (issue #92); the +POST hot path never probes (it reads the socket-free cache, if at all). """ from __future__ import annotations @@ -31,6 +50,7 @@ from lobes.catalog import as_dicts as supported_models_catalog from lobes.gateway._config import ServerConfig from lobes.gateway._pressure_policy import BUSY_RETRY_AFTER_SECONDS, decide +from lobes.gateway._readiness import ReadinessCache from lobes.gateway._routing import ( Backend, RoutingTable, @@ -257,9 +277,32 @@ class GatewayResponse: attempts: list[str] = field(default_factory=list) -def _error_body(message: str, attempts: list[str]) -> bytes: +# Retry-After (seconds) on the 503 a transiently-down owner yields. The owner is +# the ONLY backend that can serve the requested model (#91: no failover), so its +# refusal / timeout / 5xx is a "come back shortly", not a terminal "no such model" +# — a caller should retry. Mirrors BUSY_RETRY_AFTER_SECONDS (the 429 shed) and the +# audio 503 (both 5s). +BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS: int = 5 + + +def _error_body( + message: str, attempts: list[str], *, error_type: str = "upstream_unavailable" +) -> bytes: + """OpenAI-shaped gateway error body. ``error_type`` names the failure class a + client must react to differently — the four are deliberately distinct: + + * ``upstream_unavailable`` — the degenerate **502**: ``order_backends`` + returned no owner (a malformed routing table). A config/deploy bug, not + retryable. + * ``backend_unavailable`` — the **503**: the one backend that owns this model + refused / timed out / 5xx'd (#14/#91). Retryable (carries ``Retry-After``). + * ``server_busy`` — the **429** pressure shed (#85), built separately + by :func:`_busy_body`. + * a relayed upstream ``404`` "model does not exist" — the owner's own verdict, + never generated here. + """ return json.dumps( - {"error": {"message": message, "type": "upstream_unavailable", "attempts": attempts}} + {"error": {"message": message, "type": error_type, "attempts": attempts}} ).encode("utf-8") @@ -289,11 +332,27 @@ def handle_post( pressure: dict[str, float] | None = None, override: bool = False, ) -> GatewayResponse: - """Resolve the model, then try backends in failover order. - - Returns the first backend that produces a response **before the body** (2xx - or 4xx — committed), or a 502 if every backend refused / 5xx'd. ``open_upstream`` - is injected so this is unit-testable without sockets. + """Resolve the model to its ONE owning backend and try it exactly once. + + There is **no cross-backend failover** (issue #91): + :func:`lobes.gateway._routing.order_backends` returns at most one backend — + the owner of the resolved model — so this attempts that owner and nothing + else. ``open_upstream`` is injected so this is unit-testable without sockets. + + The owner's verdict is authoritative, and the status mapping reflects that a + request naming a model has exactly one honest place to go: + + * **2xx / 4xx** → commit to the owner and relay verbatim. A 4xx is a genuine + *client* error (the owner is the only backend that could serve this model, + so e.g. its 404 "model does not exist" is authoritative — never a reason to + try someone else). + * **refusal / timeout / >=500** (the loop exhausts its single attempt) → the + owner is transiently down, so return a RETRYABLE **503** ``backend_unavailable`` + + ``Retry-After`` (issue #14). It is deliberately NOT a 404 (which would be + indistinguishable from "this model id was never valid") and NOT a 502. + * **empty ``order_backends``** (no owner at all) → the only remaining **502** + ``upstream_unavailable``: a malformed routing table, a config bug, not + retryable. Pressure-aware busy shedding (#85): when ``pressure`` is supplied *and* the requested model is a capability tier (``main``/``minor``/``multimodal``, or the @@ -335,7 +394,22 @@ def handle_post( fwd_headers = filter_headers(req_headers) attempts: list[str] = [] - for backend in order_backends(table, served): + ordered = order_backends(table, served) + if not ordered: + # DEGENERATE case ONLY: no backend owns `served` AND none owns + # default_model — order_backends can return an empty list solely for a + # malformed routing table (in practice, one with no primary). That is a + # config/deploy bug, not a transient outage, so it is a TERMINAL 502 + # upstream_unavailable with NO Retry-After — never the retryable 503 a + # present-but-dead owner gets below. + return GatewayResponse( + status=502, + headers=tier_headers + [("Content-Type", _CONTENT_TYPE_JSON)], + body=_error_body("no backend owns the requested model", attempts), + attempts=attempts, + ) + + for backend in ordered: # exactly one backend — no failover chain (#91) try: up = open_upstream( backend, @@ -352,7 +426,8 @@ def handle_post( attempts.append(f"{backend.name}: HTTP {up.status}") up.close() continue - # 2xx or 4xx → commit to this backend (4xx is a client error; no failover). + # 2xx or 4xx → commit to the owner and relay verbatim. A 4xx is a genuine + # CLIENT error: the owner is the only backend that could serve this model. return GatewayResponse( status=up.status, headers=tier_headers + up.headers, @@ -361,10 +436,24 @@ def handle_post( attempts=attempts, ) + # The single owner refused / timed out / 5xx'd. With no failover (#91) it is + # the ONLY backend that could serve `served`, so this is a TRANSIENT owner-down + # state — not "model unknown". Return a retryable 503 + Retry-After whose type + # (backend_unavailable) is distinguishable from both the 429 server_busy shed + # and the degenerate 502 upstream_unavailable above, so a client retries the + # same model instead of treating the failure as terminal (issues #14, #91). return GatewayResponse( - status=502, - headers=tier_headers + [("Content-Type", _CONTENT_TYPE_JSON)], - body=_error_body("all fleet backends are unavailable", attempts), + status=503, + headers=tier_headers + + [ + ("Retry-After", str(BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS)), + ("Content-Type", _CONTENT_TYPE_JSON), + ], + body=_error_body( + "the backend serving this model is unavailable — retry shortly", + attempts, + error_type="backend_unavailable", + ), attempts=attempts, ) @@ -607,6 +696,27 @@ def probe_audio_ready( return None +def _ready_iff_true(snapshot: Mapping[str, bool | None]) -> dict[str, bool]: + """Collapse the readiness tri-state to a strict "ready iff True" bool map. + + The background :class:`~lobes.gateway._readiness.ReadinessCache` reports a + backend as ``None`` when it is dead / missing / unreachable (a probe that + never got a 200 — see that module's docstring). But + :func:`lobes.roles._gateway_role` reads a ``None`` signal as "no live signal + available → fall back to the coarse ``loaded`` proxy", which for a WIRED + backend is ``True``. Those two ``None``s mean OPPOSITE things: the cache's is + "definitely not reachable", roles.py's is "I don't know, assume configured". + Passing the cache's ``None`` straight through would therefore advertise a + wired-but-dead backend as ``ready=True`` in ``/capabilities`` — the exact + issue #92 defect (honesty h14). This bridges the vocabularies: only an + affirmative ``True`` stays ``True`` (ready); both ``None`` (unreachable) and + ``False`` (unhealthy) become a definite ``False``. It mirrors the ``is True`` + filter :func:`lobes.gateway._routing.list_models_payload` applies, so both + advertisement surfaces agree — a dead backend is advertised nowhere. + """ + return {name: signal is True for name, signal in snapshot.items()} + + def capabilities_payload( table: RoutingTable, cfg: ServerConfig, @@ -614,6 +724,7 @@ def capabilities_payload( *, gateway_url: str | None = None, audio_ready: bool | None = None, + backend_ready: Mapping[str, bool | None] | None = None, ) -> dict: """The six first-class roles (issue #81), resolved via the shared registry. @@ -631,14 +742,24 @@ def capabilities_payload( the builder derives it from ``cfg.host``/``cfg.port`` (the CLI/unit path, unchanged). ``audio_ready`` is the live stt/tts readiness signal (issue #89) from :func:`probe_audio_ready`; when ``None`` the builder falls back to the - configured ``bool(audio_url)`` (again the CLI/unit path). Both default to + configured ``bool(audio_url)`` (again the CLI/unit path). ``backend_ready`` is + the live readiness snapshot for the four gateway-fronted roles (issue #92), + keyed by internal ``Backend`` name — exactly what + :meth:`lobes.gateway._readiness.ReadinessCache.current` returns, so the HTTP + route passes it straight through; when ``None`` each role's ``ready`` falls + back to ``loaded`` (the CLI/unit path). All three signal kwargs default to ``None`` so this pure function's shape is unchanged for its non-HTTP callers. """ from lobes.roles import ROLES, build_role_registry # deferred — see the module-level NOTE resolved_env = os.environ if env is None else env registry = build_role_registry( - table, cfg, env=resolved_env, gateway_url=gateway_url, audio_ready=audio_ready + table, + cfg, + env=resolved_env, + gateway_url=gateway_url, + audio_ready=audio_ready, + backend_ready=backend_ready, ) return {role: dataclasses.asdict(registry[role]) for role in ROLES} @@ -655,6 +776,11 @@ class _Handler(BaseHTTPRequestHandler): # Non-blocking host-pressure provider (t6). None → the tier-downgrade layer # is skipped and tier aliases resolve via the static table (the t5 path). pressure_cache: PressureCache | None = None + # Non-blocking background readiness provider (issue #92). None → /v1/models + # lists every wired backend and /capabilities readiness falls back to the + # coarse `loaded` proxy (the offline/unit path). Read only via .current() + # (socket-free); the POST hot path never touches it. + readiness_cache: ReadinessCache | None = None # HTTP/1.1 so we can stream with chunked transfer encoding. protocol_version = "HTTP/1.1" @@ -670,7 +796,13 @@ def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API pressure = self.pressure_cache.current() if self.pressure_cache is not None else None self._send_json(200, fleet_status_payload(self.table, self.server_config, pressure)) elif route == "/v1/models": - self._send_json(200, list_models_payload(self.table)) + # Advertise only backends the live readiness snapshot marks ready + # (issue #92): a wired-but-dead backend must NOT appear here, so a + # client can trust that a listed model id reaches a live engine. The + # snapshot is socket-free (.current() never probes); with no cache + # wired, every backend is listed (the offline/unit path). + ready = self.readiness_cache.current() if self.readiness_cache is not None else None + self._send_json(200, list_models_payload(self.table, ready)) elif route == "/v1/models/supported": # The full catalog of gears you can change to (loaded + the rest), # not just the two currently warm. Non-OpenAI shape; /v1/models stays standard. @@ -678,14 +810,32 @@ def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API elif route == "/capabilities": # The #81 role→endpoint contract: SIX first-class roles resolved to # live metadata via the shared lobes.roles registry. The endpoint is - # the client-reachable origin this request actually dialed (#87) and - # stt/tts readiness is a live probe of the audio backend (#89). + # the client-reachable origin this request actually dialed (#87), + # stt/tts readiness is a live probe of the audio backend (#89), and the + # four gateway-fronted roles' readiness comes from the background + # ReadinessCache snapshot (#92) — read socket-free, no probe here. cfg = self.server_config origin = reachable_origin(self.headers.get("Host"), cfg.public_url) audio_ready = probe_audio_ready(cfg.audio_url) is True if cfg.audio_url else None + # Collapse the cache's tri-state to a strict "ready iff True" map before + # handing it to the registry: at the gateway a None from the cache means + # "probed, unreachable", NOT roles.py's "no signal → fall back to + # loaded". Without this a wired-but-dead backend would advertise + # ready=True here (issue #92 / honesty h14). See _ready_iff_true. + backend_ready = ( + _ready_iff_true(self.readiness_cache.current()) + if self.readiness_cache is not None + else None + ) self._send_json( 200, - capabilities_payload(self.table, cfg, gateway_url=origin, audio_ready=audio_ready), + capabilities_payload( + self.table, + cfg, + gateway_url=origin, + audio_ready=audio_ready, + backend_ready=backend_ready, + ), ) else: self._send_json(404, {"error": {"message": f"not found: {route}", "type": "not_found"}}) @@ -788,12 +938,20 @@ def log_message(self, fmt: str, *args) -> None: # keep request logs tidy in doc def _make_handler( - table: RoutingTable, cfg: ServerConfig, pressure_cache: PressureCache | None = None + table: RoutingTable, + cfg: ServerConfig, + pressure_cache: PressureCache | None = None, + readiness_cache: ReadinessCache | None = None, ) -> type[_Handler]: bound = type( "_BoundHandler", (_Handler,), - {"table": table, "server_config": cfg, "pressure_cache": pressure_cache}, + { + "table": table, + "server_config": cfg, + "pressure_cache": pressure_cache, + "readiness_cache": readiness_cache, + }, ) return bound @@ -803,6 +961,19 @@ def serve(table: RoutingTable, cfg: ServerConfig) -> None: # pragma: no cover # One pressure cache per process: a background daemon thread refreshes it so # the 150 ms sample never lands on the request path. pressure_cache = PressureCache() - httpd = ThreadingHTTPServer((cfg.host, cfg.port), _make_handler(table, cfg, pressure_cache)) + # One readiness cache per process (issue #92). Construction seeds every backend + # to None (unknown) WITHOUT probing, so we do ONE bounded synchronous refresh + # BEFORE binding — otherwise /v1/models would advertise nothing until the + # daemon's first background pass lands (up to one interval), reporting a false + # "fleet is empty" on the very first request. After the seed, start() hands + # refreshes to a background daemon thread so no probe ever lands on the request + # path. Read verbs consult it via .current() (socket-free); the POST hot path + # never touches it. + readiness_cache = ReadinessCache.from_backends(table.backends, start=False) + readiness_cache.refresh() + readiness_cache.start() + httpd = ThreadingHTTPServer( + (cfg.host, cfg.port), _make_handler(table, cfg, pressure_cache, readiness_cache) + ) sys.stderr.write(f"[gateway] listening on {cfg.host}:{cfg.port}\n") httpd.serve_forever() diff --git a/tests/test_gateway_capabilities.py b/tests/test_gateway_capabilities.py index ec0f4ad..180c92c 100644 --- a/tests/test_gateway_capabilities.py +++ b/tests/test_gateway_capabilities.py @@ -21,6 +21,7 @@ import threading import urllib.request from http.server import ThreadingHTTPServer +from types import SimpleNamespace import pytest @@ -274,3 +275,164 @@ def test_integration_capabilities_endpoint_reflects_request_host(capabilities_ga conn.close() for role in ("cortex", "senses", "embedder", "reranker"): assert payload[role]["endpoint"] == "http://gw.example:8001" + + +# --- #92 target c29/h25: origin resolution — override > Host > empty ---------- + + +def test_reachable_origin_public_url_wins_over_host() -> None: + # GATEWAY_PUBLIC_URL always outranks the Host header — a tunnel / Host- + # rewriting proxy declares the true external origin (trailing slash trimmed). + assert ( + S.reachable_origin("spark.local:8001", "https://tunnel.example/") + == "https://tunnel.example" + ) + + +def test_reachable_origin_host_header_echoes_client_own_origin() -> None: + # No GATEWAY_PUBLIC_URL: echo the origin the client actually dialed — its own + # Host — NOT localhost. Regression guard for #92: a LAN client must be told + # its OWN origin. Defaulting public_url to localhost (rejected on the rig, + # where host :8000 is a foreign uvicorn daemon) would point clients at their + # own loopback because public_url outranks Host. + assert S.reachable_origin("spark.local:8001", None) == "http://spark.local:8001" + + +def test_reachable_origin_empty_when_nothing_known() -> None: + # Neither override nor Host → None, so the caller falls back to an empty + # endpoint (never a fabricated host:port URL). + assert S.reachable_origin(None, None) is None + + +def test_capabilities_endpoint_empty_never_contains_gateway_port() -> None: + # #92 / issue #81 t5 criterion 3, end to end: GATEWAY_PUBLIC_URL unset AND no + # Host header (reachable_origin → None) → every gateway-fronted role's endpoint + # is "" — NEVER an absolute URL built from GATEWAY_PORT (host :GATEWAY_PORT may + # be an unrelated daemon). Drives the exact path do_GET /capabilities takes: + # reachable_origin(None, cfg.public_url) → capabilities_payload(gateway_url=...). + env = _full_env(GATEWAY_PORT="8001") + table, cfg = build_config(env) + assert cfg.public_url is None # GATEWAY_PUBLIC_URL unset stays override-only + origin = S.reachable_origin(None, cfg.public_url) # no Host header, no override + assert origin is None + payload = S.capabilities_payload(table, cfg, env=env, gateway_url=origin) + for role in ("cortex", "senses", "embedder", "reranker"): + assert payload[role]["endpoint"] == "" + assert "8001" not in payload[role]["endpoint"] # never the internal port + assert payload[role]["ready"] is False # empty endpoint → never advertised ready + + +def test_capabilities_public_url_wins_over_host_end_to_end() -> None: + # GATEWAY_PUBLIC_URL set → it wins over any Host header, end to end. + env = _full_env(GATEWAY_PUBLIC_URL="https://tunnel.example") + table, cfg = build_config(env) + origin = S.reachable_origin("spark.local:8001", cfg.public_url) + assert origin == "https://tunnel.example" + payload = S.capabilities_payload(table, cfg, env=env, gateway_url=origin) + for role in ("cortex", "senses", "embedder", "reranker"): + assert payload[role]["endpoint"] == "https://tunnel.example" + + +# --- #92 target c15/h14: ReadinessCache folded into /capabilities readiness ---- + + +def test_capabilities_payload_threads_backend_ready() -> None: + # The PURE builder threads a True/False live signal (keyed by internal Backend + # name) straight into each gateway-fronted role's `ready` — `loaded` stays the + # config fact (roles.py t5 contract). The None→not-ready collapse is a + # gateway-BOUNDARY concern (a cache None means "unreachable", not roles.py's + # "no signal") — exercised end to end by the loopback route test below. + env = _full_env() + table, cfg = build_config(env) + backend_ready = {"primary": True, "multimodal": False, "embed": False, "rerank": True} + payload = S.capabilities_payload( + table, cfg, env=env, gateway_url=_GATEWAY_URL, backend_ready=backend_ready + ) + assert payload["cortex"]["loaded"] is True and payload["cortex"]["ready"] is True + assert payload["senses"]["loaded"] is True and payload["senses"]["ready"] is False + assert payload["embedder"]["loaded"] is True and payload["embedder"]["ready"] is False + assert payload["reranker"]["ready"] is True + + +def test_capabilities_payload_backend_ready_none_falls_back_to_loaded() -> None: + # Default (backend_ready=None): ready falls back to loaded — the CLI/offline + # path is unchanged, so the pure builder never needs a live signal to work. + env = _full_env() + table, cfg = build_config(env) + payload = S.capabilities_payload(table, cfg, env=env, gateway_url=_GATEWAY_URL) + for role in ("cortex", "senses", "embedder", "reranker"): + assert payload[role]["ready"] == payload[role]["loaded"] + + +def test_ready_iff_true_collapses_none_and_false_but_keeps_true() -> None: + # The gateway-boundary bridge: a cache None (unreachable/dead) must NOT be read + # as roles.py's "no signal → fall back to loaded", so it is collapsed to a + # definite False; False stays False; only True stays True. This is what keeps a + # wired-but-dead backend (which probes None) off the ready list (honesty h14). + collapsed = S._ready_iff_true({"primary": True, "multimodal": None, "embed": False}) + assert collapsed == {"primary": True, "multimodal": False, "embed": False} + + +@pytest.fixture +def readiness_capabilities_gateway(monkeypatch): + """A loopback /capabilities gateway wired with a ReadinessCache whose verdicts + the test controls and whose probe counts its calls (daemon NOT started; seeded + by one synchronous refresh, mirroring serve()).""" + from lobes.gateway._readiness import ReadinessCache + + env = _full_env() + for k, v in env.items(): + monkeypatch.setenv(k, v) + table, cfg = build_config(env) + verdicts = {b.base_url: True for b in table.backends} + probe_calls: list[str] = [] + + def probe(base_url): + probe_calls.append(base_url) + return verdicts.get(base_url) + + cache = ReadinessCache.from_backends(table.backends, probe=probe, start=False) + cache.refresh() + httpd = ThreadingHTTPServer(("127.0.0.1", 0), S._make_handler(table, cfg, None, cache)) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + host, port = httpd.server_address + try: + yield SimpleNamespace( + base=f"http://{host}:{port}", verdicts=verdicts, probe_calls=probe_calls, cache=cache + ) + finally: + httpd.shutdown() + httpd.server_close() + cache.stop() + + +def test_capabilities_route_dead_backend_ready_false_h14(readiness_capabilities_gateway) -> None: + # h14 end to end: a wired-but-dead backend probes None; the HTTP /capabilities + # route collapses that to ready=False (loaded stays True) — the dead backend is + # NOT advertised ready, even though roles.py would otherwise fall back to loaded. + gw = readiness_capabilities_gateway + # All True initially → every gateway-fronted role ready. + with urllib.request.urlopen(gw.base + "/capabilities", timeout=5) as r: + payload = json.load(r) + for role in ("cortex", "senses", "embedder", "reranker"): + assert payload[role]["ready"] is True + # Kill the multimodal (senses) backend → it probes None. + gw.verdicts["http://vllm-multimodal:8000"] = None + gw.cache.refresh() + with urllib.request.urlopen(gw.base + "/capabilities", timeout=5) as r: + payload = json.load(r) + assert payload["senses"]["loaded"] is True # still wired + assert payload["senses"]["ready"] is False # but dead → not advertised ready + assert payload["cortex"]["ready"] is True # the live one is unaffected + + +def test_capabilities_route_opens_no_probe(readiness_capabilities_gateway) -> None: + # /capabilities is a read verb: it reads the cache via socket-free .current(); + # a GET must add zero probe calls beyond the one synchronous seed. + gw = readiness_capabilities_gateway + seeded = len(gw.probe_calls) + for _ in range(4): + with urllib.request.urlopen(gw.base + "/capabilities", timeout=5) as r: + assert r.status == 200 + assert len(gw.probe_calls) == seeded diff --git a/tests/test_gateway_server.py b/tests/test_gateway_server.py index 947edbb..8832221 100644 --- a/tests/test_gateway_server.py +++ b/tests/test_gateway_server.py @@ -1,6 +1,7 @@ -"""Gateway server tests: handle_post failover decisions (no sockets) + a loopback -integration covering the handler relay (buffered + chunked streaming) and the -``open_upstream`` http.client path.""" +"""Gateway server tests: handle_post routing/status decisions (no sockets, no +cross-backend failover since #91), the readiness-gated /v1/models advertisement +(#92), and a loopback integration covering the handler relay (buffered + chunked +streaming) and the ``open_upstream`` http.client path.""" from __future__ import annotations @@ -65,56 +66,95 @@ def opener(backend, path, body, headers, *, connect_timeout, read_timeout): return opener, calls -# --- handle_post: failover / default / rewrite (no sockets) --------------- +# --- handle_post: status matrix / default / rewrite (no sockets, no failover) - -def test_no_failover_on_connection_refused() -> None: - # INVERTED for issue #91 ("advertised implies reachable"): a dead primary - # must NOT retry against fallback (which serves a different model — the - # forwarded body still names "P", and fallback would either 404 on an - # unknown id or, worse, silently answer as the wrong model). order_backends - # now yields only the owner, so handle_post attempts primary once and stops. - # Interim: this surfaces as the existing 502 upstream_unavailable path (t6 / - # issue #91 converts that to 503 + Retry-After with a distinguishable error - # type — not this task's file, lobes/gateway/server.py is out of scope here). +def test_dead_owner_refused_yields_retryable_503() -> None: + # issue #91 + #14 (t6): a dead primary must NOT retry against fallback (which + # serves a different model — the forwarded body still names "P", and fallback + # would either 404 on an unknown id or, worse, silently answer as the wrong + # model). order_backends yields only the owner, so handle_post attempts primary + # once and stops. The owner is the ONLY backend that can serve "P", so a + # connection refusal is a TRANSIENT owner-down state → a retryable 503 + + # Retry-After (type backend_unavailable), NEVER a terminal 404/502. table, cfg = _cfg() opener, calls = _opener({"primary": S.UpstreamError("refused"), "fallback": 200}) resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) assert [c[0] for c in calls] == ["primary"] # fallback is never dialed - assert resp.status == 502 and resp.upstream is None + assert resp.status == 503 and resp.upstream is None + headers = dict(resp.headers) + assert headers["Retry-After"] == str(S.BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS) + assert json.loads(resp.body)["error"]["type"] == "backend_unavailable" -def test_no_failover_on_5xx() -> None: - # INVERTED for issue #91: a same-task 5xx (e.g. EngineDeadError) must not - # fail over to a backend serving a different model. See - # test_no_failover_on_connection_refused for the full rationale; same t6 - # note re: 502 -> 503+Retry-After being out of this task's scope. +def test_dead_owner_5xx_yields_retryable_503() -> None: + # issue #91 + #14 (t6): a same-owner 5xx (e.g. EngineDeadError) is not failed + # over to a backend serving a different model. The owner is the only backend + # that can serve "P", so its 5xx is a transient owner-down state → the same + # retryable 503 (backend_unavailable), not a 502. table, cfg = _cfg() opener, calls = _opener({"primary": 503, "fallback": 200}) resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) assert [c[0] for c in calls] == ["primary"] # fallback is never dialed - assert resp.status == 502 + assert resp.status == 503 + assert json.loads(resp.body)["error"]["type"] == "backend_unavailable" + assert dict(resp.headers)["Retry-After"] == str(S.BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS) -def test_no_failover_on_4xx() -> None: +def test_owner_4xx_relayed_verbatim_as_client_error() -> None: table, cfg = _cfg() opener, calls = _opener({"primary": 400, "fallback": 200}) resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) assert [c[0] for c in calls] == ["primary"] # 4xx is a client error → returned verbatim assert resp.status == 400 + assert resp.upstream is not None # relayed upstream, not a gateway-generated body -def test_all_backends_down_returns_502() -> None: +def test_owner_404_relayed_verbatim_not_converted_to_503() -> None: + # A 404 from the owner is its authoritative verdict, relayed verbatim — NOT + # converted to a 503. Since the owner is the ONLY backend that can serve this + # model (#91), its "model does not exist" IS a client error, not a transient + # backend outage. (Only a refusal / timeout / >=500 becomes the retryable 503.) + table, cfg = _cfg() + opener, calls = _opener({"primary": 404}) + resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) + assert [c[0] for c in calls] == ["primary"] + assert resp.status == 404 + assert resp.upstream is not None + + +def test_dead_owner_all_down_returns_503_with_single_attempt() -> None: # With no cross-backend failover (issue #91), only the owner is ever # attempted — fallback's outcome is irrelevant to a "P" request and is # never dialed. attempts therefore has a single entry, not one per backend. - # (t6 converts this 502 to 503 + Retry-After; out of scope for this task.) + # t6: the dead owner yields a retryable 503 + Retry-After, never a terminal 502. table, cfg = _cfg() opener, calls = _opener({"primary": S.UpstreamError("x"), "fallback": S.UpstreamError("y")}) resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) assert [c[0] for c in calls] == ["primary"] # fallback never dialed - assert resp.status == 502 and resp.upstream is None + assert resp.status == 503 and resp.upstream is None assert json.loads(resp.body)["error"]["attempts"] == ["x"] + assert json.loads(resp.body)["error"]["type"] == "backend_unavailable" + assert dict(resp.headers)["Retry-After"] == str(S.BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS) + + +def test_empty_order_backends_is_terminal_502() -> None: + # The ONLY 502 left after issue #91/#14: order_backends returns an EMPTY list, + # which can happen solely for a malformed routing table (no backend owns the + # served model AND none owns default_model — here, a table with zero backends). + # This is a config/deploy bug, not a transient outage, so it is a terminal 502 + # upstream_unavailable with NO Retry-After — distinct from the retryable 503 a + # (present but) dead owner yields. + from lobes.gateway._routing import RoutingTable + + table = RoutingTable(backends=(), default_model="P", aliases={}) + _, cfg = _cfg() + opener, calls = _opener({}) + resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"P"}', opener) + assert calls == [] # nothing to dial — no owner exists + assert resp.status == 502 and resp.upstream is None + assert json.loads(resp.body)["error"]["type"] == "upstream_unavailable" + assert "Retry-After" not in dict(resp.headers) # terminal, not retryable def test_missing_model_routes_to_default() -> None: @@ -454,9 +494,13 @@ def test_handle_post_minor_still_served_under_pressure() -> None: assert headers["X-Lobes-Tier-Reason"] == "default" -def test_busy_429_is_distinguishable_from_502_all_down() -> None: - # The busy 429 (type server_busy) and the hard 502 (type upstream_unavailable) - # differ by status code AND error type — a client can tell "retry" from "down". +def test_busy_429_is_distinguishable_from_503_owner_down() -> None: + # The busy 429 (type server_busy, the pressure shed) and the owner-down 503 + # (type backend_unavailable) differ by status code AND error type — a client + # can tell "the fleet is under pressure, back off the tier" from "this model's + # own backend is down, retry it". Both are retryable but semantically distinct, + # and both are distinct from the terminal 502 (malformed table) and from a + # relayed upstream 404 ("model does not exist"). table, cfg = _fleet_cfg() opener, _ = _opener({"minor": 200, "multimodal": 200, "primary": 200}) busy = S.handle_post( @@ -473,7 +517,7 @@ def test_busy_429_is_distinguishable_from_502_all_down() -> None: table, cfg, "/v1/chat/completions", [], b'{"model":"PRIMARY"}', down_opener ) assert busy.status == 429 and json.loads(busy.body)["error"]["type"] == "server_busy" - assert down.status == 502 and json.loads(down.body)["error"]["type"] == "upstream_unavailable" + assert down.status == 503 and json.loads(down.body)["error"]["type"] == "backend_unavailable" def test_handle_post_override_forces_main_under_pressure() -> None: @@ -662,3 +706,185 @@ def test_fleet_status_payload_omits_pressure_block_when_unwired() -> None: table, cfg = _fleet_cfg() payload = S.fleet_status_payload(table, cfg, probe=_stub_probe) assert "pressure" not in payload + + +# --- issue #91 h4: a dead primary NEVER dials the multimodal (Gemma) backend --- + + +def test_dead_primary_never_opens_multimodal_connection() -> None: + # h4 (the "no-Gemma" invariant): with the cortex/primary engine dead, a + # request naming the cortex served model must NEVER open a connection to the + # multimodal (senses/Gemma) backend — a caller who asked for cortex must not + # silently receive a Gemma answer (final_authority role-contract, issue #81). + # Asserted on the injected opener's CALL SITES, not just the status code. + table, cfg = _fleet_cfg() + opener, calls = _opener( + {"primary": S.UpstreamError("engine dead"), "multimodal": 200, "minor": 200} + ) + resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"PRIMARY"}', opener) + dialed = [c[0] for c in calls] + assert dialed == ["primary"] # ONLY the owner + assert "multimodal" not in dialed and "minor" not in dialed # never a foreign gear + assert resp.status == 503 + assert json.loads(resp.body)["error"]["type"] == "backend_unavailable" + + +# --- the "converse" honesty question: an unknown (never-advertised) model id --- + + +def test_unknown_model_id_routes_to_default_owner_documented_choice() -> None: + # DOCUMENTED CHOICE, not an accident (t6 converse honesty condition): today + # resolve_model falls back to default_model for an unknown, non-empty model + # id, so a request naming a model that was NEVER in /v1/models is served by + # the default (primary) backend, its body REWRITTEN to the primary's served + # name. This test LOCKS the current behaviour so it is a deliberate choice. + # The t6 report recommends this arguably should 404 (model_not_found) to match + # OpenAI + the "advertised implies reachable" honesty theme — but resolve_model + # lives in _routing.py (out of this task's scope), so it is not changed here. + table, cfg = _cfg() + opener, calls = _opener({"primary": 200, "fallback": 200}) + resp = S.handle_post( + table, cfg, "/v1/chat/completions", [], b'{"model":"never-in-v1-models"}', opener + ) + assert [c[0] for c in calls] == ["primary"] # routed to the default owner + assert json.loads(calls[0][1])["model"] == "P" # rewritten to primary's served name + assert resp.status == 200 + + +# --- issue #92: ReadinessCache wired into /v1/models (advertised => reachable) -- + +from lobes.gateway._readiness import ReadinessCache # noqa: E402 +from lobes.gateway._routing import list_models_payload # noqa: E402 + + +def test_list_models_payload_filters_by_readiness() -> None: + # Pure filter: with a readiness map (keyed by backend NAME), only backends + # whose signal is True are listed. None (dead/missing) and False (unhealthy) + # are both hidden — "advertise it anyway" is exactly the #92 defect. + table, _cfg_ = _cfg() # backends: primary "P", fallback "F" + all_listed = list_models_payload(table) # no map → every wired backend + assert [m["id"] for m in all_listed["data"]] == ["P", "F"] + only_primary = list_models_payload(table, {"primary": True, "fallback": None}) + assert [m["id"] for m in only_primary["data"]] == ["P"] # None hidden + still_only_primary = list_models_payload(table, {"primary": True, "fallback": False}) + assert [m["id"] for m in still_only_primary["data"]] == ["P"] # False hidden + none_ready = list_models_payload(table, {"primary": None, "fallback": None}) + assert none_ready["data"] == [] # nothing ready → nothing advertised + + +@pytest.fixture +def ready_gateway(monkeypatch): + """A loopback gateway whose ReadinessCache verdicts + owner liveness are both + caller-controllable, and whose readiness probe COUNTS its calls. + + The daemon is deliberately NOT started (``start=False``); the fixture seeds + the snapshot with a single synchronous ``refresh()`` — exactly mirroring what + ``serve()`` does before it binds — so the probe-call count is deterministic + (one per backend) and the hot-path test can assert it stays put. + """ + from types import SimpleNamespace + + table, cfg = _cfg() # backends: primary "P" @ vllm-primary, fallback "F" @ vllm-fallback + verdicts = {b.base_url: True for b in table.backends} + probe_calls: list[str] = [] + + def probe(base_url): + probe_calls.append(base_url) + return verdicts.get(base_url) + + cache = ReadinessCache.from_backends(table.backends, probe=probe, start=False) + cache.refresh() # one synchronous seed BEFORE binding — the serve() ordering + + alive = {b.name: True for b in table.backends} # owner liveness for open_upstream + open_calls: list[str] = [] + + def fake_open(backend, path, body, headers, *, connect_timeout, read_timeout): + open_calls.append(backend.name) + if not alive.get(backend.name, False): + raise S.UpstreamError(f"{backend.name}: refused") + return _FakeUpstream(200, body=b'{"echo": "' + backend.name.encode() + b'"}') + + monkeypatch.setattr(S, "open_upstream", fake_open) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), S._make_handler(table, cfg, None, cache)) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + host, port = httpd.server_address + try: + yield SimpleNamespace( + base=f"http://{host}:{port}", + verdicts=verdicts, + probe_calls=probe_calls, + alive=alive, + open_calls=open_calls, + cache=cache, + ) + finally: + httpd.shutdown() + httpd.server_close() + cache.stop() + + +def test_v1_models_correct_on_very_first_request(ready_gateway) -> None: + # The startup window is closed by the one synchronous refresh() before bind: + # /v1/models is correct on the VERY FIRST request, not empty for a probe + # interval. Both backends probe True → both listed. + gw = ready_gateway + with urllib.request.urlopen(gw.base + "/v1/models", timeout=5) as r: + ids = [m["id"] for m in json.load(r)["data"]] + assert ids == ["P", "F"] + + +def test_v1_models_hides_dead_or_unhealthy_backend(ready_gateway) -> None: + # A wired-but-dead backend must not be advertised (issue #92, honesty h14). + # None (container gone / nothing listening) and False (reached but unhealthy) + # are BOTH "not ready" — only True advertises. + gw = ready_gateway + gw.verdicts["http://vllm-fallback:8000"] = None # missing container → probes None + gw.cache.refresh() + with urllib.request.urlopen(gw.base + "/v1/models", timeout=5) as r: + assert [m["id"] for m in json.load(r)["data"]] == ["P"] # dead F dropped + gw.verdicts["http://vllm-fallback:8000"] = False # reached but unhealthy + gw.cache.refresh() + with urllib.request.urlopen(gw.base + "/v1/models", timeout=5) as r: + assert [m["id"] for m in json.load(r)["data"]] == ["P"] # False hidden too + + +def test_hot_path_opens_no_readiness_probe(ready_gateway) -> None: + # The POST hot path must open NO probe connection: readiness is read only via + # the cache's socket-free .current() (in fact do_POST never touches the + # readiness cache). Seed once (probe count == 2 backends), fire N completions, + # assert the probe count is UNCHANGED. + gw = ready_gateway + seeded = len(gw.probe_calls) # from the single refresh() in the fixture + assert seeded == 2 # one probe per wired backend, nothing more + for _ in range(5): + req = urllib.request.Request( + gw.base + "/v1/chat/completions", + data=b'{"model":"P"}', + headers={"Content-Type": "application/json"}, + ) + with urllib.request.urlopen(req, timeout=5) as r: + assert r.status == 200 + assert len(gw.probe_calls) == seeded # zero probes added by the hot path + + +def test_race_listed_then_owner_killed_yields_503_not_404(ready_gateway) -> None: + # h23 (the race invariant): model "P" is listed in /v1/models; the owner is + # then killed; a completion naming "P" returns 503 + Retry-After — NEVER 404. + # "Advertised implies reachable" held when it was listed; once the owner dies + # the honest answer is "retry", not the terminal "no such model" a + # cross-backend failover 404 (issue #91) used to produce. + gw = ready_gateway + with urllib.request.urlopen(gw.base + "/v1/models", timeout=5) as r: + assert "P" in [m["id"] for m in json.load(r)["data"]] # advertised now + gw.alive["primary"] = False # kill the owner AFTER it was advertised + req = urllib.request.Request( + gw.base + "/v1/chat/completions", + data=b'{"model":"P"}', + headers={"Content-Type": "application/json"}, + ) + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(req, timeout=5) + assert exc.value.code == 503 # NEVER 404 + assert exc.value.headers.get("Retry-After") == str(S.BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS) + assert json.loads(exc.value.read())["error"]["type"] == "backend_unavailable" From 1c53a5d05d7e9b61c67b02074d9bb71109d23fca Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 10:46:50 +0300 Subject: [PATCH 11/23] t8: replace placeholder-media smoke claims with ground-truth perception tests (#74, #96) Layer B's image/audio "confirmed" claims only proved the wire (1x1 PNG / near-silent WAV, asserting non-empty content) -- a model ignoring the media entirely would still pass. Add real perception tests instead: * test_live_multimodal_image_perception_names_colour[red|blue]: a solid-colour PNG generated in-process (stdlib zlib/struct only) must be named correctly. Verified live against http://localhost:8001 (red -> 'Red', blue -> 'Blue'), and verified falsifiable via a negative control (asserting 'red' against a blue image fails). * test_live_multimodal_audio_perception_transcribes_known_word: synthesizes a known word via the rig's own TTS (POST /v1/audio/speech) and asserts the transcription contains it. Fails loudly (not skip) on the current 404 -- AUDIO_URL not yet wired into this running gateway, issue #96, fixed in the base fleet template by t4 but pending redeploy -- and gives TTS a bounded retry so a poisoned-CUDA-context 500 (Chatterbox's known failure mode) reports as "TTS backend unhealthy", distinct from "senses cannot hear". The old placeholder-media tests are kept but renamed to test_live_multimodal_accepts_{image,audio}_content_part_wire_check with docstrings stating they assert transport only, not perception. Offline suite unchanged: 1151 passed (baseline), 9 skipped (+3 new live-gated tests, still gated on LOBES_SMOKE_BASE_URL). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- tests/test_smoke_duo.py | 272 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 263 insertions(+), 9 deletions(-) diff --git a/tests/test_smoke_duo.py b/tests/test_smoke_duo.py index ad73bff..a32ea89 100644 --- a/tests/test_smoke_duo.py +++ b/tests/test_smoke_duo.py @@ -13,10 +13,17 @@ Layer B — Live gateway (skipped unless LOBES_SMOKE_BASE_URL is set, i.e. on the DGX Spark where t7 validates the physical hardware): * model=main: a plain text prompt returns non-empty assistant content. - * model=multimodal (Gemma 4 12B): an image+text chat request using the - OpenAI content-parts shape (type=image_url) returns valid output. - * model=multimodal: an audio+text chat request using the OpenAI - content-parts shape (type=input_audio / format=wav) returns valid output. + * Perception (ground-truth) checks — these are the ones that can actually + fail if the model ignores the media, unlike the wire checks below: + - model=multimodal names the colour of an in-process-generated solid- + colour PNG (two colours, so a lucky guess/prior can't pass silently). + - model=multimodal transcribes a known word synthesized by the rig's own + TTS (Chatterbox, via POST /v1/audio/speech) and sent back as audio. + * Wire checks (transport only, NOT perception — see the docstring on each): + - model=multimodal accepts an image_url content-part built from a 1x1 + placeholder PNG and returns non-empty text. + - model=multimodal accepts an input_audio content-part built from a tiny + silent WAV and returns non-empty text. All live calls use stdlib urllib only (mirroring lobes/assess.py). """ @@ -27,7 +34,10 @@ import json import os import struct +import time +import urllib.error import urllib.request +import zlib from pathlib import Path import pytest @@ -58,13 +68,35 @@ _MINOR_ID = "Qwen/Qwen3.5-4B" _PRIMARY_ID = "sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP" -# Tiny 1×1 RGB PNG (valid PNG, useful as a minimal image payload). +# Tiny 1×1 RGB PNG (valid PNG, useful as a minimal image payload). Used only by +# the *wire-check* tests below — it proves nothing about perception because a +# 1×1 placeholder carries no verifiable content. See _solid_png() for the +# ground-truth image used by the perception tests. _TINY_PNG_B64 = ( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAIAAACQd1PeAAAADElEQVQI12P4" "z8AAAAACAAHiIbwzAAAAAElFTkSuQmCC" ) +def _solid_png(rgb: tuple[int, int, int], size: int = 96) -> bytes: + """Build a solid-colour PNG in-process, stdlib only (zlib + struct). + + Used by the live image-perception test: a model that actually reads + pixels should be able to name this colour; a model that ignores the + image cannot (verified as a negative control — see the task report for + tests/test_smoke_duo.py's t8 commit). + """ + + def chunk(tag: bytes, data: bytes) -> bytes: + c = tag + data + return struct.pack(">I", len(data)) + c + struct.pack(">I", zlib.crc32(c)) + + ihdr = struct.pack(">IIBBBBB", size, size, 8, 2, 0, 0, 0) # 8-bit truecolour + row = b"\x00" + bytes(rgb) * size # filter byte 0 + RGB pixels + idat = zlib.compress(row * size) + return b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", ihdr) + chunk(b"IDAT", idat) + chunk(b"IEND", b"") + + # Tiny silent WAV: 8 kHz, 8-bit, mono, 1 sample. Built with struct so the # bytes are self-documenting and won't drift from the actual format. def _build_tiny_wav_b64() -> str: @@ -267,6 +299,74 @@ def _post_chat(base_url: str, payload: dict, timeout: int = 120) -> dict: return json.load(r) +# The word synthesized by TTS and expected back from the transcription. Common +# enough vocabulary that a model which *can* hear will produce it verbatim. +_AUDIO_WORD = "banana" + +# Chatterbox has a recorded poisoned-CUDA-context failure mode: once tripped it +# returns 500 to every request until the container is restarted (issue #89, +# docs/chatterbox-tts.md). A bounded retry rides out a transient blip; if every +# attempt 500s we report "TTS backend unhealthy" rather than silently treating +# it as "senses cannot hear" — those are different findings. +_TTS_RETRIES = 3 +_TTS_RETRY_BACKOFF_SECONDS = 2.0 + + +def _synthesize_speech(base_url: str, text: str) -> bytes: + """POST /v1/audio/speech and return the raw WAV bytes. + + A 404 means the audio facade isn't reachable through the gateway at all — + ``AUDIO_URL`` never reached the gateway container (issue #96), fixed in the + base fleet template by task t4 but not yet applied to a running deployment + that hasn't been re-scaffolded. That is a redeploy problem, not a flaky + call, so it fails immediately (no retry) with a message naming the issue. + + A 500 (or a connection failure) is retried a bounded number of times before + failing with a message that says "TTS backend unhealthy" — distinct from a + senses-cannot-hear finding, which only the transcription assertion in + ``test_live_multimodal_audio_perception_transcribes_known_word`` can make. + """ + url = base_url.rstrip("/") + "/v1/audio/speech" + payload = { + "model": "chatterbox", + "input": text, + "voice": "default", + "response_format": "wav", + } + data = json.dumps(payload).encode() + last_status: int | str = "no response" + last_body = b"" + for attempt in range(1, _TTS_RETRIES + 1): + req = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}) + try: + with urllib.request.urlopen(req, timeout=60) as r: # local endpoint only + return r.read() + except urllib.error.HTTPError as exc: + body = exc.read() + if exc.code == 404: + pytest.fail( + "POST /v1/audio/speech returned 404 -- the audio facade is not " + "wired into this gateway deployment (AUDIO_URL never reached the " + "gateway container, issue #96; fixed in the base fleet template " + "by task t4). Re-scaffold this deployment (`lobes init --fleet " + "--audio` against the current templates) before the audio " + f"perception test can run. Response body: {body!r}" + ) + last_status, last_body = exc.code, body + except urllib.error.URLError as exc: + last_status, last_body = "connection error", str(exc.reason).encode() + if attempt < _TTS_RETRIES: + time.sleep(_TTS_RETRY_BACKOFF_SECONDS) + pytest.fail( + f"TTS backend unhealthy: POST /v1/audio/speech failed on all " + f"{_TTS_RETRIES} attempts (last status={last_status}, body={last_body!r}). " + "This matches Chatterbox's known poisoned-CUDA-context failure mode " + "(docs/chatterbox-tts.md, issue #89) -- cleared only by restarting the " + "chatterbox container -- and is a TTS-backend finding, not evidence " + "that senses cannot hear." + ) + + @_live def test_live_main_text_returns_nonempty_content() -> None: """model=main (27B primary) responds with non-empty text to a plain prompt.""" @@ -284,9 +384,156 @@ def test_live_main_text_returns_nonempty_content() -> None: assert content.strip(), f"model=main returned empty content; full response: {resp}" +# --------------------------------------------------------------------------- +# Layer B — Perception (ground-truth) checks. +# +# Unlike the wire checks below, these send content whose correct answer is +# known in advance and assert the model reports THAT answer -- a model that +# ignores the media (or a broken vision/audio path that returns fluent but +# unrelated text) fails these, not just a request that never got a 200. +# --------------------------------------------------------------------------- + +# Two colours, not one: a single colour could be a lucky guess or a language +# prior ("things are often red"). Two independent correct answers is evidence +# the model actually read the pixels. +_COLOUR_CASES = [ + ("red", (255, 0, 0)), + ("blue", (0, 0, 255)), +] + + +@_live +@pytest.mark.parametrize("colour_name,rgb", _COLOUR_CASES) +def test_live_multimodal_image_perception_names_colour(colour_name: str, rgb) -> None: + """model=multimodal (Gemma 4 12B) names the colour of a ground-truth image. + + Proves perception, not just transport: the PNG is generated in-process + (stdlib zlib/struct, no external asset) as a solid fill of a known colour, + and the assertion requires that exact colour's name in the reply. A model + that ignores the image content cannot pass this by accident -- verified as + a negative control while writing this test (feeding a blue image while + asserting for "red" fails; see the t8 task report for the transcript). + Compare test_live_multimodal_accepts_image_content_part_wire_check below, + which only proves the wire and asserts nothing about correctness. + """ + base_url = os.environ["LOBES_SMOKE_BASE_URL"] + png_b64 = base64.b64encode(_solid_png(rgb)).decode() + resp = _post_chat( + base_url, + { + "model": "multimodal", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": "What single colour fills this image? Answer with one word.", + }, + { + "type": "image_url", + "image_url": {"url": f"data:image/png;base64,{png_b64}"}, + }, + ], + } + ], + "max_tokens": 16, + "temperature": 0, + }, + ) + choices = resp.get("choices") or [] + assert ( + choices + ), f"model=multimodal image perception ({colour_name}): empty choices; response: {resp}" + content = choices[0].get("message", {}).get("content") or "" + assert colour_name in content.lower(), ( + f"model=multimodal did not name {colour_name!r} for a solid-{colour_name} " + f"ground-truth image; got {content!r}. This means the model is not reading " + f"image pixels (or the vision path is broken). Full response: {resp}" + ) + + +@_live +def test_live_multimodal_audio_perception_transcribes_known_word() -> None: + """model=multimodal (Gemma 4 12B) transcribes a ground-truth spoken word. + + Proves perception, not just transport: the word is synthesized fresh by + the rig's own TTS (Chatterbox, via POST /v1/audio/speech on this same + gateway) so the "correct answer" is known in advance, then sent back to + model=multimodal as an input_audio content-part. A model that ignores the + audio (or fabricates fluent-but-wrong text) fails the containment check. + Compare test_live_multimodal_accepts_audio_content_part_wire_check below, + which only proves the wire and asserts nothing about correctness. + + This currently fails loudly on deployments where AUDIO_URL hasn't reached + the gateway container (issue #96, /v1/audio/speech -> 404) -- that is a + redeploy problem, not a perception failure, and is reported as such by + _synthesize_speech() rather than silently skipped. + """ + base_url = os.environ["LOBES_SMOKE_BASE_URL"] + wav_bytes = _synthesize_speech(base_url, _AUDIO_WORD) + wav_b64 = base64.b64encode(wav_bytes).decode() + + resp = _post_chat( + base_url, + { + "model": "multimodal", + "messages": [ + { + "role": "user", + "content": [ + { + "type": "text", + "text": ( + "Transcribe the speech in this audio. " + "Reply with only the words spoken." + ), + }, + { + "type": "input_audio", + "input_audio": {"data": wav_b64, "format": "wav"}, + }, + ], + } + ], + "max_tokens": 32, + "temperature": 0, + }, + ) + choices = resp.get("choices") or [] + assert choices, f"model=multimodal audio perception: empty choices; response: {resp}" + content = choices[0].get("message", {}).get("content") or "" + assert _AUDIO_WORD in content.lower(), ( + f"model=multimodal did not transcribe the known word {_AUDIO_WORD!r} " + f"synthesized by Chatterbox; got {content!r}. This means senses cannot " + f"hear (or the audio path is broken) -- not a TTS-backend problem, " + f"which _synthesize_speech() would have reported separately. " + f"Full response: {resp}" + ) + + +# --------------------------------------------------------------------------- +# Layer B — Wire checks (transport only, NOT perception). +# +# These assert only that the gateway accepts an image_url / input_audio +# content-part and returns a 200 with non-empty text. The payloads are a 1x1 +# placeholder PNG and a near-silent WAV with no verifiable content, so a model +# that completely ignores the media still passes. See the perception tests +# above for assertions the model would actually fail if it weren't looking/ +# listening. +# --------------------------------------------------------------------------- + + @_live -def test_live_multimodal_image_text_returns_valid_output() -> None: - """model=multimodal (Gemma 4 12B) accepts an image+text content-parts request.""" +def test_live_multimodal_accepts_image_content_part_wire_check() -> None: + """model=multimodal (Gemma 4 12B) accepts an image_url content-part (transport only). + + This asserts the request is well-formed and answered, NOT that the model + perceived the image -- the payload is a 1x1 placeholder PNG with no + verifiable content, so a model that ignores the image entirely still + passes. See test_live_multimodal_image_perception_names_colour for the + ground-truth perception assertion. + """ base_url = os.environ["LOBES_SMOKE_BASE_URL"] resp = _post_chat( base_url, @@ -315,8 +562,15 @@ def test_live_multimodal_image_text_returns_valid_output() -> None: @_live -def test_live_multimodal_audio_text_returns_valid_output() -> None: - """model=multimodal (Gemma 4 12B) accepts an audio+text content-parts request.""" +def test_live_multimodal_accepts_audio_content_part_wire_check() -> None: + """model=multimodal (Gemma 4 12B) accepts an input_audio content-part (transport only). + + This asserts the request is well-formed and answered, NOT that the model + perceived the audio -- the payload is a near-silent single-sample WAV with + no verifiable content, so a model that ignores the audio entirely still + passes. See test_live_multimodal_audio_perception_transcribes_known_word + for the ground-truth perception assertion. + """ base_url = os.environ["LOBES_SMOKE_BASE_URL"] resp = _post_chat( base_url, From 10cc21567c375f505b3b12a9963e404c04ddbf4a Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 10:57:20 +0300 Subject: [PATCH 12/23] t6b: make advertised-implies-reachable invariants self-enforcing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes the two gaps t6 left as caller-discipline patches. Gap 1 (#92/h14) — build_role_registry now self-enforces the readiness invariant its own docstring promises. A SUPPLIED backend_ready mapping is authoritative: a present None, a present False, and a missing key all mean NOT ready (ready = get(name) is True), still clamped on loaded + non-empty endpoint. Only an OMITTED mapping falls back to the loaded proxy (back-compat). This removes the trap where the readiness cache's None (UNREACHABLE) was read as roles.py's None (no signal -> fall back to loaded=True), resurrecting the #92 defect. server.py's _ready_iff_true bridge is now redundant and deleted; the /capabilities route passes ReadinessCache.current() straight through, and the coercion is the builder's job for every caller, not one call site. Gap 2 (h23 converse) — an unknown model id is no longer silently served under the default backend's weights. New pure predicate is_unknown_model distinguishes UNKNOWN (a non-empty id that is neither an alias nor any wired backend's served name) from UNSPECIFIED (missing/blank -> default_model, still served). handle_post 404s an unknown id (model_not_found) before routing. Unknown-ness is decided against the ROUTING TABLE, never the readiness-filtered /v1/models list: a wired-but-dead backend is dropped from /v1/models but is still KNOWN, so it routes to its owner and yields the retryable 503, not a 404 (keeps #91 fixed). resolve_model's signature is unchanged (companion-predicate design), so its many callers and the tier tests are unaffected. Inverted tests: test_backend_ready_missing_entry_falls_back_to_loaded -> _is_not_ready; added present-None coverage; replaced test_unknown_model_id_routes_to_default_owner_documented_choice with the h23 converse 404 test plus wired-but-dead 503-not-404 proofs at both the handle_post and route level. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/gateway/_routing.py | 54 +++++++++++++++ lobes/gateway/server.py | 87 ++++++++++++++++-------- lobes/roles.py | 104 ++++++++++++++++++++++------- tests/test_gateway_capabilities.py | 45 ++++++++----- tests/test_gateway_routing.py | 50 +++++++++++++- tests/test_gateway_server.py | 89 ++++++++++++++++++++---- tests/test_roles.py | 46 ++++++++++--- 7 files changed, 384 insertions(+), 91 deletions(-) diff --git a/lobes/gateway/_routing.py b/lobes/gateway/_routing.py index 44a6f2c..7e20d73 100644 --- a/lobes/gateway/_routing.py +++ b/lobes/gateway/_routing.py @@ -102,6 +102,16 @@ def resolve_model(table: RoutingTable, requested: str | None) -> str: An alias resolves to its target; a name some backend already serves resolves to itself; anything else (``None`` or unknown) resolves to ``default_model``. + + This never distinguishes *unspecified* from *unknown* — it always returns a + concrete served name so :func:`order_backends` always has an owner to try. + That UNSPECIFIED-vs-UNKNOWN policy lives one level up, in + :func:`is_unknown_model` + :func:`lobes.gateway.server.handle_post`: an + unknown non-empty id is rejected (404) *before* ``handle_post`` ever calls + this, so ``resolve_model``'s unknown→default fall-back is now a pure-routing + safety net (e.g. for an internal caller passing a stale name), not the path a + client's unknown model id takes. Kept unchanged so its many callers (the tier + tests, ``order_backends``) are unaffected. """ if requested: if requested in table.aliases: @@ -112,6 +122,50 @@ def resolve_model(table: RoutingTable, requested: str | None) -> str: return table.default_model +def is_unknown_model(table: RoutingTable, requested: str | None) -> bool: + """True when ``requested`` is a NON-EMPTY id that was NEVER advertised — + neither an alias nor any wired backend's ``served_name`` — so it must not be + silently served under the default backend's weights (honesty h23, issue #91). + + Distinguishes UNKNOWN from UNSPECIFIED — the distinction :func:`resolve_model` + deliberately does not make: + + * **Unspecified** — ``requested`` is ``None`` or ``""`` (a missing/blank + ``model`` field). This is NOT unknown: it intentionally routes to + ``default_model`` (see :func:`resolve_model`) and is served. Returns + ``False``. + * **Known** — ``requested`` is an alias or a wired backend's served name. + Returns ``False``. + * **Unknown** — a non-empty id that is neither. Returns ``True``; the caller + (:func:`lobes.gateway.server.handle_post`) turns this into a 404 + ``model_not_found`` rather than routing it to the default owner. + + **Decided against the ROUTING TABLE, never the readiness-filtered + ``/v1/models`` list.** A backend that is wired but dead/warming is dropped + from ``/v1/models`` (see :func:`list_models_payload`) yet its ``served_name`` + is still in ``table.backends`` — so it is *known*, and a request naming it + routes to its owner and yields a retryable **503** (owner down), NOT a 404. + Deciding unknown-ness against the readiness list instead would 404 a merely + warming/transiently-dead backend and reintroduce issue #91. Unknown-ness is a + question about *wiring* ("is this id in the table at all"), not *liveness*. + + ``default_model`` is always treated as KNOWN, even in the degenerate case of a + malformed table where no backend actually serves it: naming the deployment's + declared default identity explicitly is equivalent to leaving ``model`` + unspecified — both route to ``default_model`` — so that path stays a terminal + **502** ``upstream_unavailable`` (the malformed-table signal), not a 404. In a + well-formed table ``default_model`` is some wired backend's served name, so + this clause is redundant there; it only matters for the pathological table. + """ + if not requested: + return False # unspecified (missing/blank) → routes to default, not unknown + if requested == table.default_model: + return False # the declared default identity is known (see docstring) + if requested in table.aliases: + return False + return not any(backend.served_name == requested for backend in table.backends) + + def _backend_for(table: RoutingTable, served_name: str) -> Backend | None: for backend in table.backends: if backend.served_name == served_name: diff --git a/lobes/gateway/server.py b/lobes/gateway/server.py index 56d8861..8fdaefb 100644 --- a/lobes/gateway/server.py +++ b/lobes/gateway/server.py @@ -55,6 +55,7 @@ Backend, RoutingTable, is_audio_path, + is_unknown_model, list_models_payload, order_backends, resolve_model, @@ -306,6 +307,28 @@ def _error_body( ).encode("utf-8") +def _model_not_found_body(model: str) -> bytes: + """OpenAI/vLLM-shaped 404 body for an id that was NEVER advertised (honesty h23). + + Mirrors the ``model_not_found`` error an OpenAI/vLLM backend emits for an + unknown model, so a client sees a consistent 404 shape whether it hit the + gateway or a backend directly. This is NOT a contradiction of "advertised + implies reachable" (issue #92): the invariant is that a model *listed in + ``/v1/models``* never 404s — an id that was never listed *should* 404. It is + the deliberate converse of the never-404 race guarantee (see + :func:`handle_post`). + """ + return json.dumps( + { + "error": { + "message": f"The model `{model}` does not exist.", + "type": "model_not_found", + "code": "model_not_found", + } + } + ).encode("utf-8") + + def _busy_body(requested_tier: str) -> bytes: """Return the JSON body for a 429 busy (shed) response.""" _LANE_LABELS = {"main": "cortex", "multimodal": "senses"} @@ -342,6 +365,18 @@ def handle_post( The owner's verdict is authoritative, and the status mapping reflects that a request naming a model has exactly one honest place to go: + * **unknown model id** → a **404** ``model_not_found`` generated HERE, before + any routing (honesty h23). A non-empty ``model`` that is neither an alias + nor any WIRED backend's served name (:func:`is_unknown_model`) was never + advertised, so it must not be silently served under the default backend's + weights. This is the deliberate converse of "advertised implies reachable" + (issue #92): a model *listed in ``/v1/models``* never 404s, but one never + listed *should*. Unknown-ness is decided against the ROUTING TABLE, never + the readiness-filtered ``/v1/models`` list — a wired-but-dead backend + (dropped from ``/v1/models`` but still in the table) is KNOWN and takes the + retryable-503 path below, NOT this 404 (that is what keeps issue #91 fixed). + A missing/blank ``model`` is *unspecified*, not unknown → it routes to + ``default_model`` and is served. * **2xx / 4xx** → commit to the owner and relay verbatim. A 4xx is a genuine *client* error (the owner is the only backend that could serve this model, so e.g. its 404 "model does not exist" is authoritative — never a reason to @@ -388,6 +423,21 @@ def handle_post( ("X-Lobes-Tier-Reason", decision["reason"]), ] else: + # h23 converse: an UNKNOWN non-empty id (never an alias, never a wired + # backend's served name) must NOT be silently served under the default + # backend's weights — reject it with a 404 model_not_found BEFORE routing, + # matching what a real OpenAI/vLLM backend emits. Unknown-ness is decided + # against the ROUTING TABLE (is_unknown_model), never the readiness-filtered + # /v1/models list — so a wired-but-dead backend (dropped from /v1/models but + # still in the table) is KNOWN and routes on to the retryable 503 below, not + # a 404 (that distinction is what keeps issue #91 fixed). An UNSPECIFIED + # (missing/blank) model is not unknown — it routes to default_model. + if is_unknown_model(table, requested): + return GatewayResponse( + status=404, + headers=[("Content-Type", _CONTENT_TYPE_JSON)], + body=_model_not_found_body(requested), + ) served = resolve_model(table, requested) streaming = is_streaming(body) fwd_body = rewrite_model(body, served) @@ -696,27 +746,6 @@ def probe_audio_ready( return None -def _ready_iff_true(snapshot: Mapping[str, bool | None]) -> dict[str, bool]: - """Collapse the readiness tri-state to a strict "ready iff True" bool map. - - The background :class:`~lobes.gateway._readiness.ReadinessCache` reports a - backend as ``None`` when it is dead / missing / unreachable (a probe that - never got a 200 — see that module's docstring). But - :func:`lobes.roles._gateway_role` reads a ``None`` signal as "no live signal - available → fall back to the coarse ``loaded`` proxy", which for a WIRED - backend is ``True``. Those two ``None``s mean OPPOSITE things: the cache's is - "definitely not reachable", roles.py's is "I don't know, assume configured". - Passing the cache's ``None`` straight through would therefore advertise a - wired-but-dead backend as ``ready=True`` in ``/capabilities`` — the exact - issue #92 defect (honesty h14). This bridges the vocabularies: only an - affirmative ``True`` stays ``True`` (ready); both ``None`` (unreachable) and - ``False`` (unhealthy) become a definite ``False``. It mirrors the ``is True`` - filter :func:`lobes.gateway._routing.list_models_payload` applies, so both - advertisement surfaces agree — a dead backend is advertised nowhere. - """ - return {name: signal is True for name, signal in snapshot.items()} - - def capabilities_payload( table: RoutingTable, cfg: ServerConfig, @@ -817,15 +846,15 @@ def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API cfg = self.server_config origin = reachable_origin(self.headers.get("Host"), cfg.public_url) audio_ready = probe_audio_ready(cfg.audio_url) is True if cfg.audio_url else None - # Collapse the cache's tri-state to a strict "ready iff True" map before - # handing it to the registry: at the gateway a None from the cache means - # "probed, unreachable", NOT roles.py's "no signal → fall back to - # loaded". Without this a wired-but-dead backend would advertise - # ready=True here (issue #92 / honesty h14). See _ready_iff_true. + # Pass the cache's tri-state snapshot STRAIGHT THROUGH — no boundary + # coercion here. build_role_registry treats a SUPPLIED backend_ready + # as authoritative and collapses the cache's None (dead/unreachable) + # to ready=False itself (issue #92 / honesty h14): coercing the + # tri-state is the builder's job, not this call site's, so a dead + # backend can never be advertised ready=True no matter who calls the + # builder. (This deletes t6's _ready_iff_true bridge — see roles.py.) backend_ready = ( - _ready_iff_true(self.readiness_cache.current()) - if self.readiness_cache is not None - else None + self.readiness_cache.current() if self.readiness_cache is not None else None ) self._send_json( 200, diff --git a/lobes/roles.py b/lobes/roles.py index cdca695..20e01b7 100644 --- a/lobes/roles.py +++ b/lobes/roles.py @@ -169,11 +169,25 @@ class RoleInfo: # for the four gateway-fronted roles, `audio_ready` for stt/tts (issue # #89). Generalised from the stt/tts-only split (issue #89/#90) to all six # roles (issue #81 t5) — `ready` is no longer a bare alias of `loaded`. - # When a caller supplies no signal (the parameter is `None`, the default), - # `ready` falls back to the coarse `loaded` "configured/wired" proxy — the - # original t4 behaviour, still exercised by every non-HTTP caller (the - # CLI's non-live paths, most of this module's own test suite). - # Structurally CLAMPED either way: a role whose backend is not wired + # + # `backend_ready` is TRI-STATE PER BACKEND but resolves to `ready` under a + # SUPPLIED-vs-OMITTED rule the builder self-enforces (issue #92 / honesty + # h14 — do not let this drift back to caller discipline): + # * mapping OMITTED entirely (`backend_ready is None`, the default) → + # back-compat: `ready == loaded`, the coarse "configured/wired" proxy. + # Still exercised by every non-HTTP caller (the CLI's non-live paths, + # most of this module's own test suite). + # * mapping SUPPLIED → AUTHORITATIVE: `ready = (backend_ready.get(name) + # is True)`. A present `None`, a present `False`, and a MISSING KEY all + # mean NOT ready — "no live signal" is never evidence of health. + # THE TRAP this closes: `ReadinessCache.current()` reports a dead/missing/ + # unreachable backend as `None`. That cache-`None` means UNREACHABLE — the + # OPPOSITE of "no signal, assume the wired/`loaded` default". A caller that + # passes `current()` straight in (exactly what this contract invites) must + # get `ready=False` for that backend, NOT a resurrected #92 `ready=True`. + # Because the SUPPLIED branch is authoritative, it does. + # + # Structurally CLAMPED regardless: a role whose backend is not wired # (`loaded is False`) or whose `endpoint` is empty can never report # `ready=True`, no matter what signal a caller passes in. This mirrors — # and is enforced by the same code path as — the stt/tts clamp on @@ -256,13 +270,23 @@ def _gateway_role( ) -> RoleInfo: """Resolve a gateway-fronted role (cortex/senses/embedder/reranker). - ``ready_signal`` is the caller's live-readiness tri-state for THIS role's - backend (the value :func:`build_role_registry` looked up in its - ``backend_ready`` mapping by :data:`ROLE_BACKEND` name) — ``True``/ - ``False`` reflect an actual probe result; ``None`` means no live signal is - available (no ``backend_ready`` mapping was supplied, or it had no entry - for this backend), in which case ``ready`` falls back to the coarse - ``loaded`` proxy, matching the original t4 behaviour. + ``ready_signal`` carries only TWO meanings here, never the readiness cache's + tri-state — :func:`build_role_registry` has already resolved that away: + + * ``True``/``False`` — an AUTHORITATIVE readiness verdict for this backend. + The builder passes a concrete bool whenever a ``backend_ready`` mapping was + supplied, having already collapsed a present ``None``, a present ``False``, + and a missing key all to ``False`` (issue #92 / honesty h14). ``ready`` + takes this value directly (subject to the clamp below). + * ``None`` — NO live signal at all (``backend_ready`` was omitted entirely), + in which case ``ready`` falls back to the coarse ``loaded`` proxy — the + original t4 behaviour. + + Crucially, ``None`` here is *only ever* "no mapping supplied", never "the + cache said unreachable": those two ``None``s mean opposite things, and + conflating them (reading the cache's unreachable-``None`` as "fall back to + loaded=True") is the #92 defect. The builder resolves the cache's ``None`` to + a concrete ``False`` on the supplied path so this function can never see it. Either way, ``ready`` is CLAMPED to ``False`` whenever the backend is not wired (``loaded is False``) or the resolved ``endpoint`` is empty (see @@ -382,16 +406,26 @@ def build_role_registry( values — ``"primary"``/``"multimodal"``/``"embed"``/``"rerank"``), one tri-state value (``True``/``False``/``None``) per backend — exactly the shape :meth:`lobes.gateway._readiness.ReadinessCache.current` - returns. Mirrors ``audio_ready``'s shape and defaulting: when a role's - backend has an entry, that entry sets ``ready`` (the runtime signal); - ``loaded`` stays the config fact "is this backend wired". When - ``backend_ready`` is ``None`` (the default) — OR a role's backend has - no entry in it — ``ready`` falls back to ``loaded``, the original t4 - behaviour, so every existing non-HTTP caller (the CLI, this module's - own offline test suite) is unchanged. ``roles.py`` itself never probes - anything to produce this signal — it is computed elsewhere (t3's - :class:`~lobes.gateway._readiness.ReadinessCache`, socket-free to read) - and handed in, exactly like ``audio_ready``. + returns, so a caller passes ``current()`` STRAIGHT THROUGH with no + translation and no per-call-site coercion. **When it is supplied it is + AUTHORITATIVE**, and this builder self-enforces the invariant its shape + implies (issue #92 / honesty h14): ``ready = (backend_ready.get(name) + is True)`` — a present ``None``, a present ``False``, and a MISSING KEY + all mean NOT ready. That matters because the readiness cache reports a + dead/missing/unreachable backend as ``None`` — and the cache's ``None`` + means UNREACHABLE, the OPPOSITE of "no signal, assume the wired + default". Reading that ``None`` as "fall back to ``loaded`` (=``True`` + for a wired backend)" is the exact #92 defect a dead backend advertised + as ``ready=True``); because the supplied branch is authoritative, that + cannot recur, and no caller-side ``_ready_iff_true``-style bridge is + needed. Only when ``backend_ready`` is ``None`` (the default — the + mapping OMITTED, not a per-backend ``None``) does ``ready`` fall back to + ``loaded``, the original t4 behaviour, so every existing non-HTTP caller + (the CLI, this module's own offline test suite) is unchanged. ``loaded`` + stays the config fact "is this backend wired" in all cases. ``roles.py`` + itself never probes anything to produce this signal — it is computed + elsewhere (t3's :class:`~lobes.gateway._readiness.ReadinessCache`, + socket-free to read) and handed in, exactly like ``audio_ready``. :returns: an ordered ``dict`` keyed by role name with EXACTLY the six roles. Every role is always present — an unconfigured/opt-in role (stt/tts with ``audio_url`` unset, or an unwired embed/rerank/multimodal backend) is @@ -399,8 +433,9 @@ def build_role_registry( Readiness (``RoleInfo.ready``) is no longer a bare alias of ``loaded`` (issue #81 t5 — generalising the stt/tts split from issue #89/#90 to all - six roles): it reflects ``backend_ready``/``audio_ready`` when the caller - supplies a live signal, else it falls back to the coarse "configured/wired" + six roles). When a caller supplies ``backend_ready``/``audio_ready`` it is + AUTHORITATIVE (a present ``None``/``False`` or a missing key ⇒ not ready); + only an OMITTED signal falls back to the coarse "configured/wired" ``loaded`` proxy. Either way it is CLAMPED, here, to ``False`` whenever a role's backend is not wired OR its resolved ``endpoint`` is empty — a caller can never fabricate ``ready=True`` for a role with nothing to dial, @@ -414,7 +449,26 @@ def build_role_registry( registry: dict[str, RoleInfo] = {} for role in ("cortex", "senses", "embedder", "reranker"): - signal = backend_ready.get(ROLE_BACKEND[role]) if backend_ready is not None else None + if backend_ready is None: + # NOT SUPPLIED → back-compat: no live signal at all, so fall back to + # the coarse `loaded` proxy (the original t4 behaviour). `None` here + # is `_gateway_role`'s "fall back to loaded" sentinel — never confused + # with the AUTHORITATIVE branch below, which never passes it a `None`. + signal = None + else: + # SUPPLIED → AUTHORITATIVE, and resolved to a concrete bool HERE so a + # present `None`, a present `False`, and a MISSING KEY all collapse to + # "not ready" (issue #92 / honesty h14). This is the invariant this + # builder now SELF-ENFORCES rather than leaving to caller discipline: + # a supplied mapping is the single source of truth, and "no live + # signal" is never evidence of health. In particular + # `ReadinessCache.current()` reports a dead/unreachable backend as + # `None`; reading that `None` as "no signal → fall back to loaded" + # (which for a wired backend is `True`) is the exact #92 defect — the + # cache's `None` means UNREACHABLE, the opposite of "unknown, assume + # configured". By passing `_gateway_role` a concrete `True`/`False` + # (never `None`) on the supplied path, that trap cannot recur. + signal = backend_ready.get(ROLE_BACKEND[role]) is True registry[role] = _gateway_role(role, table, gateway, resolved_env, signal) audio_url = (server.audio_url or "").rstrip("/") diff --git a/tests/test_gateway_capabilities.py b/tests/test_gateway_capabilities.py index 180c92c..5069243 100644 --- a/tests/test_gateway_capabilities.py +++ b/tests/test_gateway_capabilities.py @@ -339,9 +339,11 @@ def test_capabilities_public_url_wins_over_host_end_to_end() -> None: def test_capabilities_payload_threads_backend_ready() -> None: # The PURE builder threads a True/False live signal (keyed by internal Backend # name) straight into each gateway-fronted role's `ready` — `loaded` stays the - # config fact (roles.py t5 contract). The None→not-ready collapse is a - # gateway-BOUNDARY concern (a cache None means "unreachable", not roles.py's - # "no signal") — exercised end to end by the loopback route test below. + # config fact (roles.py t5 contract). A SUPPLIED backend_ready is authoritative, + # and the None→not-ready collapse is now the BUILDER's own job (issue #92 / + # honesty h14), not a caller-side bridge — proved directly below by + # test_capabilities_payload_backend_ready_none_value_is_not_ready and end to + # end by the loopback route test. env = _full_env() table, cfg = build_config(env) backend_ready = {"primary": True, "multimodal": False, "embed": False, "rerank": True} @@ -354,6 +356,25 @@ def test_capabilities_payload_threads_backend_ready() -> None: assert payload["reranker"]["ready"] is True +def test_capabilities_payload_backend_ready_none_value_is_not_ready() -> None: + # #92 / honesty h14, self-enforced by the builder (not a call-site bridge): a + # per-backend None in a SUPPLIED mapping — exactly what ReadinessCache.current() + # reports for a dead/unreachable backend — collapses to ready=False. `loaded` + # stays the config fact. This is the property the deleted _ready_iff_true helper + # used to guarantee at the boundary; it now lives inside build_role_registry, so + # ANY caller passing current() straight through is safe, not just the HTTP route. + env = _full_env() + table, cfg = build_config(env) + backend_ready = {"primary": None, "multimodal": True, "embed": False, "rerank": None} + payload = S.capabilities_payload( + table, cfg, env=env, gateway_url=_GATEWAY_URL, backend_ready=backend_ready + ) + assert payload["cortex"]["loaded"] is True and payload["cortex"]["ready"] is False # None + assert payload["senses"]["ready"] is True # present True + assert payload["embedder"]["ready"] is False # present False + assert payload["reranker"]["loaded"] is True and payload["reranker"]["ready"] is False # None + + def test_capabilities_payload_backend_ready_none_falls_back_to_loaded() -> None: # Default (backend_ready=None): ready falls back to loaded — the CLI/offline # path is unchanged, so the pure builder never needs a live signal to work. @@ -364,15 +385,6 @@ def test_capabilities_payload_backend_ready_none_falls_back_to_loaded() -> None: assert payload[role]["ready"] == payload[role]["loaded"] -def test_ready_iff_true_collapses_none_and_false_but_keeps_true() -> None: - # The gateway-boundary bridge: a cache None (unreachable/dead) must NOT be read - # as roles.py's "no signal → fall back to loaded", so it is collapsed to a - # definite False; False stays False; only True stays True. This is what keeps a - # wired-but-dead backend (which probes None) off the ready list (honesty h14). - collapsed = S._ready_iff_true({"primary": True, "multimodal": None, "embed": False}) - assert collapsed == {"primary": True, "multimodal": False, "embed": False} - - @pytest.fixture def readiness_capabilities_gateway(monkeypatch): """A loopback /capabilities gateway wired with a ReadinessCache whose verdicts @@ -408,9 +420,12 @@ def probe(base_url): def test_capabilities_route_dead_backend_ready_false_h14(readiness_capabilities_gateway) -> None: - # h14 end to end: a wired-but-dead backend probes None; the HTTP /capabilities - # route collapses that to ready=False (loaded stays True) — the dead backend is - # NOT advertised ready, even though roles.py would otherwise fall back to loaded. + # h14 end to end: a wired-but-dead backend probes None; the route passes the + # cache snapshot STRAIGHT through and build_role_registry (authoritative on a + # supplied mapping) collapses that None to ready=False — loaded stays True, the + # dead backend is NOT advertised ready. This is the proof the invariant lives in + # the builder now (t6b removed the route's _ready_iff_true bridge), so it holds + # for every caller, not just this one call site. gw = readiness_capabilities_gateway # All True initially → every gateway-fronted role ready. with urllib.request.urlopen(gw.base + "/capabilities", timeout=5) as r: diff --git a/tests/test_gateway_routing.py b/tests/test_gateway_routing.py index be5d178..a930bc7 100644 --- a/tests/test_gateway_routing.py +++ b/tests/test_gateway_routing.py @@ -7,6 +7,7 @@ from lobes.gateway._routing import ( Backend, RoutingTable, + is_unknown_model, list_models_payload, order_backends, resolve_model, @@ -35,10 +36,57 @@ def test_resolve_model_exact_alias_default() -> None: assert resolve_model(t, "fast") == "F" # alias assert resolve_model(t, "big") == "P" # alias assert resolve_model(t, None) == "P" # missing → default - assert resolve_model(t, "who-knows") == "P" # unknown → default + # UNSPECIFIED-vs-UNKNOWN is a caller policy (is_unknown_model / handle_post), + # NOT resolve_model's: resolve_model still maps an unknown non-empty id to + # default_model so order_backends always has an owner. handle_post 404s the + # unknown id BEFORE it ever calls resolve_model (honesty h23), so this + # fall-back is now a pure-routing safety net, not the serving path. + assert resolve_model(t, "who-knows") == "P" # unknown → default (routing net) assert resolve_model(t, "") == "P" # empty → default +# --- is_unknown_model: UNKNOWN vs UNSPECIFIED (honesty h23) ---------------- + + +def test_is_unknown_model_distinguishes_unspecified_from_unknown() -> None: + # UNSPECIFIED (missing/empty) is NOT unknown — it deliberately routes to + # default_model and must be served, so is_unknown_model is False for it. + t = _table() + assert is_unknown_model(t, None) is False # missing → unspecified, not unknown + assert is_unknown_model(t, "") is False # empty → unspecified, not unknown + # A known alias or any wired backend's served name is KNOWN → not unknown. + assert is_unknown_model(t, "P") is False # served name + assert is_unknown_model(t, "F") is False # served name + assert is_unknown_model(t, "fast") is False # alias → F + assert is_unknown_model(t, "big") is False # alias → P + # A non-empty id that is neither an alias nor any wired served name is UNKNOWN. + assert is_unknown_model(t, "who-knows") is True + assert is_unknown_model(t, "never-advertised") is True + + +def test_is_unknown_model_decided_against_routing_table_not_readiness() -> None: + # CRITICAL (issue #91): unknown-ness is decided against the ROUTING TABLE + # (wired backends + aliases), NEVER the readiness-filtered /v1/models list. A + # backend that is WIRED is KNOWN even if it is dead/warming and thus absent + # from /v1/models — its served name is still in table.backends. So "F" is + # known regardless of any readiness verdict; a request for a dead-but-wired F + # must route (→ 503 owner-down), never 404 as if it were never advertised. + t = _table() # F is wired + assert is_unknown_model(t, "F") is False # wired → known, dead or not + + +def test_is_unknown_model_default_model_is_known_even_in_degenerate_table() -> None: + # The deployment's declared default identity is always KNOWN — naming it + # explicitly equals leaving `model` unspecified (both route to default). Even + # a malformed zero-backend table (which nothing serves) must NOT 404 its own + # default_model: that request stays the terminal 502 malformed-table signal, + # not a 404. In a well-formed table this is redundant (default_model is a + # wired served name) — it only guards the pathological case. + degenerate = RoutingTable(backends=(), default_model="P", aliases={}) + assert is_unknown_model(degenerate, "P") is False # default → known + assert is_unknown_model(degenerate, "Q") is True # anything else → unknown + + # --- order_backends ------------------------------------------------------- diff --git a/tests/test_gateway_server.py b/tests/test_gateway_server.py index 8832221..710d6f6 100644 --- a/tests/test_gateway_server.py +++ b/tests/test_gateway_server.py @@ -732,23 +732,43 @@ def test_dead_primary_never_opens_multimodal_connection() -> None: # --- the "converse" honesty question: an unknown (never-advertised) model id --- -def test_unknown_model_id_routes_to_default_owner_documented_choice() -> None: - # DOCUMENTED CHOICE, not an accident (t6 converse honesty condition): today - # resolve_model falls back to default_model for an unknown, non-empty model - # id, so a request naming a model that was NEVER in /v1/models is served by - # the default (primary) backend, its body REWRITTEN to the primary's served - # name. This test LOCKS the current behaviour so it is a deliberate choice. - # The t6 report recommends this arguably should 404 (model_not_found) to match - # OpenAI + the "advertised implies reachable" honesty theme — but resolve_model - # lives in _routing.py (out of this task's scope), so it is not changed here. +def test_unknown_model_id_never_silently_served_returns_404() -> None: + # h23 CONVERSE: an id that was NEVER in /v1/models (neither an alias nor any + # wired backend's served name) must NOT be silently served by the default + # backend under a different model's weights. It 404s (model_not_found) and NO + # backend is dialed — the request never touches primary. This REPLACES t6's + # test_unknown_model_id_routes_to_default_owner_documented_choice, which locked + # in exactly the behaviour h23 forbids (unknown id served under primary's + # weights, body rewritten to "P"). This is CONSISTENT with "advertised implies + # reachable": a model *listed* in /v1/models never 404s; one never listed + # SHOULD 404 (contrast the never-404 race test below). table, cfg = _cfg() opener, calls = _opener({"primary": 200, "fallback": 200}) resp = S.handle_post( table, cfg, "/v1/chat/completions", [], b'{"model":"never-in-v1-models"}', opener ) - assert [c[0] for c in calls] == ["primary"] # routed to the default owner - assert json.loads(calls[0][1])["model"] == "P" # rewritten to primary's served name - assert resp.status == 200 + assert calls == [] # NOT silently served — no backend dialed + assert resp.status == 404 + assert resp.upstream is None + body = json.loads(resp.body) + assert body["error"]["type"] == "model_not_found" + assert "never-in-v1-models" in body["error"]["message"] + + +def test_wired_but_dead_backend_yields_503_not_404() -> None: + # CRITICAL distinction (issue #91): "F" IS a wired backend's served name, so it + # is KNOWN even though a dead F is filtered out of /v1/models. A request naming + # F when F is down must route to F's owner and get the retryable 503 (owner + # down), NOT the 404 an UNKNOWN (never-wired) id gets. Getting this backwards + # would 404 a merely-down backend and reintroduce #91. Unknown-ness is decided + # against the routing table, never the readiness-filtered /v1/models list. + table, cfg = _cfg() + opener, calls = _opener({"primary": 200, "fallback": S.UpstreamError("refused")}) + resp = S.handle_post(table, cfg, "/v1/chat/completions", [], b'{"model":"F"}', opener) + assert [c[0] for c in calls] == ["fallback"] # dialed the owner — F is KNOWN + assert resp.status == 503 # owner down → retryable, NOT 404 + assert dict(resp.headers)["Retry-After"] == str(S.BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS) + assert json.loads(resp.body)["error"]["type"] == "backend_unavailable" # --- issue #92: ReadinessCache wired into /v1/models (advertised => reachable) -- @@ -888,3 +908,48 @@ def test_race_listed_then_owner_killed_yields_503_not_404(ready_gateway) -> None assert exc.value.code == 503 # NEVER 404 assert exc.value.headers.get("Retry-After") == str(S.BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS) assert json.loads(exc.value.read())["error"]["type"] == "backend_unavailable" + + +def test_wired_but_dead_unlisted_backend_yields_503_not_404(ready_gateway) -> None: + # The exact #91 trap, end to end: a backend that is WIRED but dead is filtered + # OUT of /v1/models (readiness), yet its served name is still in the routing + # table. A completion naming it must return 503 (retry), NOT 404 — unknown-ness + # is decided against the routing TABLE, never the readiness-filtered list. + # Getting this backwards (deciding unknown-ness against /v1/models) would 404 a + # merely-dead backend and reintroduce #91. + gw = ready_gateway + gw.verdicts["http://vllm-fallback:8000"] = None # dead → dropped from /v1/models + gw.cache.refresh() + gw.alive["fallback"] = False # and the owner is actually down + with urllib.request.urlopen(gw.base + "/v1/models", timeout=5) as r: + assert "F" not in [m["id"] for m in json.load(r)["data"]] # NOT listed + req = urllib.request.Request( + gw.base + "/v1/chat/completions", + data=b'{"model":"F"}', + headers={"Content-Type": "application/json"}, + ) + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(req, timeout=5) + assert exc.value.code == 503 # wired + dead + unlisted → retry, NOT 404 + assert exc.value.headers.get("Retry-After") == str(S.BACKEND_UNAVAILABLE_RETRY_AFTER_SECONDS) + assert json.loads(exc.value.read())["error"]["type"] == "backend_unavailable" + assert "fallback" in gw.open_calls # the owner WAS dialed — F is known + + +def test_unknown_unlisted_model_yields_404_not_served(ready_gateway) -> None: + # h23 converse at the route: an id that is neither wired nor aliased is never in + # /v1/models AND is never silently served under the default backend's weights — + # it 404s (model_not_found) and no backend is dialed. Contrast the wired-but-dead + # case above (503): the two differ precisely on "is it in the routing table". + gw = ready_gateway + before = len(gw.open_calls) + req = urllib.request.Request( + gw.base + "/v1/chat/completions", + data=b'{"model":"phantom-never-advertised"}', + headers={"Content-Type": "application/json"}, + ) + with pytest.raises(urllib.error.HTTPError) as exc: + urllib.request.urlopen(req, timeout=5) + assert exc.value.code == 404 # never advertised → 404, not served under primary + assert json.loads(exc.value.read())["error"]["type"] == "model_not_found" + assert len(gw.open_calls) == before # no backend dialed — not silently served diff --git a/tests/test_roles.py b/tests/test_roles.py index 9bab5d1..347bb2f 100644 --- a/tests/test_roles.py +++ b/tests/test_roles.py @@ -528,21 +528,49 @@ def test_backend_ready_sets_ready_independent_of_loaded() -> None: assert registry["reranker"].ready is True -def test_backend_ready_missing_entry_falls_back_to_loaded() -> None: - """A backend_ready mapping that simply has no entry for a role's backend - (e.g. a readiness cache that hasn't probed it yet) degrades to the coarse - `loaded` proxy for THAT role only — never raises, never guesses.""" +def test_backend_ready_missing_entry_is_not_ready() -> None: + """INVERTED (issue #92 / honesty h14): a SUPPLIED backend_ready mapping is + AUTHORITATIVE, so a role whose backend has no entry is NOT ready — never a + fall-back to the coarse `loaded` proxy. "No live signal" is not evidence of + health: a missing key is treated exactly like a present None/False. (This + used to assert `ready == loaded` for the missing-entry roles; that let a + caller who omitted a backend advertise it ready, the caller-discipline trap + build_role_registry now self-enforces against.)""" registry = _registry(_full_env(), backend_ready={"primary": True}) - assert registry["cortex"].ready is True # has a live entry + assert registry["cortex"].ready is True # has a live True entry → ready for role in ("senses", "embedder", "reranker"): info = registry[role] - assert info.ready == info.loaded # no entry → falls back, same as backend_ready=None + assert info.loaded is True # still wired... + assert info.ready is False # ...but a missing key ⇒ NOT ready (authoritative) + + +def test_backend_ready_present_none_is_not_ready() -> None: + """INVERTED (issue #92 / honesty h14 — the core trap): a present per-backend + `None` means NOT ready, NOT "fall back to loaded". `ReadinessCache.current()` + reports a dead/unreachable backend as `None`; passing that straight in (as + build_role_registry's own contract invites) must yield `ready=False` for + that backend — never the resurrected #92 `ready=True` a wired-but-dead + backend would get if `None` fell back to the `loaded` proxy. `loaded` stays + the config fact.""" + registry = _registry( + _full_env(), + backend_ready={"primary": None, "multimodal": True, "embed": None, "rerank": True}, + ) + assert registry["cortex"].loaded is True # wired... + assert registry["cortex"].ready is False # ...but the cache said UNREACHABLE + assert registry["embedder"].loaded is True + assert registry["embedder"].ready is False + assert registry["senses"].ready is True # a present True is still ready + assert registry["reranker"].ready is True def test_backend_ready_none_preserves_t4_behaviour() -> None: - """backend_ready omitted entirely (the default) → ready == loaded for every - gateway-fronted role, exactly the pre-t5 behaviour — every existing - non-HTTP caller is unaffected.""" + """backend_ready OMITTED entirely (the whole mapping is None, the default) + → ready == loaded for every gateway-fronted role, exactly the pre-t5 + behaviour — every existing non-HTTP caller is unaffected. Contrast a + per-backend None WITHIN a supplied mapping, which is authoritatively "not + ready" (test_backend_ready_present_none_is_not_ready) — the two Nones mean + opposite things (issue #92 / honesty h14).""" registry = _registry(_full_env()) for role in ("cortex", "senses", "embedder", "reranker"): assert registry[role].ready == registry[role].loaded From 692443b46cb468e4694bebc02132dfabac348b3c Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 11:19:38 +0300 Subject: [PATCH 13/23] t7: CLI becomes a gateway client; doctor detects deployed-artifact skew (#96, #99) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lobes capabilities / lobes endpoint no longer re-derive the six-role contract from .env — they GET the gateway's own /capabilities and render it verbatim, falling back to the offline .env-derived registry (every role's ready forced false, source="offline"/"gateway" marks which) only when no gateway answers. This makes the CLI/gateway honesty condition (h3) true by construction instead of by keeping two independent derivations in sync, which had already drifted in both directions (#92 underreported, #96 overreported stt/tts as ready=true on a 404 path). GET /health now reports the gateway's own lobes-cli version (additive), and `lobes doctor` gained a gateway_version_match check comparing it to the CLI's own version: mismatch is a real error (fails the run, names the exact MODEL_GEAR_VERSION fix), an unreachable/pre-#99 gateway degrades to a non-fatal info result rather than a false pass. This targets issue #99, the structural cause of #92 (a gateway image pinned once at `lobes init` time and never re-pinned). tests/conftest.py's autouse fixture now also neutralises the new gateway probes (matching how it already neutralises /health), since the dev rig has a real unrelated daemon on host port 8000 that could otherwise leak into the "offline" test path. Full suite: 1169 passed, 9 skipped (was 1158 passed, 9 skipped). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- docs/gateway-fleet.md | 11 ++ lobes/cli/_commands/capabilities.py | 208 +++++++++++++++++++++++----- lobes/cli/_commands/doctor.py | 94 ++++++++++++- lobes/explain/catalog.py | 14 +- lobes/gateway/server.py | 16 ++- lobes/runtime/_health.py | 34 +++++ tests/conftest.py | 27 +++- tests/test_cli_capabilities.py | 207 ++++++++++++++++++++++++++- tests/test_doctor.py | 120 +++++++++++++++- tests/test_gateway_server.py | 7 +- 10 files changed, 680 insertions(+), 58 deletions(-) diff --git a/docs/gateway-fleet.md b/docs/gateway-fleet.md index af6bcaa..7328228 100644 --- a/docs/gateway-fleet.md +++ b/docs/gateway-fleet.md @@ -347,6 +347,17 @@ is required (pinning keeps the image reproducible); from-source/dev boxes that r ahead of a PyPI release point `MODEL_GEAR_VERSION` at a published TestPyPI `.devN` build. +**That pin is written once and nothing re-pins it (issue #99).** `lobes init` +sets `MODEL_GEAR_VERSION` at scaffold time; no other verb touches it +afterwards, so a long-lived deployment can silently keep running an old +gateway image long after the host's own `lobes` binary — and PyPI — have +moved on. `GET /health` reports the deployed gateway's own `lobes-cli` +`version` (additive field, issue #99) precisely so this is detectable without +docker: `lobes doctor`'s `gateway_version_match` check compares it against the +CLI's own version and fails the run on a real mismatch (remediation: bump +`MODEL_GEAR_VERSION` in `.env` and `docker compose up -d --build gateway`) — +see [Verbs](#verbs) below. + ### Auth (known limitation) The gateway is a **pass-through** and is **not auth-aware** — it does not inspect diff --git a/lobes/cli/_commands/capabilities.py b/lobes/cli/_commands/capabilities.py index 3e06526..0ea71aa 100644 --- a/lobes/cli/_commands/capabilities.py +++ b/lobes/cli/_commands/capabilities.py @@ -1,40 +1,90 @@ -"""``lobes capabilities`` / ``lobes endpoint`` — the role registry (issue #81, t5). +"""``lobes capabilities`` / ``lobes endpoint`` — the live role→endpoint contract +(issue #81; the gateway-client rewrite is issue #96, plan "advertised implies +reachable" task t7). -Read-only, CLI-side view of the SIX first-class Colleague-facing roles -(``cortex`` / ``senses`` / ``embedder`` / ``reranker`` / ``stt`` / ``tts``), -built by the ONE canonical registry builder in :mod:`lobes.roles` — -:func:`lobes.roles.role_registry_from_env` — the same builder the gateway's -``GET /capabilities`` (t6) will call, so the role→endpoint contract has exactly -one source of truth. +Both verbs used to REBUILD the six-role registry from the deployment's +``.env`` and call that the truth — this module's docstring used to claim the +CLI and the gateway share "exactly one source of truth" because both called +the same pure builder, :func:`lobes.roles.role_registry_from_env` / +:func:`lobes.roles.build_role_registry`. That claim was false in practice: the +CLI reads what an operator's ``.env`` *says*, while the gateway answers with +what its *own container process* was actually started with. Those are two +independent derivations of the same contract, sourced from two different +places, and they have now drifted in **both** directions: + +* issue #92 — the gateway under-reported: a live backend its background + readiness probe hadn't caught up with yet looked dead over HTTP, while the + CLI, going by config, was (in that instance) right. +* issue #96 — the CLI over-reported: ``AUDIO_URL`` was present in the + deployment's ``.env`` but was never wired into the gateway *container's* + environment (a ``docker compose`` env-passthrough gap), so the CLI's + offline registry advertised ``stt``/``tts`` as ``ready=true`` on a path + that actually 404s/503s — while the gateway, which only knows its own + actual environment, correctly said ``ready=false``. + +Calling a single shared *function* the "one source of truth" was never +enough, because the CLI and the gateway don't share an address space — each +evaluates that function against its own, independently-sourced ``env`` +mapping. Making the two derivations agree by testing them against each other +after the fact is exactly the failure mode that produced both bugs: a config +file on the host disk is not evidence of what an already-started container +process actually has wired, and no amount of keeping :mod:`lobes.roles` in +sync can make it so. + +So this module is no longer a second, independent derivation. It is a CLIENT +of the gateway: + +* ``lobes capabilities`` / ``lobes endpoint `` first try ``GET + http://localhost:/capabilities`` against the gateway that + would actually serve a request. On a clean 200 whose body is a JSON object + carrying all six roles, THAT payload is rendered verbatim — the gateway is + the one process that knows what it actually has wired and what its own + readiness probes actually last observed, so its answer is the only one + worth reporting. +* On any failure to get that authoritative answer — no gateway listening, + connection refused, a timeout, a non-200, or a malformed/incomplete body — + this degrades to the offline view built from the deployment's ``.env`` + (:func:`lobes.roles.role_registry_from_env`), same as before t7. But the + fallback is now honest about what it is: every role's ``ready`` is forced + to ``False`` (a config file was never probed, so it can never be evidence + of health — this generalises issue #96's fix past stt/tts to all six + roles), and both the JSON and the table carry an explicit ``source`` + marker (``"gateway"`` when live, ``"offline"`` when degraded) so a caller — + human or agent — can never mistake a configured-defaults guess for a live + observation. + +This makes the honesty condition (h3: the CLI's and the gateway's answers +agree) true **by construction** whenever a gateway is reachable — there is +then exactly one source of truth, and the CLI is only ever rendering it, so +there is nothing left to drift. When no gateway is reachable there is nothing +to agree *with*, so the fallback trades completeness for honesty (an +"unverified, treat as down" default) instead of silently guessing. Deployment resolution ---------------------- -The CLI process never has the deployment's env vars in its own ``os.environ`` -(those are injected into the *containers* by ``docker compose``, not the host -shell), so the deployment dir's ``.env`` is read straight off disk via -:func:`lobes.runtime._env.read_env_file` and handed to -:func:`~lobes.roles.role_registry_from_env` as the ``env`` mapping — this is -what drives BOTH the routing table (which roles are wired) and the served- -context overlay (``PRIMARY_MAX_MODEL_LEN`` and friends, t5). Resolution is -**soft**: an unscaffolded deployment (no ``lobes init`` yet) degrades to ``{}`` -rather than erroring, mirroring ``lobes overview --live`` / ``assess`` / -``benchmark`` — a read-only introspection verb should always answer, showing -catalog defaults with every role ``loaded=False`` except the always-present -``cortex``. - -The reachable gateway URL is derived exactly like every other read-only probe +The gateway URL to dial is derived exactly like every other read-only probe in this CLI (``status``, ``overview --live``, ``fleet status``): ``http://localhost:``, where the port is ``--port`` if given, else ``VLLM_PORT`` from the deployment's ``.env``, else 8000 -(:func:`lobes.cli._runtime_ops.resolve_port_soft`). +(:func:`lobes.cli._runtime_ops.resolve_port_soft`). Resolving that port never +requires a scaffolded deployment (an unscaffolded one degrades to 8000), so +both verbs still answer — falling straight to the offline path — even before +``lobes init`` has ever run, mirroring ``lobes overview --live`` / ``assess`` +/ ``benchmark``. -Both verbs are strictly read-only: no compose/docker call, no ``--apply``. +Both verbs stay strictly read-only: the only I/O is one bounded HTTP GET to +the local gateway (stdlib ``urllib``, a short timeout so an unreachable +gateway degrades fast rather than stalling an agent's loop) plus, on the +fallback path only, a read of the deployment's ``.env`` off disk. No +compose/docker call, no ``--apply``. """ from __future__ import annotations import argparse import dataclasses +import json +import urllib.request from lobes.cli import _runtime_ops from lobes.cli._errors import EXIT_USER_ERROR, ModelGearError @@ -45,8 +95,60 @@ _COMPOSE_DIR_HELP = "Deployment dir (default: $LOBES_DIR or ~/.lobes)." _PORT_HELP = "Gateway host port (default: VLLM_PORT in .env)." +# Bounded so an unreachable/foreign process on the resolved port degrades to +# the offline fallback quickly — this is a read-only introspection verb, and +# an agent looping on it must never be made to wait on a dead socket. +_GATEWAY_TIMEOUT_SECONDS = 2.0 + +# The full RoleInfo field set — used to sanity-check a gateway response before +# trusting it. A 200 from *something* listening on the resolved port whose +# body happens to be a dict keyed by all six role names but missing fields a +# real /capabilities response always carries is treated as malformed, not +# authoritative (see _fetch_gateway_capabilities). +_ROLE_INFO_FIELDS = {f.name for f in dataclasses.fields(RoleInfo)} + + +def _fetch_gateway_capabilities( + port: int, timeout: float = _GATEWAY_TIMEOUT_SECONDS +) -> dict[str, dict] | None: + """``GET /capabilities`` from the live gateway on ``port``. + + Returns the parsed payload — a dict keyed by all six :data:`ROLES`, each + value the role's JSON metadata — on a clean 200 with a well-shaped body. + Returns ``None`` on ANY failure to get an authoritative answer: + connection refused (nothing listening), a DNS/socket error, a timeout, a + non-2xx status, an undecodable body, or a 200 whose body doesn't have the + expected shape (missing a role, or a role missing an expected field — a + stale/foreign process happens to be answering on this port; see the + ``lobes.roles._gateway_base_url`` docstring for why a foreign daemon on a + guessed port is a real hazard on this rig, not a hypothetical one). + + ``None`` is not an error to the caller: it means "fall back to the + offline registry", exactly like every other read-only probe in this CLI + degrades when the thing it would rather ask isn't there. + """ + url = f"http://localhost:{port}/capabilities" + try: + with urllib.request.urlopen(url, timeout=timeout) as resp: # local endpoint only + if not (200 <= resp.status < 300): + return None + raw = resp.read() + except OSError: # URLError (incl. HTTPError) subclasses OSError + return None + try: + data = json.loads(raw) + except (ValueError, TypeError): + return None + if not isinstance(data, dict): + return None + for role in ROLES: + entry = data.get(role) + if not isinstance(entry, dict) or not _ROLE_INFO_FIELDS <= set(entry): + return None + return data + -def _registry(args: argparse.Namespace) -> dict[str, RoleInfo]: +def _offline_registry(args: argparse.Namespace) -> dict[str, RoleInfo]: env = _runtime_ops.deployment_env_soft(args) port, _ = _runtime_ops.resolve_port_soft(args) gateway_url = f"http://localhost:{port}" @@ -64,27 +166,63 @@ def _role_payload(info: RoleInfo) -> dict[str, object]: return dataclasses.asdict(info) -def _render_table(registry: dict[str, RoleInfo]) -> str: +def _capabilities_view(args: argparse.Namespace) -> tuple[dict[str, dict], str]: + """Resolve the six-role payload, preferring the live gateway. + + Returns ``(payload, source)``: ``payload`` is a dict keyed by all six + :data:`ROLES` (each value the role's JSON metadata), and ``source`` is + ``"gateway"`` when it came straight from a live ``GET /capabilities``, or + ``"offline"`` when it is the ``.env``-derived fallback. In the offline + case every role's ``ready`` is forced to ``False`` — see the module + docstring and issue #96: a config file was never probed, so it can never + honestly claim a role is reachable, no matter what the offline registry's + own ``loaded``/``ready`` computation would otherwise say. + """ + port, _ = _runtime_ops.resolve_port_soft(args) + live = _fetch_gateway_capabilities(port) + if live is not None: + return live, "gateway" + registry = _offline_registry(args) + offline = {role: _role_payload(registry[role]) for role in ROLES} + for role in ROLES: + offline[role]["ready"] = False + return offline, "offline" + + +def _render_table(registry: dict[str, dict], source: str) -> str: header = f"{'role':<9} {'model':<48} {'context':>8} loaded endpoint" - lines = [header, "-" * len(header)] + lines: list[str] = [] + if source == "offline": + lines.append( + "# source: offline — gateway unreachable; showing .env-configured " + "defaults, NOT a live probe (every role's ready=false)" + ) + else: + lines.append("# source: gateway — live GET /capabilities") + lines.append(header) + lines.append("-" * len(header)) for role in ROLES: info = registry[role] - model = info.model if len(info.model) <= 48 else info.model[:45] + "..." + model = info["model"] if len(info["model"]) <= 48 else info["model"][:45] + "..." lines.append( - f"{info.role:<9} {model:<48} {info.context:>8} " - f"{'yes' if info.loaded else 'no ':<6} {info.endpoint or '(none)'}" + f"{info['role']:<9} {model:<48} {info['context']:>8} " + f"{'yes' if info['loaded'] else 'no ':<6} {info['endpoint'] or '(none)'}" ) - lines.append(f" responsibilities: {', '.join(info.responsibilities)}") + lines.append(f" responsibilities: {', '.join(info['responsibilities'])}") return "\n".join(lines) def cmd_capabilities(args: argparse.Namespace) -> int: json_mode = bool(getattr(args, "json", False)) - registry = _registry(args) + payload, source = _capabilities_view(args) if json_mode: - emit_result({role: _role_payload(registry[role]) for role in ROLES}, json_mode=True) + # "source" is an added top-level sibling of the six role keys — it never + # collides with a role name, so every existing consumer that reads + # payload[] is unaffected; only a strict `set(payload) == ROLES` + # check needs to account for it. + emit_result({**payload, "source": source}, json_mode=True) else: - emit_result(_render_table(registry), json_mode=False) + emit_result(_render_table(payload, source), json_mode=False) return 0 @@ -97,8 +235,8 @@ def cmd_endpoint(args: argparse.Namespace) -> int: message=f"unknown role {role!r}", remediation=f"valid roles: {', '.join(ROLES)}", ) - registry = _registry(args) - endpoint = registry[role].endpoint + payload, _source = _capabilities_view(args) + endpoint = payload[role]["endpoint"] if json_mode: emit_result({"role": role, "endpoint": endpoint}, json_mode=True) else: diff --git a/lobes/cli/_commands/doctor.py b/lobes/cli/_commands/doctor.py index 760d690..301a570 100644 --- a/lobes/cli/_commands/doctor.py +++ b/lobes/cli/_commands/doctor.py @@ -1,9 +1,11 @@ """``lobes doctor`` — diagnose the local model deployment. Real checks (no longer a stub): is docker available, is a deployment scaffolded, -is the ``.env`` coherent with ``culture.yaml``, and is ``/health`` reachable. A -down model is *not* an error (bringing it up is the tool's job) — only missing -docker or an un-scaffolded deployment fail the run. +is the ``.env`` coherent with ``culture.yaml``, is ``/health`` reachable, and +does the deployed gateway's own ``lobes-cli`` release match this CLI's (issue +#99). A down model is *not* an error (bringing it up is the tool's job) — only +missing docker, an un-scaffolded deployment, or a deployed-artifact version +mismatch fail the run. JSON contract: ``{healthy, checks:[{id, passed, severity, message, remediation}]}``. """ @@ -13,6 +15,7 @@ import argparse from pathlib import Path +from lobes import __version__ from lobes.cli._commands.whoami import _find_culture_yaml from lobes.cli._errors import ModelGearError from lobes.cli._output import emit_result @@ -92,6 +95,90 @@ def _health_check(port: int) -> dict: ) +def _version_skew_remediation(deploy_dir: Path | None) -> str: + """The exact fix for a version mismatch — names both the file and the pin + to change, plus the follow-up rebuild, so this is copy-pasteable.""" + env_path = f"{deploy_dir}/.env" if deploy_dir is not None else "/.env" + return ( + f"set MODEL_GEAR_VERSION={__version__} in {env_path}, " + "then docker compose up -d --build gateway" + ) + + +def _version_skew_check(port: int, deploy_dir: Path | None) -> dict: + """Detect deployed-artifact version skew between the gateway and this CLI. + + This is the structural fix for issue #99, the root cause behind issue #92: + ``Dockerfile.gateway`` runs ``pip install "lobes-cli==${MODEL_GEAR_VERSION}"`` + with ``MODEL_GEAR_VERSION`` written ONCE, by ``lobes init``, at scaffold + time — no verb ever re-pins it afterwards. A gateway container can + therefore silently keep running a stale ``lobes-cli`` release for as long + as the deployment stays up, even after the host's own ``lobes`` binary + (and PyPI) have moved on. On the reference rig this went undetected for + five days: the gateway ran ``0.36.0`` and the realtime bridge ``0.34.1`` + against a host CLI at ``0.39.0``, and issue #92 was filed and + investigated as a fresh code regression when the fix behind it was + already published and simply undeployed. + + This check is docker-free: the gateway now reports its own deployed + ``lobes-cli`` version over ``GET /health`` (issue #99, additive — + :mod:`lobes.gateway.server`), so this only needs + :func:`lobes.runtime._health.fetch_health` (a bounded HTTP GET) to compare + that against this process's own :data:`lobes.__version__`. + + The three outcomes are NOT symmetric, deliberately: + + * **match** — ``passed=True``. Nothing to report. + * **mismatch** — ``passed=False``, ``severity="error"`` (this DOES fail + the overall run): a real, actionable defect — the deployed gateway is + running code the operator's own CLI no longer believes is current, and + that gap is exactly what let issue #92 masquerade as a live bug. + * **gateway unreachable** — ``passed=False``, ``severity="info"`` (this + does NOT fail the run): a down gateway is ordinary here (per this + module's own docstring, "bringing it up is the tool's job", same as + ``health_reachable``), so it must not be conflated with a real skew + defect. Critically this is ALSO not a silent pass: reporting + ``passed=True`` ("versions match") when nothing was actually verified + would be exactly the #96/#92 mistake this whole plan exists to close — + an unverified claim standing in for a live observation. The message + says plainly that verification did not happen. + """ + payload = _health.fetch_health(port) + if payload is None: + return _check( + "gateway_version_match", + False, + "info", + f"gateway not reachable on :{port} — cannot verify deployed version", + "start the server ('lobes serve --apply' or 'lobes fleet up --apply'), " + "then re-run doctor", + ) + gateway_version = payload.get("version") + if not gateway_version: + return _check( + "gateway_version_match", + False, + "info", + f"gateway on :{port} did not report a version — cannot verify (pre-#99 gateway build)", + "rebuild the deployed gateway image to pick up /health's version field", + ) + if gateway_version != __version__: + return _check( + "gateway_version_match", + False, + "error", + f"deployed gateway reports lobes-cli {gateway_version}, this CLI is " + f"{__version__} — deployed-artifact version skew (issue #99)", + _version_skew_remediation(deploy_dir), + ) + return _check( + "gateway_version_match", + True, + "error", + f"gateway and CLI both report lobes-cli {__version__}", + ) + + def _diagnose(compose_dir: str | None = None) -> dict[str, object]: checks: list[dict] = [_docker_check()] @@ -111,6 +198,7 @@ def _diagnose(compose_dir: str | None = None) -> dict[str, object]: port = _env.parse_port(_env.read_env(env_path, "VLLM_PORT", "8000")) checks.append(_health_check(port)) + checks.append(_version_skew_check(port, deploy_dir)) # Only error-severity failures make the run unhealthy. healthy_overall = all(c["passed"] for c in checks if c["severity"] == "error") diff --git a/lobes/explain/catalog.py b/lobes/explain/catalog.py index dd359b5..2439ba9 100644 --- a/lobes/explain/catalog.py +++ b/lobes/explain/catalog.py @@ -443,10 +443,16 @@ Diagnoses the deployment with real checks: `docker_available` (docker + compose resolve), `compose_present` (a deployment is scaffolded), `env_coherence` -(`.env` has `VLLM_SERVED_NAME` and it matches `culture.yaml`), and -`health_reachable` (`/health` responds). A down model is a *warning*, not a -failure — only missing docker or an un-scaffolded deployment make the run exit -non-zero. JSON contract: `{"healthy", "checks"}`. Supports `--json`. +(`.env` has `VLLM_SERVED_NAME` and it matches `culture.yaml`), +`health_reachable` (`/health` responds), and `gateway_version_match` (the +deployed gateway's own `lobes-cli` release, read from its `GET /health` +`version` field, matches this CLI's — issue #99, catching a stale +`MODEL_GEAR_VERSION` pin baked into `Dockerfile.gateway` at scaffold time and +never re-pinned). A down model is a *warning*, not a failure; an unreachable +gateway degrades the version check to a non-fatal informational result (not a +false pass) — only missing docker, an un-scaffolded deployment, or an actual +version *mismatch* make the run exit non-zero. JSON contract: `{"healthy", +"checks"}`. Supports `--json`. """ _EMBEDDINGS = """\ diff --git a/lobes/gateway/server.py b/lobes/gateway/server.py index 8fdaefb..f1405de 100644 --- a/lobes/gateway/server.py +++ b/lobes/gateway/server.py @@ -46,7 +46,7 @@ from typing import Callable, Iterable from urllib.parse import urlsplit -from lobes import _metrics +from lobes import __version__, _metrics from lobes.catalog import as_dicts as supported_models_catalog from lobes.gateway._config import ServerConfig from lobes.gateway._pressure_policy import BUSY_RETRY_AFTER_SECONDS, decide @@ -817,7 +817,19 @@ class _Handler(BaseHTTPRequestHandler): def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API route = self.path.split("?", 1)[0] if route == "/health": - self._send_json(200, {"status": "ok", "service": "model-gear-gateway"}) + # `version` is the deployed lobes-cli release THIS gateway process was + # built from (`__version__`, read off installed package metadata inside + # the container) — additive, issue #99. It is what lets a remote client + # (or `lobes doctor`, via lobes.runtime._health.fetch_health) detect + # deployed-artifact skew docker-free: Dockerfile.gateway pins + # `pip install "lobes-cli==${MODEL_GEAR_VERSION}"` once, at `lobes init` + # time, and nothing re-pins it afterwards, so a gateway container can + # silently run a stale release for days after the host CLI (and PyPI) + # moved on — exactly what made issue #92 look like a code regression + # when the fix was already published and simply undeployed. + self._send_json( + 200, {"status": "ok", "service": "model-gear-gateway", "version": __version__} + ) elif route == "/status": # Live aggregate the host CLI can't get otherwise: the backends are # internal-only, so the gateway fans out to each one's /health + /metrics. diff --git a/lobes/runtime/_health.py b/lobes/runtime/_health.py index 97f5be1..9f6099f 100644 --- a/lobes/runtime/_health.py +++ b/lobes/runtime/_health.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json import time import urllib.request @@ -24,6 +25,39 @@ def is_healthy(port: int, timeout: float = 3.0) -> bool: return False +def fetch_health(port: int, timeout: float = 3.0) -> dict | None: + """Non-blocking: the parsed JSON body of ``/health``, or ``None`` on any failure. + + :func:`is_healthy` collapses ``/health`` to a bare up/down boolean; this + returns the payload itself so a caller can inspect what the gateway + *reports* beyond bare liveness — e.g. its own deployed ``lobes-cli`` + ``version`` (issue #99), which ``lobes doctor`` compares against the CLI's + own :data:`lobes.__version__` to catch deployed-artifact skew (the + structural cause of issue #92: a gateway container built from a stale pin + silently outliving a PyPI release that already fixed the bug it was + misdiagnosed as). + + ``None`` covers every failure mode uniformly — connection refused, DNS + failure, timeout, a non-2xx status, and a malformed/non-JSON or non-object + body — because a caller here only ever needs one distinction: "the gateway + told us something" vs "we could not learn anything", never *why* it + couldn't be reached. + """ + url = f"http://localhost:{port}/health" + try: + with urllib.request.urlopen(url, timeout=timeout) as r: # local endpoint only + if not (200 <= r.status < 300): + return None + raw = r.read() + except OSError: # URLError subclasses OSError + return None + try: + data = json.loads(raw) + except (ValueError, TypeError): + return None + return data if isinstance(data, dict) else None + + def wait_health( port: int, *, diff --git a/tests/conftest.py b/tests/conftest.py index 6b40c13..b818988 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,16 +1,24 @@ """Shared fixtures: make external probes deterministic and offline. -Several verbs shell out to docker / nvidia-smi and probe ``/health``. The -autouse fixture below neutralises those so the suite never depends on a running -container, a GPU, or the host's ``~/.lobes`` — every probe degrades to its -"nothing there" answer. Tests that need a deployment scaffold one into a tmp dir -and pass ``--compose-dir``. +Several verbs shell out to docker / nvidia-smi and probe ``/health`` / +``/capabilities``. The autouse fixture below neutralises those so the suite +never depends on a running container, a GPU, the host's ``~/.lobes``, or +whatever happens to already be listening on a guessed port on the machine +running the tests — every probe degrades to its "nothing there" answer. This +is not hypothetical: the reference dev rig has an unrelated daemon bound to +host port 8000 (see ``lobes.roles._gateway_base_url``'s docstring), so a test +that skipped this neutralisation could observe a real, but *wrong*, answer. +Tests that need a deployment scaffold one into a tmp dir and pass +``--compose-dir``; tests that need a genuinely live gateway spin up their own +loopback server on an ephemeral port and explicitly restore the real probe +function (see ``tests/test_cli_capabilities.py``'s fake-gateway test). """ from __future__ import annotations import pytest +from lobes.cli._commands import capabilities as _capabilities from lobes.runtime import _compose, _health @@ -20,6 +28,15 @@ def offline_runtime(monkeypatch, tmp_path): monkeypatch.setattr(_compose, "_probe", lambda *a, **k: None) # /health never responds. monkeypatch.setattr(_health, "is_healthy", lambda *a, **k: False) + # /health's parsed-JSON sibling (issue #99 — `lobes doctor`'s version-skew + # check) also never responds by default, for the same determinism reason. + monkeypatch.setattr(_health, "fetch_health", lambda *a, **k: None) + # `lobes capabilities` / `lobes endpoint` (issue #96, t7) try a live GET + # /capabilities against the resolved port before falling back to the + # offline .env-derived registry — neutralise that probe too, for the same + # reason /health is neutralised above, so the whole suite is deterministic + # regardless of what is (or isn't) actually listening on the guessed port. + monkeypatch.setattr(_capabilities, "_fetch_gateway_capabilities", lambda *a, **k: None) # No deployment scaffolded by default: point the home at an empty tmp dir. monkeypatch.delenv("LOBES_DIR", raising=False) monkeypatch.delenv("MODEL_GEAR_DIR", raising=False) # also clear legacy back-compat var diff --git a/tests/test_cli_capabilities.py b/tests/test_cli_capabilities.py index e4d1e25..25f6517 100644 --- a/tests/test_cli_capabilities.py +++ b/tests/test_cli_capabilities.py @@ -1,20 +1,33 @@ -"""Tests for ``lobes capabilities`` / ``lobes endpoint`` (issue #81, task t5). +"""Tests for ``lobes capabilities`` / ``lobes endpoint`` (issue #81, tasks t5/t7). These verbs are the CLI-side view of the six first-class Colleague-facing -roles (``cortex``/``senses``/``embedder``/``reranker``/``stt``/``tts``), built -by the ONE canonical registry builder in :mod:`lobes.roles`. Both are strictly -read-only — no compose/docker call, no ``--apply``. +roles (``cortex``/``senses``/``embedder``/``reranker``/``stt``/``tts``). Since +issue #96 (plan "advertised implies reachable", task t7) they are CLIENTS of +the gateway's own ``GET /capabilities`` rather than a second, independent +derivation from the deployment's ``.env`` — see the module docstring in +``lobes.cli._commands.capabilities`` for why re-deriving the same contract +twice from two different config sources is exactly what let issue #92 and +issue #96 drift in opposite directions. The tests below that don't care about +the live-vs-offline distinction hit the *offline* fallback (the autouse +``offline_runtime`` fixture in ``tests/conftest.py`` neutralises the gateway +probe, matching how it already neutralises ``/health``); the fake-gateway +tests near the bottom of this file explicitly restore the real probe against +a real loopback server. Both verbs are strictly read-only — no compose/docker +call, no ``--apply``. """ from __future__ import annotations import json +import threading +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer import pytest from lobes.cli import main +from lobes.cli._commands import capabilities as capabilities_module from lobes.roles import ROLES -from lobes.runtime import _compose +from lobes.runtime import _compose, _env _ROLE_INFO_FIELDS = { "role", @@ -31,6 +44,15 @@ "loaded", } +# Captured at import time — BEFORE tests/conftest.py's autouse ``offline_runtime`` +# fixture (correctly) neutralises ``capabilities_module._fetch_gateway_capabilities`` +# for every other test in this file. The fake-gateway tests near the bottom +# restore this real implementation via ``monkeypatch`` (which shares one instance +# across a test's whole fixture graph, including the autouse fixture — the same +# pattern ``tests/test_cli_tunnel.py`` uses to re-enable ``_health.is_healthy`` +# for its own tests) so they exercise the actual HTTP round trip, not a stub. +_REAL_FETCH_GATEWAY_CAPABILITIES = capabilities_module._fetch_gateway_capabilities + def _scaffold_fleet(path): """Write the packaged fleet templates verbatim — the SAME .env `lobes init @@ -51,7 +73,11 @@ def test_capabilities_json_returns_all_six_roles_with_full_metadata(tmp_path, ca rc = main(["capabilities", "--compose-dir", str(tmp_path), "--json"]) assert rc == 0 payload = json.loads(capsys.readouterr().out) - assert set(payload) == set(ROLES) + # "source" is the added top-level sibling (issue #96, t7): no gateway is + # listening on the resolved port in this offline fixture, so the CLI + # degrades to the .env-derived fallback and says so honestly. + assert payload["source"] == "offline" + assert set(payload) - {"source"} == set(ROLES) for role in ROLES: info = payload[role] assert _ROLE_INFO_FIELDS <= set(info) @@ -107,7 +133,8 @@ def test_capabilities_unscaffolded_still_answers_all_six_roles(capsys) -> None: rc = main(["capabilities", "--json"]) assert rc == 0 payload = json.loads(capsys.readouterr().out) - assert set(payload) == set(ROLES) + assert payload["source"] == "offline" + assert set(payload) - {"source"} == set(ROLES) assert payload["cortex"]["loaded"] is True # primary is always wired assert payload["senses"]["loaded"] is False assert payload["embedder"]["loaded"] is False @@ -121,6 +148,46 @@ def test_capabilities_unscaffolded_still_answers_all_six_roles(capsys) -> None: assert payload["cortex"]["context"] == primary_native +def test_capabilities_offline_fallback_never_reports_ready_true(tmp_path, capsys) -> None: + """Job 3 / issue #96: a config file is not evidence of health. + + ``AUDIO_URL`` present in the deployment's ``.env`` makes ``stt``/``tts`` + ``loaded`` (a config fact — the overlay is configured), but the gateway is + unreachable in this test (the autouse ``offline_runtime`` fixture stubs + the probe, matching a real down-gateway), so nothing was ever probed. The + offline fallback must therefore report ``ready=false`` for EVERY role — + not just stt/tts — no matter what ``loaded``/config truth it can compute. + This is the literal issue #96 scenario: ``AUDIO_URL`` was in ``.env`` but + never reached the gateway container's own environment, and the CLI's old + ``.env``-derived registry advertised ``ready=true`` on a path that + actually 404s/503s. + """ + _scaffold_fleet(tmp_path) + _env.set_env(tmp_path / _compose.ENV_FILE, "AUDIO_URL", "http://realtime:8080") + rc = main(["capabilities", "--compose-dir", str(tmp_path), "--json"]) + assert rc == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["source"] == "offline" + # AUDIO_URL is configured, so stt/tts ARE "loaded" (a config fact) ... + assert payload["stt"]["loaded"] is True + assert payload["tts"]["loaded"] is True + # cortex/senses/embedder/reranker are also loaded in the scaffolded fleet. + assert payload["cortex"]["loaded"] is True + # ... but NOTHING was probed, so every single role's `ready` is False. + for role in ROLES: + assert payload[role]["ready"] is False, role + + +def test_capabilities_non_json_table_marks_offline_source(capsys) -> None: + """The human-readable table must also say this is a configured-defaults + view, not a live one — not just the JSON ``source`` key (Job 1).""" + rc = main(["capabilities"]) + assert rc == 0 + out = capsys.readouterr().out + assert "offline" in out + assert "gateway unreachable" in out + + def test_capabilities_never_touches_docker(tmp_path, monkeypatch, capsys) -> None: _scaffold_fleet(tmp_path) @@ -216,6 +283,132 @@ def boom(*a, **k): assert rc == 0 +# --------------------------------------------------------------------------- +# Gateway-client mode (issue #96, task t7): a real loopback fake gateway +# --------------------------------------------------------------------------- +# +# Every test above exercises the OFFLINE fallback (the autouse +# ``offline_runtime`` fixture neutralises the live probe). The tests below +# restore the real ``_fetch_gateway_capabilities`` and point it at an actual +# ``ThreadingHTTPServer`` on an ephemeral port, proving the CLI performs a +# genuine HTTP round trip and renders exactly what the gateway said — not a +# re-derivation, not a re-shaped version of it. + + +class _FakeGatewayHandler(BaseHTTPRequestHandler): + """Serves a fixed JSON body on GET /capabilities; 404s everything else.""" + + payload: dict = {} + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler API + if self.path == "/capabilities": + body = json.dumps(self.payload).encode() + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + else: + self.send_response(404) + self.end_headers() + + def log_message(self, *_a) -> None: # silence test noise + pass + + +def _known_capabilities_payload() -> dict: + """A hand-built payload that deliberately does NOT match anything the + offline `.env`-derived fallback would ever compute (fake models, fake + ports, fake context sizes) — so an exact match against it proves the CLI + rendered the gateway's own answer, not its own guess.""" + payload: dict[str, dict] = {} + for i, role in enumerate(ROLES): + payload[role] = { + "role": role, + "model": f"fake/{role}-model", + "runtime": "vllm", + "endpoint": "http://localhost:9999", + "path": "/v1/fake", + "context": 1000 + i, + "quant": "fake-quant", + "mtp": bool(i % 2), + "responsibilities": [f"{role}-thing"], + "forbidden_responsibilities": [], + "ready": bool(i % 2 == 0), + "loaded": True, + } + return payload + + +@pytest.fixture +def fake_gateway(monkeypatch): + """A real gateway stand-in on an ephemeral port, with the real gateway + probe restored (the autouse fixture stubs it to `None` by default).""" + payload = _known_capabilities_payload() + handler = type("_BoundFakeGatewayHandler", (_FakeGatewayHandler,), {"payload": payload}) + httpd = ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=httpd.serve_forever, daemon=True) + thread.start() + # Restores the REAL implementation (captured at module-import time, before + # tests/conftest.py's autouse fixture stubbed it) for this test only — + # mirrors how tests/test_cli_tunnel.py re-enables `_health.is_healthy`. + monkeypatch.setattr( + capabilities_module, "_fetch_gateway_capabilities", _REAL_FETCH_GATEWAY_CAPABILITIES + ) + try: + yield httpd.server_address[1], payload + finally: + httpd.shutdown() + httpd.server_close() + + +def test_capabilities_json_reproduces_live_gateway_payload_exactly(fake_gateway, capsys) -> None: + """Job 1's core assertion: against a fake gateway serving a known + /capabilities payload, `lobes capabilities --json` reproduces it exactly + (modulo the added `source` sibling).""" + port, known_payload = fake_gateway + rc = main(["capabilities", "--port", str(port), "--json"]) + assert rc == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["source"] == "gateway" + rendered_roles = {k: v for k, v in payload.items() if k != "source"} + assert rendered_roles == known_payload + + +def test_capabilities_non_json_table_marks_gateway_source(fake_gateway, capsys) -> None: + port, known_payload = fake_gateway + rc = main(["capabilities", "--port", str(port)]) + assert rc == 0 + out = capsys.readouterr().out + assert "gateway" in out + assert "live GET /capabilities" in out + # A fake model id from the known payload shows up verbatim in the table. + assert known_payload["cortex"]["model"] in out + + +def test_endpoint_gateway_mode_uses_live_payload_not_offline_guess(fake_gateway, capsys) -> None: + """`lobes endpoint` also asks the gateway first (Job 1: both verbs).""" + port, known_payload = fake_gateway + rc = main(["endpoint", "cortex", "--port", str(port), "--json"]) + assert rc == 0 + payload = json.loads(capsys.readouterr().out) + assert payload == {"role": "cortex", "endpoint": known_payload["cortex"]["endpoint"]} + + +def test_capabilities_gateway_mode_never_touches_docker(fake_gateway, monkeypatch, capsys) -> None: + port, _known_payload = fake_gateway + + def boom(*a, **k): + raise AssertionError("capabilities must never invoke docker/compose") + + monkeypatch.setattr(_compose, "compose_up_build", boom) + monkeypatch.setattr(_compose, "compose_down", boom) + monkeypatch.setattr(_compose, "_run", boom) + monkeypatch.setattr(_compose, "_probe", boom) + rc = main(["capabilities", "--port", str(port), "--json"]) + assert rc == 0 + + # --------------------------------------------------------------------------- # Registration — both verbs show up in --help / overview, don't break either # --------------------------------------------------------------------------- diff --git a/tests/test_doctor.py b/tests/test_doctor.py index a596915..13a6e4c 100644 --- a/tests/test_doctor.py +++ b/tests/test_doctor.py @@ -4,8 +4,9 @@ import json +from lobes import __version__ from lobes.cli import main -from lobes.runtime import _compose, _env +from lobes.runtime import _compose, _env, _health def test_doctor_offline_is_unhealthy(capsys) -> None: @@ -63,3 +64,120 @@ def test_doctor_env_mismatch_warns_but_passes(tmp_path, monkeypatch, capsys) -> # A warn does not fail the run. assert payload["healthy"] is True assert rc == 0 + + +# --------------------------------------------------------------------------- +# gateway_version_match — issue #99 (the structural cause of issue #92): +# Dockerfile.gateway pins MODEL_GEAR_VERSION once at `lobes init` time and +# nothing ever re-pins it, so a deployed gateway container can silently run a +# stale lobes-cli release long after the host CLI (and PyPI) moved on. +# --------------------------------------------------------------------------- + + +def test_doctor_version_check_degrades_when_gateway_unreachable( + tmp_path, monkeypatch, capsys +) -> None: + """Unreachable must NOT be a false pass, and must NOT fail the run either + — a down gateway is ordinary here (the health check treats it the same + way), not evidence of a real skew defect.""" + _compose.write_scaffold(tmp_path, force=True) + monkeypatch.setenv("LOBES_DIR", str(tmp_path)) + monkeypatch.setattr(_compose, "docker_available", lambda: True) + # conftest's autouse fixture already stubs _health.fetch_health to None + # (unreachable) by default — this test relies on that default explicitly. + + rc = main(["doctor", "--json"]) + payload = json.loads(capsys.readouterr().out) + ids = {c["id"]: c for c in payload["checks"]} + check = ids["gateway_version_match"] + assert check["passed"] is False + assert check["severity"] == "info" # non-fatal — never contributes to healthy=False + assert "cannot verify" in check["message"] + assert payload["healthy"] is True # unreachable alone must not fail the run + assert rc == 0 + + +def test_doctor_version_check_degrades_when_gateway_reports_no_version( + tmp_path, monkeypatch, capsys +) -> None: + """A reachable gateway from before issue #99 (no `version` in /health) is + also "cannot verify", not a mismatch and not a match — same non-fatal + info severity as fully unreachable.""" + _compose.write_scaffold(tmp_path, force=True) + monkeypatch.setenv("LOBES_DIR", str(tmp_path)) + monkeypatch.setattr(_compose, "docker_available", lambda: True) + monkeypatch.setattr(_health, "fetch_health", lambda *a, **k: {"status": "ok"}) + + rc = main(["doctor", "--json"]) + payload = json.loads(capsys.readouterr().out) + ids = {c["id"]: c for c in payload["checks"]} + check = ids["gateway_version_match"] + assert check["passed"] is False + assert check["severity"] == "info" + assert payload["healthy"] is True + assert rc == 0 + + +def test_doctor_version_match_passes_at_error_severity(tmp_path, monkeypatch, capsys) -> None: + _compose.write_scaffold(tmp_path, force=True) + monkeypatch.setenv("LOBES_DIR", str(tmp_path)) + monkeypatch.setattr(_compose, "docker_available", lambda: True) + monkeypatch.setattr( + _health, "fetch_health", lambda *a, **k: {"status": "ok", "version": __version__} + ) + + rc = main(["doctor", "--json"]) + payload = json.loads(capsys.readouterr().out) + ids = {c["id"]: c for c in payload["checks"]} + check = ids["gateway_version_match"] + assert check["passed"] is True + assert check["severity"] == "error" # class of failure this check guards, not a claim of one + assert __version__ in check["message"] + assert payload["healthy"] is True + assert rc == 0 + + +def test_doctor_version_mismatch_names_both_versions_and_fails_the_run( + tmp_path, monkeypatch, capsys +) -> None: + """Issue #99's core assertion: a version mismatch is a real, actionable + defect — passed=False, severity="error" (fails the overall run), the + message names BOTH versions, and the remediation names the exact fix.""" + _compose.write_scaffold(tmp_path, force=True) + monkeypatch.setenv("LOBES_DIR", str(tmp_path)) + monkeypatch.setattr(_compose, "docker_available", lambda: True) + monkeypatch.setattr( + _health, "fetch_health", lambda *a, **k: {"status": "ok", "version": "0.1.0"} + ) + + rc = main(["doctor", "--json"]) + payload = json.loads(capsys.readouterr().out) + ids = {c["id"]: c for c in payload["checks"]} + check = ids["gateway_version_match"] + assert check["passed"] is False + assert check["severity"] == "error" + assert "0.1.0" in check["message"] + assert __version__ in check["message"] + assert f"MODEL_GEAR_VERSION={__version__}" in check["remediation"] + assert f"{tmp_path}/.env" in check["remediation"] or str(tmp_path) in check["remediation"] + assert "docker compose up -d --build gateway" in check["remediation"] + # A real skew defect DOES fail the overall run — that is the whole point. + assert payload["healthy"] is False + assert rc == 1 + + +def test_doctor_version_mismatch_remediation_names_placeholder_when_unscaffolded( + monkeypatch, +) -> None: + """No deployment resolved → the remediation still names the exact fix, + with a generic placeholder instead of a concrete path.""" + from lobes.cli._commands import doctor as doctor_module + + monkeypatch.setattr( + _health, "fetch_health", lambda *a, **k: {"status": "ok", "version": "0.1.0"} + ) + report = doctor_module._diagnose(None) + ids = {c["id"]: c for c in report["checks"]} + check = ids["gateway_version_match"] + assert check["passed"] is False + assert "/.env" in check["remediation"] diff --git a/tests/test_gateway_server.py b/tests/test_gateway_server.py index 710d6f6..e5887aa 100644 --- a/tests/test_gateway_server.py +++ b/tests/test_gateway_server.py @@ -14,6 +14,7 @@ import pytest +from lobes import __version__ from lobes.gateway import server as S from lobes.gateway._config import build_config @@ -224,7 +225,11 @@ def fake_open(backend, path, body, headers, *, connect_timeout, read_timeout): def test_integration_health_and_models(gateway) -> None: with urllib.request.urlopen(gateway + "/health", timeout=5) as r: assert r.status == 200 - assert json.load(r)["status"] == "ok" + health = json.load(r) + assert health["status"] == "ok" + # issue #99: /health reports the deployed lobes-cli release so a remote + # client (or `lobes doctor`) can detect artifact skew without docker. + assert health["version"] == __version__ with urllib.request.urlopen(gateway + "/v1/models", timeout=5) as r: payload = json.load(r) assert [m["id"] for m in payload["data"]] == ["P", "F"] From 2bfe7638f50257e9ee3a28a7f499b3f009370a87 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 12:25:16 +0300 Subject: [PATCH 14/23] t8b: fix reasoning-model text smoke test, xfail audio perception (#101) test_live_main_text_returns_nonempty_content assumed a non-thinking model: with max_tokens=16 the cortex model (a Qwen3.6 reasoning model with preserve_thinking, issue #93) can spend the whole budget on its reasoning trace and return content=None with finish_reason="length". Raise the budget to 256 and accept the reasoning trace as evidence of life only when finish_reason explains it as budget exhaustion, so the fallback can't mask a genuine empty-content bug. test_live_multimodal_audio_perception_transcribes_known_word fails for a real, now-tracked reason: the deployed Gemma silently discards input_audio content parts (vLLM gemma4_unified gap, not a checkpoint gap -- issue #101). Mark it xfail(strict=True) citing the mechanism and evidence; the probe body is untouched so it flips to XPASS (and fails the suite) the day audio ingestion starts working. Verified against the live rig (LOBES_SMOKE_BASE_URL=http://localhost:8001): main-text passes, both colour perception tests pass, audio perception xfails, both wire checks pass. Offline: 1169 passed, 9 skipped (unchanged from baseline). --- tests/test_smoke_duo.py | 63 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 59 insertions(+), 4 deletions(-) diff --git a/tests/test_smoke_duo.py b/tests/test_smoke_duo.py index a32ea89..4347e6c 100644 --- a/tests/test_smoke_duo.py +++ b/tests/test_smoke_duo.py @@ -43,6 +43,7 @@ import pytest import yaml +from lobes.assess import _trace_field from lobes.catalog import SUPPORTED_MODELS, resolve_tier from lobes.gateway._config import ( _DEFAULT_MINOR, @@ -369,19 +370,52 @@ def _synthesize_speech(base_url: str, text: str) -> bytes: @_live def test_live_main_text_returns_nonempty_content() -> None: - """model=main (27B primary) responds with non-empty text to a plain prompt.""" + """model=main (27B primary) responds with non-empty text to a plain prompt. + + The cortex model (sakamakismile/Qwen3.6-27B-Text-NVFP4-MTP) is a + *reasoning* model: on every turn it emits a ``reasoning``/``reasoning_content`` + trace before ``content``, and with ``preserve_thinking`` (issue #93) that + trace is populated by default rather than trimmed. A tight ``max_tokens`` + budget can be entirely consumed by the reasoning trace, leaving ``content`` + empty with ``finish_reason: "length"`` -- e.g. ``{"content": None, + "reasoning": "Here's a thinking process:\\n\\n1. **Analyze User Input:**", + ...}``. That is budget exhaustion mid-thought, not a broken model, so this + test gives the model enough budget to finish thinking and answer (256 + tokens) and falls back to the reasoning trace -- only accepting that + fallback when ``finish_reason`` explains it as budget exhaustion, so the + check can't be satisfied by a silently swallowed failure. + """ base_url = os.environ["LOBES_SMOKE_BASE_URL"] resp = _post_chat( base_url, { "model": "main", "messages": [{"role": "user", "content": "Reply with the word hello."}], - "max_tokens": 16, + "max_tokens": 256, "temperature": 0, }, ) - content = resp["choices"][0]["message"].get("content") or "" - assert content.strip(), f"model=main returned empty content; full response: {resp}" + choices = resp.get("choices") or [] + assert choices, f"model=main returned no choices; full response: {resp}" + message = choices[0].get("message", {}) + finish_reason = choices[0].get("finish_reason") + content = (message.get("content") or "").strip() + _, trace_len = _trace_field(message) + + assert content or trace_len, ( + "model=main returned neither content nor a reasoning trace -- nothing " + f"was generated at all (finish_reason={finish_reason!r}); full " + f"response: {resp}" + ) + if not content: + # All 256 tokens went to reasoning with none left to answer. That is + # only an acceptable outcome when finish_reason says so -- otherwise + # this branch would silently paper over a genuine empty-content bug. + assert finish_reason == "length", ( + "model=main returned only a reasoning trace with no content, but " + f"finish_reason was {finish_reason!r} (expected 'length' for a " + f"budget-exhausted reasoning trace); full response: {resp}" + ) # --------------------------------------------------------------------------- @@ -454,6 +488,19 @@ def test_live_multimodal_image_perception_names_colour(colour_name: str, rgb) -> @_live +@pytest.mark.xfail( + strict=True, + reason=( + "issue #101: the deployed Gemma silently discards input_audio content " + "parts -- vLLM's gemma4_unified path wires the vision tower but drops " + "audio (prompt_tokens barely moves and the model claims it heard " + "nothing), even though the checkpoint config declares audio_config/" + "audio_token_id. Sample rate is irrelevant (24 kHz and 16 kHz both " + "fail). This is a vLLM gap, not a checkpoint gap. strict=True: the day " + "audio ingestion starts working this XPASSes and fails the suite, " + "forcing us to remove the xfail and record the fix." + ), +) def test_live_multimodal_audio_perception_transcribes_known_word() -> None: """model=multimodal (Gemma 4 12B) transcribes a ground-truth spoken word. @@ -469,6 +516,14 @@ def test_live_multimodal_audio_perception_transcribes_known_word() -> None: the gateway container (issue #96, /v1/audio/speech -> 404) -- that is a redeploy problem, not a perception failure, and is reported as such by _synthesize_speech() rather than silently skipped. + + Known-failing today for a different, tracked reason (issue #101): the + deployed Gemma silently discards input_audio content parts (a vLLM + gemma4_unified gap, not a checkpoint gap -- see the xfail reason above for + the evidence). Marked xfail(strict=True) so this stays a tracked, visible + regret rather than a silent failure, and so a future fix flips it to an + XPASS that fails the suite until the xfail is removed. The test body is + left exactly as it is -- it is the probe that will prove the fix. """ base_url = os.environ["LOBES_SMOKE_BASE_URL"] wav_bytes = _synthesize_speech(base_url, _AUDIO_WORD) From 65ad98001362ecdff8a26cbeddc79830b39f6845 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 12:35:34 +0300 Subject: [PATCH 15/23] docs(t10): record senses perception evidence, stop advertising audio, retire DSpark route MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Job 1 — docs/gemma-4-12b-nvfp4.md's #71/#73 admission ("not independently re-run against the base checkpoint") is now resolvable: live evidence against coolthor/gemma-4-12B-it-NVFP4A16 via model=multimodal shows image+text VERIFIED (ground truth + negative control) and audio+text NOT SUPPORTED — vLLM's gemma4_unified silently drops the input_audio content part instead of rejecting it (200 OK, audio ignored). Tracked as issue #101. Also corrects the provenance of the coder checkpoint's original "audio+text ✓ (transcribed verbatim)" claim: that check only asserted HTTP 200 + non-empty content against a placeholder clip, never ground truth — and the claim didn't hold when tested properly. lobes/catalog.py's coolthor and coder entry comments carry the same correction. Job 2 (user decision) — a capability lobes cannot serve must not appear in a contract Colleague reads. docs/colleague-stack.md and CLAUDE.md described `senses` as "vision+audio intake/perception"; corrected to vision-only, with a clearly-marked note pointing at issue #101 and the purpose-built `stt` role as the supported path for speech. README.md carried the same overclaim in two spots ("vision+audio gear"); fixed for consistency. lobes/roles.py's ROLE_RESPONSIBILITIES tokens are untouched — they never claimed audio. Job 3 (user decision, closed answered-negative) — docs/gemma4-mtp-draft.md still presented the DSpark route as "the ONE route task t3 should wire next," inconsistent with issue #75's live finding (Gemma4DSparkModel does not load on vLLM 0.23) already recorded in docs/gemma-4-12b-nvfp4.md and docs/vllm-nightly-migration.md. Added a superseded banner citing #75 and pointing at the two docs carrying the current story; the research content stays below it as the record of how the question was answered. Also states that issue #69's last acceptance criterion (a disabled-by-default DSpark entry) is answered-negative: no catalog entry ships for a model that cannot load. Job 4 — README.md's quickstart still documented bare `lobes init --apply` as scaffolding the single-model deployment and framed `--fleet` as the way to opt into the multi-container deployment. Both have been false since issue #69: the fleet duo is the default, `--fleet` is a back-compat no-op, and `--single`/`--legacy` opts out. Reworked both sections accordingly and added a note on the :8000-means-two-different-things ambiguity (container port vs. gateway-published port) that issue #92 was about. uv run pytest -n auto -q: 1169 passed, 9 skipped (matches baseline). markdownlint-cli2 passes on every touched markdown file. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- CLAUDE.md | 7 +++-- README.md | 57 ++++++++++++++++++++++++++---------- docs/colleague-stack.md | 15 ++++++++-- docs/gemma-4-12b-nvfp4.md | 61 +++++++++++++++++++++++++++++++++++---- docs/gemma4-mtp-draft.md | 33 +++++++++++++++++++++ lobes/catalog.py | 19 ++++++++++-- 6 files changed, 164 insertions(+), 28 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 24be134..5d574c6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -58,8 +58,11 @@ output quality is expected and opt-in, not guaranteed by the diagnostic. See Beyond `cortex`, the **fleet** exposes SIX first-class, Colleague-facing **roles** (issue #81) — the primary contract callers should address, not raw model ids: `cortex` (the 27B primary — reasoning/deciding/final authority), -`senses` (the Gemma 4 12B multimodal gear — vision+audio intake/perception; -never decides or takes repo actions), `embedder` (`Qwen/Qwen3-Embedding-0.6B` → +`senses` (the Gemma 4 12B multimodal gear — vision intake/perception; never +decides or takes repo actions; the checkpoint declares audio support but it is +**not currently served** on this vLLM path — issue #101 — so `senses` is +vision-only in practice, and the purpose-built `stt` role, below, is the +supported path for speech), `embedder` (`Qwen/Qwen3-Embedding-0.6B` → `POST /v1/embeddings`), `reranker` (`Qwen/Qwen3-Reranker-0.6B` → `POST /v1/rerank` + `/v1/score`), and the opt-in audio overlay's `stt`/`tts`. Roles are routed by **task family** (`generate` / `embed` / `score` / `rerank`) and diff --git a/README.md b/README.md index 46f7680..90a6f56 100644 --- a/README.md +++ b/README.md @@ -50,9 +50,16 @@ CLIs in loops, so safe-by-default is mandatory. ## Running the model locally (vLLM) `lobes init` scaffolds a deployment directory (default `$HOME/.lobes`) from the -packaged templates: a `docker-compose.yml` that stands up the vLLM model as an -OpenAI-compatible server on `:8000`, plus a `.env`. Tuned for DGX Spark (GB10 -Grace Blackwell, 128 GB unified memory) per +packaged templates. **Since issue #69, bare `lobes init` (no flags) scaffolds +the fleet duo by default** — see ["Running the model behind a gateway +(fleet)"](#running-the-model-behind-a-gateway-fleet) below; that is almost +certainly what you want. This section instead walks through the **legacy +single-model** scaffold: one vLLM server, no gateway, opted into with +`--single` (alias `--legacy`) — a `docker-compose.yml` that stands up the +vLLM model directly as an OpenAI-compatible server on `:8000` (the +*container's own* port; there is no gateway in front of it in this mode — +see the port note in the fleet section below), plus a `.env`. Tuned for DGX +Spark (GB10 Grace Blackwell, 128 GB unified memory) per [build.nvidia.com/spark/vllm](https://build.nvidia.com/spark/vllm). Prerequisites: the [NVIDIA Container Toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html), @@ -60,7 +67,7 @@ and `docker login nvcr.io` with an [NGC API key](https://org.ngc.nvidia.com/setu to pull the `nvcr.io/nvidia/vllm` image. ```bash -lobes init --apply # writes $HOME/.lobes/{docker-compose.yml,.env} +lobes init --single --apply # writes $HOME/.lobes/{docker-compose.yml,.env} (legacy single-model) # edit $HOME/.lobes/.env to set HF_TOKEN if the model repo is gated lobes serve --apply # first run downloads ~28 GB of weights (the 27B primary) lobes status # waits/reports until /health is up @@ -178,19 +185,37 @@ tunnel` for the full flow. ## Running the model behind a gateway (fleet) -`lobes init --fleet` scaffolds a **multi-container** deployment instead of one: the -always-warm Qwen generate primary, two tiny co-resident **embedding** and -**reranker** gears, and a single stdlib **gateway** that fronts them on the host -port the acp `vllm-local` provider already expects. The gateway routes each -request by its `model` field — to the primary, the embedder, or the reranker by -task family (generate / embed / score / rerank) — and defaults an unknown/missing -name to the primary, so existing single-model clients keep working unchanged. The -same front fans `/v1/audio/*` out to the `--audio` overlay, and a warm *generate* -fallback can be wired in later (the gateway adds it, with failover, only when one -is configured). +**This is the default scaffold since issue #69** — plain `lobes init --apply` +(no flags) already gives you this; `--fleet` is a back-compat no-op kept only +so old scripts that passed it keep working, and `--single` (alias `--legacy`) +is what opts you *out*, back to the one-container deployment the previous +section describes. `lobes init` scaffolds a **multi-container** deployment: +the always-warm Qwen generate primary, the Gemma 4 12B multimodal gear, two +tiny co-resident **embedding** and **reranker** gears, and a single stdlib +**gateway** that fronts them all on the host port the acp `vllm-local` +provider already expects — `VLLM_PORT` (packaged default `:8000`; `:8001` on +the reference DGX Spark deployment, deliberately set apart from the +single-model story's default). The gateway routes each request by its +`model` field — to the primary, the multimodal gear, the embedder, or the +reranker by task family (generate / embed / score / rerank) — and defaults +an unknown/missing name to the primary, so existing single-model clients +keep working unchanged. The same front fans `/v1/audio/*` out to the +`--audio` overlay, and a warm *generate* fallback can be wired in later (the +gateway adds it, with failover, only when one is configured). + +> **`:8000` means two different things depending on scaffold — this +> ambiguity is what issue #92 was about.** In the legacy single-model +> section above, `:8000` is the vLLM **container's own** port, published +> straight to the host with no gateway in front of it. Here, `:8000` (or +> whatever `VLLM_PORT` is set to) is the **gateway's** published port, +> fronting several backends by `model` field. They happen to share the same +> default number, but a client must dial whichever origin its deployment +> actually publishes — never assume a bare port number implies a particular +> topology. `lobes doctor` and `GET /capabilities` are the source of truth +> for what a given deployment actually serves and where. ```bash -lobes init --fleet --apply # $HOME/.lobes/{docker-compose.yml,.env,Dockerfile.gateway} +lobes init --apply # $HOME/.lobes/{docker-compose.yml,.env,Dockerfile.gateway} — the default docker login nvcr.io # NGC API key for the vLLM image lobes fleet up --apply # builds the gateway image + starts the backend lobes fleet status # container states + gateway /health + /v1/models @@ -204,7 +229,7 @@ curl -s http://localhost:8000/v1/chat/completions -d '{"model":"sakamakismile/Qw The fleet runs a **default-on `cortex` + `senses` duo** (the `main` + `multimodal` backends) — the 27B Qwen text generate primary served at **128K** (`cortex`, util -`0.30`) and the Gemma 4 12B vision+audio gear served at **32K** (`senses`, util +`0.30`) and the Gemma 4 12B vision gear served at **32K** (`senses`, util `0.14` — provisional pending live validation; `coolthor/gemma-4-12B-it-NVFP4A16`, native MTP default-on — the coder fine-tune, `sakamakismile/gemma-4-12B-coder-…`, is kept as an opt-in `multimodal-coder` gear; see diff --git a/docs/colleague-stack.md b/docs/colleague-stack.md index 6b78f32..74f39a0 100644 --- a/docs/colleague-stack.md +++ b/docs/colleague-stack.md @@ -27,12 +27,23 @@ below. | Role | Backend / service | Endpoint path | What it's for | |---|---|---|---| | `cortex` | `primary` (generate) | `POST /v1/chat/completions` | Reasoning, deciding, planning, tool use, repo actions — the final authority. | -| `senses` | `multimodal` (generate) | `POST /v1/chat/completions` | Intake/perception (text+image+audio) and speaking back to the user. Does **not** decide or act. | +| `senses` | `multimodal` (generate) | `POST /v1/chat/completions` | Intake/perception (text+image) and speaking back to the user. Does **not** decide or act. | | `embedder` | `embed` (pooling) | `POST /v1/embeddings` | Dense text embeddings for memory/retrieval. | | `reranker` | `rerank` (pooling) | `POST /v1/rerank` (+ `/v1/score`) | Reordering/scoring retrieved candidates. | | `stt` | Parakeet (audio overlay, opt-in) | `POST /v1/audio/transcriptions` | Speech-to-text. | | `tts` | Chatterbox (audio overlay, opt-in) | `POST /v1/audio/speech` | Text-to-speech. | +> **`senses` is vision-only intake — audio is not currently served (issue +> #101).** The `coolthor/gemma-4-12B-it-NVFP4A16` checkpoint behind `senses` +> declares an `audio_config` in its own model config, but on this vLLM serving +> path (`gemma4_unified`) an `input_audio` content part is silently **dropped** +> rather than rejected: a caller gets `200 OK` and a fluent answer that ignored +> the audio. Live evidence and the tracking issue are in +> [`docs/gemma-4-12b-nvfp4.md`](gemma-4-12b-nvfp4.md#live-validation-status-71). +> For speech, use the purpose-built **`stt`** role (Parakeet, `POST +> /v1/audio/transcriptions`) instead — it remains first-class and is +> unaffected by this gap. + `cortex`, `senses`, `embedder`, and `reranker` are always enumerated (present with `loaded=false` if their gear isn't wired in this deployment); `stt`/`tts` require `lobes init --fleet --audio`. **`brain` is not a valid role name** — @@ -313,7 +324,7 @@ scaffold did solo, in exchange for co-residency: | **Fleet duo, pre-rebalance** | 64K, util 0.30 | 128K (`MULTIMODAL_MAX_MODEL_LEN=131072`, util 0.22) | | **Fleet duo, current (this doc)** | **128K** (`PRIMARY_MAX_MODEL_LEN=131072`, util 0.30 — util-bound, not context-bound) | **32K** (`MULTIMODAL_MAX_MODEL_LEN=32768`, util 0.14) | -The pre-rebalance duo gave the vision/audio gear its full native 128K at the +The pre-rebalance duo gave the vision gear its full native 128K at the cost of trimming `cortex` to 64K; the current default flips that trade-off — `cortex` (the final-authority reasoning role) now gets its full native 128K, and `senses` (intake/perception) is trimmed to 32K, which is ample for the diff --git a/docs/gemma-4-12b-nvfp4.md b/docs/gemma-4-12b-nvfp4.md index 02e6973..329a3fa 100644 --- a/docs/gemma-4-12b-nvfp4.md +++ b/docs/gemma-4-12b-nvfp4.md @@ -268,15 +268,50 @@ resume from the resolved route against a now-serving gear. ## Live-validation status (#71/#73) {#live-validation-status-71} > **RESOLVED on the DGX Spark (GB10, sm_121, 2026-07-01, #71/#73): the gear -> SERVES and answers text + image + audio requests.** `status` = `load-tested`. +> SERVES and answers text + image requests.** `status` = `load-tested`. > > This #71/#73 validation ran against the **coder** checkpoint (the default > gear at the time). The **base** checkpoint (`coolthor/gemma-4-12B-it-NVFP4A16`, > promoted to default in §7, 2026-07-02) shares the identical > `Gemma4UnifiedForConditionalGeneration` architecture and serve story — the > §7 measurement independently re-confirmed it *serves* and *decodes* correctly -> (28.6 tok/s + MTP) — but the text+image+audio content-correctness checks below -> were not independently re-run against the base checkpoint specifically. +> (28.6 tok/s + MTP). The gap this admission used to flag — whether the +> text+image+audio content-correctness checks further down this section hold +> against the base checkpoint specifically — is now resolved, and the answer +> **splits**: image + text does, audio + text doesn't. See the next +> subsection for the live evidence, and the correction to the coder's +> original "audio + text ✓" line further down. + +### Base checkpoint (coolthor): image + text verified, audio + text not served (#101) + +Live evidence gathered against `coolthor/gemma-4-12B-it-NVFP4A16` on the DGX +Spark (vLLM `0.23.1rc1.dev672+g93d8f834d`), via `model=multimodal`: + +| request | `prompt_tokens` | result | +|---|---|---| +| text only | 15 | — | +| text + image (96×96 solid PNG, stdlib-generated) | **273** (+258) | replies `"Red"` for a red image, `"Blue"` for a blue one | +| text + audio (0.68 s WAV, 24 kHz, from the rig's own Chatterbox TTS) | 34 (+19) | `""` (immediate EOS) | +| text + audio (same clip resampled to 16 kHz) | 48 | *"I cannot hear any audio because you haven't provided a file or a link…"* | + +**Image + text: VERIFIED**, against known ground truth, with a negative +control (a blue image correctly fails a `"red"` assertion). This replaces +the earlier hedge — the base checkpoint's image intake is confirmed, not +merely inherited by architectural similarity to the coder checkpoint. + +**Audio + text: NOT SUPPORTED on this vLLM path.** The image expands the +prompt by 258 tokens — real content injected into the sequence. The audio +adds only ~19 placeholder tokens and no content: the model receives no +signal that audio was attached, so it either emits an immediate EOS (24 kHz) +or, resampled to 16 kHz, a fluent claim that no audio was provided. The +checkpoint's own `config.json` declares `audio_config` and `audio_token_id` +— so this is a **vLLM `gemma4_unified` gap, not a checkpoint gap**. vLLM +**drops** the `input_audio` content part rather than rejecting it: a caller +gets `200 OK` and a fluent answer that silently ignored the audio, which is +the worst failure mode for a caller relying on the advertised capability. +Tracked as **issue #101**. Until #101 lands, treat `senses`/`multimodal` as +vision-only intake — for speech, use the purpose-built `stt` role (Parakeet, +`POST /v1/audio/transcriptions`; see [`docs/colleague-stack.md`](colleague-stack.md)). **The fix: vLLM nightly's native class.** `gemma4_unified` is **early-fusion** multimodal (no separate vision/audio towers — a `vision_embedder` and audio @@ -306,14 +341,23 @@ Runtime matrix tested: |---|---|---|---| | `nvcr.io/nvidia/vllm:26.04-py3` | 0.19.0 | ❌ | ❌ | | `…:26.06-py3` + Transformers `181beb3` | 0.22.1 | ❌ (transformers-backend fallback) | ❌ (o_proj 4096≠8192) | -| **`vllm/vllm-openai:nightly` (shipped)** | **0.23.1rc1.dev** | **✅** | **✅ text+image+audio** | +| **`vllm/vllm-openai:nightly` (shipped)** | **0.23.1rc1.dev** | **✅** | **✅ text+image** (audio accepted, silently dropped — #101) | **Live validation results (util 0.25, `--max-model-len 4096`, GB10):** - ✅ **Serves** — `/health` 200, native class resolved, TRITON auto-forced. - ✅ **Text** — arithmetic + factual answered correctly. - ✅ **Image + text** — described a test image (red circle + text) correctly. -- ✅ **Audio + text** — transcribed a 24 kHz TTS clip **verbatim** (needed `av`). +- ⚠️ **Audio + text** — recorded at the time as "transcribed a 24 kHz TTS clip + **verbatim** (needed `av`)," but the check behind that line asserted only + `HTTP 200` + non-empty response content against a placeholder clip — it never + diffed the transcription against the clip's actual words, so it was **never + verified against ground truth**. When audio + text was later tested properly + against the base checkpoint (the evidence table in + ["Live-validation status"](#live-validation-status-71) above), the claim did + **not** hold: vLLM drops the `input_audio` content part on this serving path + and the model answers as if no audio was attached. Do not rely on this line; + see **#101**. - ✅ **GPU util** — ~**15.7 GiB** actual (weights 8.1 + cudagraph 0.46 + KV 7.2) ≈ **0.12** of the then-**0.69** default-fleet budget. **Superseded 2026-07-02** by the always-on duo retune — see the note below and @@ -337,7 +381,12 @@ Runtime matrix tested: Resolved: - ✅ **Serve-enablement** — native `gemma4_unified` class on vLLM nightly + `TRITON_ATTN`. -- ✅ **Image + audio** — functional (audio needs the `vllm[audio]` extra / PyAV). +- ✅ **Image** — functional, verified against ground truth (see the base-checkpoint + table in ["Live-validation status"](#live-validation-status-71) above). +- ⚠️ **Audio** — NOT served on this vLLM path (#101): the `vllm[audio]` extra + installs cleanly, but `gemma4_unified` silently drops the `input_audio` + content part instead of feeding it to the model. Use the purpose-built `stt` + role (Parakeet, `POST /v1/audio/transcriptions`) for speech instead. - ✅ **Quantization** — `compressed-tensors` (not `modelopt_fp4`). - ✅ **GPU util** — ~15.7 GiB ≈ 0.12 budget. - ✅ **Native MTP (base gear, default-on)** — the public `google/gemma-4-12B-it-assistant` diff --git a/docs/gemma4-mtp-draft.md b/docs/gemma4-mtp-draft.md index 35a3009..292cdfe 100644 --- a/docs/gemma4-mtp-draft.md +++ b/docs/gemma4-mtp-draft.md @@ -1,5 +1,38 @@ # Gemma 4 12B gear — speculative-decoding draft route (issue #75, task t1) +> **SUPERSEDED — the DSpark route this doc resolved to is INVALID (issue +> #75).** The "resolved route" below, `deepseek-ai/dspark_gemma4_12b_block7` +> via the `draft_model` method, was wired and tested live on vLLM 0.23 and +> **fails to load**: `Value error, Model architectures ['Gemma4DSparkModel'] +> are not supported for now`. DSpark's custom drafter architecture is not in +> vLLM 0.23's supported speculative-draft set — this is a dead end, not a +> config problem. The route this repo actually shipped is the **native `mtp` +> method** with the public `google/gemma-4-12B-it-assistant` draft (documented +> below as the "escalation candidate" — it turned out to be the only one that +> works), wired on by default for the base gear. The current, authoritative +> story lives in two other docs: +> +> - [`docs/gemma-4-12b-nvfp4.md`](gemma-4-12b-nvfp4.md#dspark-experiment--invalid-do-not-wire-6) +> — "DSpark experiment — INVALID, do not wire" and the shipped native-MTP +> config. +> - [`docs/vllm-nightly-migration.md`](vllm-nightly-migration.md) §6 — the live +> test that proved DSpark invalid, and §7 — the native-MTP measurement +> (28.6 tok/s @ 57.9% draft acceptance) behind the shipped default. +> +> This also closes issue #69's last acceptance criterion — "a disabled-by-default +> DSpark experiment entry" — as **answered-negative**: no catalog entry is +> shipped for DSpark, because a model that cannot load has nothing to disable +> into. The catalog instead carries the native-MTP config that actually works +> (see `lobes/catalog.py`'s `coolthor/gemma-4-12B-it-NVFP4A16` entry). +> +> The research below is kept (cite-don't-delete) as the record of how the +> question was answered — the HF sourcing search, the tokenizer/vocab +> compatibility check, and the reasoning that led to DSpark being chosen as +> the route to try first are all still accurate as *history*; only the "wire +> DSpark next" recommendation at the top is superseded. + +--- + > Resolves the `unknown_nonblocking` risk in > [`docs/specs/2026-07-01-gemma-4-12b-gear-gets-speculative-decoding-draft.md`](specs/2026-07-01-gemma-4-12b-gear-gets-speculative-decoding-draft.md) > ("whether a `model_type==gemma4_assistant` draft for this exact diff --git a/lobes/catalog.py b/lobes/catalog.py index 96ae0b6..7bb6932 100644 --- a/lobes/catalog.py +++ b/lobes/catalog.py @@ -230,6 +230,16 @@ class SupportedModel: # no-spec floor). "Less coder, more MTP" — see §7 for the full comparison # table. Tool calls use the Python-style "pythonic" parser (matches # runtime._parser.infer_parser, which returns "pythonic" for gemma-4* ids). + # + # Content-correctness, live on THIS checkpoint via model=multimodal: image+text + # VERIFIED against ground truth with a negative control (replies "Red"/"Blue" + # for red/blue test images; a blue image correctly fails a "red" assertion). + # audio+text is NOT served — vLLM's gemma4_unified drops the input_audio + # content part (adds ~19 placeholder tokens, no content) instead of rejecting + # it, so a caller gets 200 OK and a fluent answer that ignored the audio. This + # is a vLLM gap, not a checkpoint gap (config.json declares audio_config / + # audio_token_id). Tracked as #101. See docs/gemma-4-12b-nvfp4.md + # #live-validation-status-71 for the full evidence table. role_hint="multimodal", shape="unified multimodal (text+image+audio)", # Same base-model family as the coder entry — text_config.max_position_ @@ -279,7 +289,12 @@ class SupportedModel: # heterogeneous per-layer head sizes (40 sliding@256 + 8 full@512) that broke # released vLLM <=0.22.1 (transformers-backend fallback → o_proj marlin_gemm # 4096≠8192; a backend flag does NOT fix it — the native class does). Validated - # live: text ✓, image+text ✓, audio+text ✓ (transcribed a TTS clip verbatim); + # live: text ✓, image+text ✓. The original "audio+text ✓ (transcribed a TTS + # clip verbatim)" note was never actually verified against ground truth — the + # check behind it asserted only HTTP 200 + non-empty content against a + # placeholder clip. When tested properly against the base checkpoint (see the + # coolthor entry below), audio+text did NOT hold: vLLM's gemma4_unified drops + # the input_audio content part rather than serving it. Tracked as #101. # ~15.7 GiB footprint ≈ 0.12 budget. See docs/gemma-4-12b-nvfp4.md and #71. role_hint="candidate", shape="unified multimodal (text+image+audio)", @@ -293,7 +308,7 @@ class SupportedModel: # nvidia modelopt. vLLM must be told --quantization=compressed-tensors; # passing modelopt_fp4 fails with a quant-method-mismatch (verified #71). quantization="compressed-tensors", - status="load-tested", # GB10 2026-07-01: text+image+audio ✓ on vLLM nightly (#71/#73) + status="load-tested", # GB10 2026-07-01: text+image ✓; audio+text NOT served (#101) doc="gemma-4-12b-nvfp4.md", task="generate", # No speculative_config: native MTP was measured on this checkpoint (§6/§7) From baf5d86516ed221fdd209601656b936748e32ed1 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 12:42:53 +0300 Subject: [PATCH 16/23] t9: live "advertised implies reachable" gate (tests + single-trigger runner) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the executable local pre-PR gate the "advertised implies reachable" plan exists to create. It is NOT a CI job (CI has no GPU/fleet): a developer runs it against a running deployment before opening a PR. New files: * tests/test_live_capabilities.py — five checks, each mapped to its issue: 1. every ready role in GET /capabilities is reachable at endpoint+path (404 or a bare 5xx-without-Retry-After = fail; #92/#96/#89). 2. every GET /v1/models id is reachable on its own task lane, never a 404 "model does not exist" (#91) — lane resolved from the live contract so pooling models aren't falsely dialed on a chat route. 3. `lobes capabilities --json` agrees with GET /capabilities on endpoint/ready/loaded for all six roles, via source==gateway (#95/#92). 4. gateway GET /health version == lobes.__version__; a missing version field is skew, not a pass (#99). 5. a Colleague can resolve+dial cortex/senses from the contract alone, no hardcoded model ids (#81/#87). * scripts/live-check.sh — the single trigger: resolves the port the way the CLI does (--port -> VLLM_PORT in .env -> 8000), arms the gate via LOBES_SMOKE_BASE_URL, runs pytest, prints a human summary + pass/fail exit. Fail-not-skip: the module skips cleanly when LOBES_SMOKE_BASE_URL is unset (offline suite stays green: 1169 passed, 14 skipped) but FAILS — never skips — on any fault once armed. 429 (pressure shed #88) and 503+Retry-After (honest warming/dead-owner) are treated as reachable so a busy box never goes red. Stdlib only (urllib/json/struct). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- scripts/live-check.sh | 141 +++++++++ tests/test_live_capabilities.py | 535 ++++++++++++++++++++++++++++++++ 2 files changed, 676 insertions(+) create mode 100755 scripts/live-check.sh create mode 100644 tests/test_live_capabilities.py diff --git a/scripts/live-check.sh b/scripts/live-check.sh new file mode 100755 index 0000000..7dc9ca8 --- /dev/null +++ b/scripts/live-check.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash +# scripts/live-check.sh — the single-trigger local "advertised implies reachable" +# gate (plan of the same name, task t9). +# +# Runs `tests/test_live_capabilities.py` against a RUNNING deployment and returns +# a pass/fail exit code. One command, unattended, no prompts, no per-role +# babysitting: it resolves the deployment port the way the CLI does, arms the +# gate via LOBES_SMOKE_BASE_URL, runs pytest, and prints a short human summary. +# +# This is a LOCAL pre-PR gate, not a CI job — CI has no GPU and no fleet. Run it +# from a repo checkout, against the box that serves the model, before opening a +# PR. It is strictly READ-ONLY: it dials the gateway and never restarts, +# rebuilds, switches, or otherwise mutates any container. +# +# Usage: +# ./scripts/live-check.sh [OPTIONS] [-- ] +# +# Options: +# --port N Gateway host port (default: VLLM_PORT in .env, else 8000) +# --compose-dir DIR Deployment dir (default: $LOBES_DIR, $MODEL_GEAR_DIR, +# ~/.lobes, or ~/.model-gear) +# --base-url URL Full gateway origin to test (overrides port resolution, +# e.g. http://localhost:8001). Default: http://localhost: +# -h, --help Show this help and exit +# +# Exit code: 0 iff every check passes; non-zero on any 404 on an advertised path, +# any unreachable endpoint, a CLI/gateway disagreement, or a version skew. A 429 +# (pressure shed) or a 503+Retry-After (honest warming/dead-owner) is treated as +# reachable and does NOT fail the gate. +# +# Why a shell script and not a Makefile target: the job is imperative — resolve a +# deployment dir with a fallback chain, parse a port out of .env, compose a base +# URL, export an env var, run pytest, and translate the result into a one-line +# human verdict. That is script logic, not a dependency graph; there is no +# Makefile in this repo, and `scripts/` already holds the sibling live harness +# `validate-tiers.sh`, whose deployment-dir resolution this mirrors. + +set -euo pipefail + +# --------------------------------------------------------------------------- +# Defaults / argument parsing +# --------------------------------------------------------------------------- +PORT="" +COMPOSE_DIR="" +BASE_URL="" +PYTEST_EXTRA=() + +_usage() { + grep '^#' "$0" | sed -n '/^# Usage:/,/^# Exit code:/p' | head -n -1 | sed 's/^# \?//' + exit 0 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --port) PORT="$2"; shift 2 ;; + --compose-dir) COMPOSE_DIR="$2"; shift 2 ;; + --base-url) BASE_URL="$2"; shift 2 ;; + -h|--help) _usage ;; + --) shift; PYTEST_EXTRA=("$@"); break ;; + *) printf 'error: unknown option: %s\n' "$1" >&2; exit 2 ;; + esac +done + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${REPO_ROOT}" + +# --------------------------------------------------------------------------- +# Resolve the base URL the way the CLI resolves its port: +# --base-url wins outright; else --port; else VLLM_PORT in the deployment's +# .env; else 8000. Deployment dir precedence mirrors lobes.runtime._compose +# (and scripts/validate-tiers.sh): --compose-dir → $LOBES_DIR → +# $MODEL_GEAR_DIR → ~/.lobes → ~/.model-gear. +# --------------------------------------------------------------------------- +_read_env_port() { + # Echo the VLLM_PORT value from an .env file (last assignment wins), stripping + # an inline "# comment", surrounding quotes, and whitespace. Empty if absent. + local envf="$1" + [[ -f "${envf}" ]] || return 0 + grep -E '^[[:space:]]*VLLM_PORT[[:space:]]*=' "${envf}" \ + | tail -n1 \ + | sed -E 's/^[[:space:]]*VLLM_PORT[[:space:]]*=[[:space:]]*//; s/[[:space:]]*#.*$//; s/^"//; s/"$//; s/[[:space:]]*$//' +} + +if [[ -z "${BASE_URL}" ]]; then + if [[ -z "${PORT}" ]]; then + if [[ -z "${COMPOSE_DIR}" ]]; then + if [[ -n "${LOBES_DIR:-}" ]]; then COMPOSE_DIR="${LOBES_DIR}" + elif [[ -n "${MODEL_GEAR_DIR:-}" ]]; then COMPOSE_DIR="${MODEL_GEAR_DIR}" + elif [[ -d "${HOME}/.lobes" ]]; then COMPOSE_DIR="${HOME}/.lobes" + elif [[ -d "${HOME}/.model-gear" ]]; then COMPOSE_DIR="${HOME}/.model-gear" + fi + fi + if [[ -n "${COMPOSE_DIR}" ]]; then + PORT="$(_read_env_port "${COMPOSE_DIR}/.env")" + fi + PORT="${PORT:-8000}" + fi + BASE_URL="http://localhost:${PORT}" +fi + +# --------------------------------------------------------------------------- +# Pick the pytest runner: prefer uv (this repo's documented workflow), fall back +# to a plain `python -m pytest` so the gate still runs outside a uv checkout. +# --------------------------------------------------------------------------- +if command -v uv >/dev/null 2>&1; then + RUNNER=(uv run pytest) + PYBIN=(uv run python) +else + RUNNER=(python -m pytest) + PYBIN=(python) +fi + +CLI_VERSION="$("${PYBIN[@]}" -c 'import lobes; print(lobes.__version__)' 2>/dev/null || echo '?')" + +printf '=== lobes live capabilities gate ===\n' +printf ' deployment dir : %s\n' "${COMPOSE_DIR:-}" +printf ' gateway origin : %s\n' "${BASE_URL}" +printf ' cli version : %s\n' "${CLI_VERSION}" +printf ' invariant : advertised implies reachable (404/unreachable/skew fail; 429/503+Retry-After pass)\n\n' + +# --------------------------------------------------------------------------- +# Arm the gate and run. LOBES_SMOKE_BASE_URL set ⇒ the module fails (never skips) +# on any fault. We do NOT pass -x: the operator wants the full picture — every +# role and model reported in one run. +# --------------------------------------------------------------------------- +export LOBES_SMOKE_BASE_URL="${BASE_URL}" + +set +e +"${RUNNER[@]}" tests/test_live_capabilities.py -v "${PYTEST_EXTRA[@]}" +RC=$? +set -e + +printf '\n=== result ===\n' +if [[ ${RC} -eq 0 ]]; then + printf ' PASS — everything the deployment advertises is reachable, and %s matches.\n' "${BASE_URL}" +else + printf ' FAIL (exit %d) — an advertised capability is NOT reachable, or the deployed\n' "${RC}" + printf ' gateway is version-skewed from this CLI. See the per-check detail above:\n' + printf ' each fault names the role/model and the issue (#91/#92/#95/#96/#99) it maps to.\n' +fi +exit "${RC}" diff --git a/tests/test_live_capabilities.py b/tests/test_live_capabilities.py new file mode 100644 index 0000000..6609b29 --- /dev/null +++ b/tests/test_live_capabilities.py @@ -0,0 +1,535 @@ +"""Live capabilities gate — the executable "advertised implies reachable" check. + +This is the local pre-PR gate the plan of the same name exists to create. It is +**not** a CI job: CI has no GPU and no fleet. It is run by a developer, against a +real running deployment, before opening a PR — the single trigger is +``scripts/live-check.sh`` (which resolves the port the way the CLI does, arms +this module via ``LOBES_SMOKE_BASE_URL``, and returns a pass/fail exit code). + +Why it exists +------------- +Its absence is precisely why the #87 reachability fix shipped in 0.38.0 while the +reference rig kept running 0.36.0 for five days, and why #92 was filed against +lobes as a *code* regression when the code was already correct and merely +undeployed. A green unit suite says the code is right; it says nothing about +whether the code a caller actually dials is the code that shipped. This gate +closes that gap by dialing the deployment and asserting that everything the +deployment *advertises* it can actually *reach*. + +Fail, never skip, when armed +---------------------------- +The module is gated on ``LOBES_SMOKE_BASE_URL`` (reused from +``tests/test_smoke_duo.py`` for consistency). When it is **unset**, every test +here skips cleanly, so the offline suite stays green. When it is **set**, an +operator has explicitly asked for the gate — so an unreachable deployment, +a 404 on an advertised path, or a version skew must **FAIL the run**, never +degrade to ``pytest.skip``. A gate that skips when things are broken is worse +than no gate: it is the exact silent-pass that let the drift above happen. There +is therefore no runtime ``pytest.skip`` anywhere below the module gate. + +The five checks, and the issues each enforces +--------------------------------------------- +1. ``test_advertised_ready_roles_are_reachable`` — for every role in + ``GET /capabilities`` whose ``ready`` is true, dial ``endpoint + path`` and + assert the response proves the path is reachably served. A **404** on an + advertised, ready path is the defect (#92, #96). A dead backend that a + pre-honest-readiness gateway relays as a bare **5xx** without ``Retry-After`` + (e.g. the poisoned-CUDA-context STT sidecar, #89) is also caught — it is + "advertised ready but not honestly reachable". +2. ``test_advertised_models_are_reachable`` — for every id in ``GET /v1/models``, + dial it on its own task lane and assert it is never a 404 "model does not + exist" (#91). A model the fleet lists must never read as "will never exist". + The lane is resolved from the live contract (pooling models go to + ``/v1/embeddings`` / ``/v1/rerank``, everything else to + ``/v1/chat/completions``) so a listed *generate* model that 404s is the + genuine #91 defect, while a pooling model is never falsely dialed on a chat + route it was never meant to answer. +3. ``test_cli_and_gateway_capabilities_agree`` — ``lobes capabilities --json`` + and ``GET /capabilities`` must agree on ``endpoint``, ``ready`` and + ``loaded`` for all six roles (#95, folded into #92). Since t7 the CLI is a + *client* of the gateway, so agreement is by construction whenever the gateway + answers — this check proves the deployment actually implements that (a pre-t7 + CLI/gateway pair would diverge) and that ``lobes capabilities`` reaches the + gateway at all (``source == "gateway"``, not the degraded offline fallback). +4. ``test_deployed_gateway_version_matches_cli`` — the gateway's ``GET /health`` + reports ``{"version": ...}`` (#99). Compare it to the CLI's + ``lobes.__version__`` and FAIL on mismatch. ``Dockerfile.gateway`` runs + ``pip install "lobes-cli==${MODEL_GEAR_VERSION}"``, ``lobes init`` writes that + pin once, and no verb ever re-bumps it — so merged fixes never reach a + deployment unless someone redeploys. A ``/health`` with **no** ``version`` + field at all means the gateway image predates this work: that is skew, not a + pass. +5. ``test_colleague_discovers_and_dials_cortex_and_senses`` — reproduce a + Colleague's discovery path (#81, #87): given ONLY the gateway origin and no + ``COLLEAGUE_*_BASE_URL`` override, resolve ``cortex`` and ``senses`` from the + contract alone and get an answer. No model id is hardcoded — the model, the + endpoint and the path all come from ``GET /capabilities``. + +Reachability classification — 404 vs 429 vs 503 vs 5xx +------------------------------------------------------ +A busy or warming box must not turn the gate red; only a genuinely unreachable +or dishonest one must. So :func:`_classify` treats a response as **reachable** +when it is any 2xx (served), any non-404 4xx (the endpoint parsed and rejected +our deliberately-minimal request, so the path exists), a **429** ``server_busy`` +(the pressure-shed policy working, #88 — the reference rig currently misreads +sticky swap occupancy, #100, which is exactly why 429 must not be a failure), or +a **503 that carries ``Retry-After``** (the honest "owner dead / backend warming" +answer this plan introduced, #14/#89). It treats a response as **unreachable** +only when it is a **404** (advertised path absent), a **connection failure**, a +**503 without ``Retry-After``**, or any **other 5xx** (a bare relay of a dead +backend with no honest retry signal — the STT 502 case). On a 429 the probe +retries once with the documented ``X-Lobes-Override: 1`` header, which forces the +request past the shed so a real 404 hiding behind pressure is still surfaced. + +Everything uses the standard library only (``urllib`` + ``json`` + ``struct``), +mirroring ``lobes/assess.py`` and ``tests/test_smoke_duo.py``. +""" + +from __future__ import annotations + +import json +import os +import struct +import subprocess # nosec B404 — invokes this repo's own `python -m lobes`, no shell +import sys +import urllib.error +import urllib.parse +import urllib.request +from dataclasses import dataclass + +import pytest + +from lobes import __version__ as LOBES_VERSION +from lobes.roles import ROLES + +# --------------------------------------------------------------------------- +# Module gate — armed iff LOBES_SMOKE_BASE_URL is set (fail-not-skip below it). +# --------------------------------------------------------------------------- +_BASE_URL = (os.environ.get("LOBES_SMOKE_BASE_URL") or "").rstrip("/") + +pytestmark = pytest.mark.skipif( + not _BASE_URL, + reason=( + "live capabilities gate needs a running deployment — set " + "LOBES_SMOKE_BASE_URL=http://localhost: (use scripts/live-check.sh, " + "which resolves the port from .env and arms this gate)" + ), +) + +# Generate lanes (cortex/senses, and any un-roled generate model) can be slow +# under load and cortex is a thinking model — be generous. Pooling/audio lanes +# answer fast. Neither ever blocks forever: a stalled socket must FAIL the armed +# gate, not hang an operator's pre-PR run. +_GENERATE_TIMEOUT = 120 +_LANE_TIMEOUT = 60 +_META_TIMEOUT = 15 + +_OVERRIDE_HEADER = "X-Lobes-Override" + + +# --------------------------------------------------------------------------- +# Stdlib HTTP with a structured result (no requests, mirroring lobes/assess.py). +# --------------------------------------------------------------------------- +@dataclass +class _Probe: + """The outcome of one dial: an HTTP status (or None on connection failure).""" + + status: int | None + retry_after: str | None + body: bytes + error: str | None + + +def _url(path: str) -> str: + return _BASE_URL + path + + +def _http(method: str, url: str, *, headers=None, data=None, timeout: int) -> _Probe: + """One request. Never raises — a connection failure folds into ``status=None``.""" + req = urllib.request.Request(url, data=data, method=method, headers=headers or {}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: # local deployment only + return _Probe(resp.status, resp.headers.get("Retry-After"), resp.read(), None) + except urllib.error.HTTPError as exc: # a real HTTP status (4xx/5xx) — reachable wire + return _Probe(exc.code, exc.headers.get("Retry-After"), exc.read(), None) + except (urllib.error.URLError, OSError) as exc: # refused / DNS / timeout — no wire + reason = getattr(exc, "reason", exc) + return _Probe(None, None, b"", str(reason)) + + +def _classify(p: _Probe) -> tuple[bool, str]: + """Map a probe to ``(reachable, human_verdict)`` — the heart of the gate. + + See the module docstring for the full 404/429/503/5xx rationale. + """ + if p.status is None: + return False, f"connection failure ({p.error})" + if p.status == 404: + return False, "404 not-found" + if p.status == 429: + return True, "429 server_busy (pressure shed #88 — reachable)" + if p.status == 503: + if p.retry_after is not None: + return True, "503 + Retry-After (honest warming/dead-owner — reachable)" + return False, "503 without Retry-After (bare relay, not honest)" + if 200 <= p.status < 300: + return True, f"{p.status} served" + if 400 <= p.status < 500: + return True, f"{p.status} (request rejected — path exists, reachable)" + return False, f"{p.status} backend error (no honest Retry-After — dishonest relay)" + + +def _dial(method: str, url: str, *, data=None, content_type=None, timeout: int) -> _Probe: + """Dial once; on a 429 pressure-shed, force-serve once via ``X-Lobes-Override``. + + Returns the probe the gate should classify. A 429 already means "reachable", + but forcing the request past the shed surfaces the true underlying status + (so a 404 hiding behind host pressure is still caught). If the override retry + is itself shed again or blips on the wire, keep the original 429 (still + reachable — a busy box must not turn the gate red). + """ + headers = {"Content-Type": content_type} if content_type else {} + p = _http(method, url, headers=headers, data=data, timeout=timeout) + if p.status == 429: + forced = _http( + method, + url, + headers={**headers, _OVERRIDE_HEADER: "1"}, + data=data, + timeout=timeout, + ) + if forced.status is not None and forced.status != 429: + return forced + return p + + +# --------------------------------------------------------------------------- +# Payload builders — minimal, but real enough to EXERCISE the backend so a dead +# one reveals itself (an empty STT request 422s at the facade without ever +# reaching the poisoned sidecar; a real file 502s — the fault the gate exists +# to catch). +# --------------------------------------------------------------------------- +def _tiny_wav() -> bytes: + """A 1-sample 8 kHz/8-bit mono WAV, built with struct so it can't drift.""" + num_channels, sample_rate, bits = 1, 8000, 8 + block_align = num_channels * bits // 8 + byte_rate = sample_rate * block_align + audio = b"\x80\x00" # one silence sample + even-length pad + fmt = struct.pack(" tuple[bytes, str]: + """A multipart/form-data body carrying a tiny WAV + model field for STT.""" + boundary = "----lobeslivecheckboundary7f3a" + parts = [ + ( + f'--{boundary}\r\nContent-Disposition: form-data; name="model"\r\n\r\n' f"{model}\r\n" + ).encode(), + ( + f'--{boundary}\r\nContent-Disposition: form-data; name="file"; ' + f'filename="probe.wav"\r\nContent-Type: audio/wav\r\n\r\n' + ).encode(), + _tiny_wav() + b"\r\n", + f"--{boundary}--\r\n".encode(), + ] + return b"".join(parts), f"multipart/form-data; boundary={boundary}" + + +def _role_dial(role: str, info: dict) -> _Probe: + """Dial a role's advertised ``endpoint + path`` with a backend-exercising payload.""" + url = (info.get("endpoint") or "").rstrip("/") + (info.get("path") or "") + model = info.get("model") or "" + if role == "stt": + data, ctype = _stt_multipart(model or "whisper-1") + return _dial("POST", url, data=data, content_type=ctype, timeout=_LANE_TIMEOUT) + if role == "tts": + payload = {"model": model or "chatterbox", "input": "ping", "response_format": "wav"} + timeout = _LANE_TIMEOUT + elif role == "embedder": + payload = {"model": model, "input": "ping"} + timeout = _LANE_TIMEOUT + elif role == "reranker": + payload = {"model": model, "query": "ping", "documents": ["alpha", "beta"]} + timeout = _LANE_TIMEOUT + else: # cortex, senses, or any other generate role + payload = { + "model": model, + "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 1, + } + timeout = _GENERATE_TIMEOUT + return _dial( + "POST", + url, + data=json.dumps(payload).encode(), + content_type="application/json", + timeout=timeout, + ) + + +# --------------------------------------------------------------------------- +# Fixtures — the live contract, fetched once and shared. A failure to read the +# contract when the gate is ARMED is itself a hard FAIL (not a skip). +# --------------------------------------------------------------------------- +@pytest.fixture(scope="module") +def caps() -> dict: + """``GET /capabilities`` — the six-role contract, keyed by role name.""" + p = _http("GET", _url("/capabilities"), timeout=_META_TIMEOUT) + if p.status is None: + pytest.fail( + f"GATE ARMED but the gateway is unreachable at {_url('/capabilities')} " + f"({p.error}). LOBES_SMOKE_BASE_URL is set, so this is a FAIL, not a skip: " + "an operator asked for the gate and there is nothing serving. Start the " + "deployment (`lobes serve --apply` / `lobes fleet up --apply`) or point " + "LOBES_SMOKE_BASE_URL at the running gateway." + ) + if p.status != 200: + pytest.fail(f"GET /capabilities returned {p.status}, expected 200. Body: {p.body[:400]!r}") + try: + data = json.loads(p.body) + except ValueError as exc: + pytest.fail(f"GET /capabilities returned a non-JSON body: {exc}; body={p.body[:400]!r}") + missing = [r for r in ROLES if r not in data] + if missing: + pytest.fail(f"GET /capabilities is missing roles {missing}; got keys {sorted(data)}") + return data + + +@pytest.fixture(scope="module") +def model_ids() -> list[str]: + """The ids advertised by ``GET /v1/models`` (the fleet's dialable models).""" + p = _http("GET", _url("/v1/models"), timeout=_META_TIMEOUT) + if p.status is None: + pytest.fail(f"GATE ARMED but GET /v1/models is unreachable ({p.error}) — FAIL, not skip.") + if p.status != 200: + pytest.fail(f"GET /v1/models returned {p.status}, expected 200. Body: {p.body[:400]!r}") + data = json.loads(p.body) + return [entry["id"] for entry in data.get("data", [])] + + +# --------------------------------------------------------------------------- +# Check 1 — advertised-ready roles are reachable (#92, #96, #89). +# --------------------------------------------------------------------------- +def test_advertised_ready_roles_are_reachable(caps: dict) -> None: + """Every role advertised ``ready=true`` must answer its own ``endpoint+path``.""" + reachable: list[str] = [] + faults: list[str] = [] + for role in ROLES: + info = caps[role] + if not info.get("ready"): + continue # only ready roles carry the "advertised implies reachable" promise + url = (info.get("endpoint") or "").rstrip("/") + (info.get("path") or "") + ok, verdict = _classify(_role_dial(role, info)) + line = f" {role:9} {url:42} -> {verdict}" + (reachable if ok else faults).append(line) + assert not faults, ( + "advertised-ready roles that are NOT reachably served " + "(a 404 is #92/#96; a bare 5xx without Retry-After is a dishonestly relayed " + "dead backend, e.g. the poisoned STT sidecar #89):\n" + + "\n".join(faults) + + ("\n\nreachable:\n" + "\n".join(reachable) if reachable else "") + ) + + +# --------------------------------------------------------------------------- +# Check 2 — every advertised model is reachable on its lane (#91). +# --------------------------------------------------------------------------- +def test_advertised_models_are_reachable(caps: dict, model_ids: list[str]) -> None: + """No id in ``/v1/models`` may read as "model does not exist" on its lane.""" + assert model_ids, "GET /v1/models advertised no models at all — nothing is reachable." + embed_model = caps["embedder"].get("model") + rerank_model = caps["reranker"].get("model") + reachable: list[str] = [] + faults: list[str] = [] + for mid in model_ids: + # Resolve the id to its correct task lane from the live contract, so a + # pooling model is never falsely dialed on a chat route it can't answer. + if mid == embed_model: + probe = _dial( + "POST", + _url("/v1/embeddings"), + data=json.dumps({"model": mid, "input": "ping"}).encode(), + content_type="application/json", + timeout=_LANE_TIMEOUT, + ) + lane = "/v1/embeddings" + elif mid == rerank_model: + probe = _dial( + "POST", + _url("/v1/rerank"), + data=json.dumps({"model": mid, "query": "ping", "documents": ["a", "b"]}).encode(), + content_type="application/json", + timeout=_LANE_TIMEOUT, + ) + lane = "/v1/rerank" + else: # cortex/senses and any un-roled generate candidate → completion + probe = _dial( + "POST", + _url("/v1/chat/completions"), + data=json.dumps( + { + "model": mid, + "messages": [{"role": "user", "content": "ping"}], + "max_tokens": 1, + } + ).encode(), + content_type="application/json", + timeout=_GENERATE_TIMEOUT, + ) + lane = "/v1/chat/completions" + ok, verdict = _classify(probe) + line = f" {mid:58} [{lane}] -> {verdict}" + (reachable if ok else faults).append(line) + report = ( + "#91: GET /v1/models advertises models that read as 'model does not exist' " + "when dialed on their own lane (a listed model must never be undialable):\n" + + "\n".join(faults) + + ("\n\nreachable:\n" + "\n".join(reachable) if reachable else "") + ) + assert not faults, report + + +# --------------------------------------------------------------------------- +# Check 3 — `lobes capabilities --json` agrees with GET /capabilities (#95/#92). +# --------------------------------------------------------------------------- +def test_cli_and_gateway_capabilities_agree(caps: dict) -> None: + """The CLI's contract view and the gateway's must agree for all six roles.""" + port = urllib.parse.urlsplit(_BASE_URL).port or 80 + # Run the REAL CLI in a fresh subprocess: tests/conftest.py's autouse + # `offline_runtime` neutralises the in-process gateway probe, and the point + # here is to exercise the actual `lobes capabilities` client end to end. + proc = subprocess.run( # nosec B603 — fixed argv, this repo's own module, no shell + [sys.executable, "-m", "lobes", "capabilities", "--json", "--port", str(port)], + capture_output=True, + text=True, + timeout=30, + ) + assert ( + proc.returncode == 0 + ), f"`lobes capabilities --json --port {port}` exited {proc.returncode}: {proc.stderr.strip()}" + cli = json.loads(proc.stdout) + assert cli.get("source") == "gateway", ( + f"`lobes capabilities` degraded to source={cli.get('source')!r} — it could not " + f"reach the gateway on port {port} and fell back to its offline .env view. " + "The CLI and the gateway can only be proven to agree when the CLI actually " + "reaches the gateway (#96); a degraded fallback is itself a reachability fault." + ) + disagreements: list[str] = [] + for role in ROLES: + for field in ("endpoint", "ready", "loaded"): + cli_val = cli.get(role, {}).get(field) + gw_val = caps[role].get(field) + if cli_val != gw_val: + disagreements.append(f" {role}.{field}: cli={cli_val!r} != gateway={gw_val!r}") + assert not disagreements, ( + "#95/#92: `lobes capabilities --json` and GET /capabilities disagree on " + "endpoint/ready/loaded (they must be one source of truth):\n" + "\n".join(disagreements) + ) + + +# --------------------------------------------------------------------------- +# Check 4 — deployed gateway version matches the CLI (#99 version skew). +# --------------------------------------------------------------------------- +def test_deployed_gateway_version_matches_cli() -> None: + """The gateway's ``/health`` version must equal ``lobes.__version__``.""" + p = _http("GET", _url("/health"), timeout=_META_TIMEOUT) + if p.status is None: + pytest.fail(f"GATE ARMED but GET /health is unreachable ({p.error}) — FAIL, not skip.") + assert p.status == 200, f"GET /health returned {p.status}, expected 200. Body: {p.body[:400]!r}" + health = json.loads(p.body) + gw_version = health.get("version") + assert gw_version is not None, ( + "#99 version skew: GET /health reports NO 'version' field, so this gateway " + "image PREDATES the honest-version work. Dockerfile.gateway pins " + "'lobes-cli==${MODEL_GEAR_VERSION}' once (at `lobes init` time) and no verb " + "re-pins it, so a container can silently run a stale release for days after " + f"the host CLI ({LOBES_VERSION}) and PyPI moved on. A missing version is skew, " + f"not a pass: redeploy the gateway to at least {LOBES_VERSION}. Body: {health!r}" + ) + assert gw_version == LOBES_VERSION, ( + f"#99 version skew: the gateway /health reports {gw_version!r} but this CLI is " + f"{LOBES_VERSION!r}. A merged fix has not reached this deployment — redeploy " + f"(re-run `lobes init` / rebuild the gateway image) so Dockerfile.gateway " + f"re-pins lobes-cli=={LOBES_VERSION}. This is the exact drift (#99) that made " + "#92 look like a code regression when the fix was already published." + ) + + +# --------------------------------------------------------------------------- +# Check 5 — a Colleague can discover + dial cortex/senses from the contract alone +# (#81, #87). No model id is hardcoded — everything comes from /capabilities. +# --------------------------------------------------------------------------- +def test_colleague_discovers_and_dials_cortex_and_senses(caps: dict) -> None: + """Given only the gateway origin, resolve cortex/senses and get an answer.""" + answers: list[str] = [] + faults: list[str] = [] + for role in ("cortex", "senses"): + info = caps[role] + if not info.get("ready"): + faults.append(f" {role}: /capabilities reports ready=false — cannot be discovered") + continue + endpoint = (info.get("endpoint") or "").rstrip("/") + path = info.get("path") or "" + model = info.get("model") or "" + if not endpoint or not path or not model: + faults.append( + f" {role}: contract incomplete (endpoint={endpoint!r} path={path!r} " + f"model={model!r}) — a Colleague cannot dial it" + ) + continue + payload = { + "model": model, # discovered, never hardcoded + "messages": [{"role": "user", "content": "Reply with the single word: ok"}], + "max_tokens": 8, + "temperature": 0, + } + probe = _dial( + "POST", + endpoint + path, + data=json.dumps(payload).encode(), + content_type="application/json", + timeout=_GENERATE_TIMEOUT, + ) + ok, verdict = _classify(probe) + if not ok: + faults.append( + f" {role}: discovered {endpoint + path} but it is not reachable: {verdict}" + ) + continue + if probe.status is not None and 200 <= probe.status < 300: + try: + choices = json.loads(probe.body).get("choices") + except ValueError: + choices = None + if not choices: + faults.append( + f" {role}: dialed {endpoint + path} and got {probe.status} but no " + f"'choices' — not a usable answer. Body: {probe.body[:200]!r}" + ) + continue + answers.append( + f" {role}: discovered via contract, dialed {endpoint + path}, got an answer" + ) + else: + # Reachable but shed/warming (429/503+Retry-After). Discovery still + # worked — the endpoint was resolved from the contract and answered. + answers.append(f" {role}: discovered {endpoint + path}, reachable ({verdict})") + report = ( + "#81/#87 Colleague discovery path failed — a peer given ONLY the gateway " + "origin could not resolve-and-dial these roles from the contract:\n" + + "\n".join(faults) + + ("\n\nsucceeded:\n" + "\n".join(answers) if answers else "") + ) + assert not faults, report From 84fef46f89995a455c68e197b2e2684a079d3c55 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 12:45:53 +0300 Subject: [PATCH 17/23] chore: bump 0.39.0 -> 0.40.0 (advertised implies reachable) CHANGELOG covers the twelve merged tasks. Also records this session's findings into the in-repo eidetic store: the frozen fleet image pin (#99), the sticky swap-occupancy pressure trigger (#100), and senses' silent audio drop (#101). Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- .eidetic/memory/lobes__public.jsonl | 24 ++++++++++++++--------- CHANGELOG.md | 30 +++++++++++++++++++++++++++++ pyproject.toml | 2 +- uv.lock | 2 +- 4 files changed, 47 insertions(+), 11 deletions(-) diff --git a/.eidetic/memory/lobes__public.jsonl b/.eidetic/memory/lobes__public.jsonl index b140e1a..f0137a9 100644 --- a/.eidetic/memory/lobes__public.jsonl +++ b/.eidetic/memory/lobes__public.jsonl @@ -1,18 +1,18 @@ {"id": "qwen3.5-4b-verified-facts", "hash": "3f61d119c727d0f228e4d2e9dd8dfd71690bf0d526c8511dca023eb553c7d64f", "content": "Qwen/Qwen3.5-4B verified from HF config.json (2026-06-26): architecture is HYBRID linear-attention (Gated Delta/SSM) + full-attention (32 layers: 24 linear + 8 full), NOT MoE and NOT plain dense -> no vLLM --moe-backend. It is MULTIMODAL (full ViT, image+video tokens) so serve text-only via --language-model-only. Ships a BUILT-IN MTP draft head (mtp_num_hidden_layers=1) -> native speculative decoding, no separate draft checkpoint. 256K native (max_position_embeddings 262144), mRoPE. dtype bf16, public, Apache-2.0. Tool-call format = qwen3_coder (XML ), NOT hermes. No official nvidia/RedHatAI NVFP4 for the 4B; community cosmicproc/Qwen3.5-4B-NVFP4 exists (NVIDIA ModelOpt NVFP4 W4A4 -> vLLM --quantization modelopt_fp4; conv1d + lm_head left bf16; needs Blackwell for W4A4 kernels).", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "hf-config-json", "topic": "qwen3.5-4b", "relates_to": "lobes-catalog", "date": "2026-06-26"}, "created": "2026-06-26T09:44:18.173778+00:00", "last_recall": "2026-07-03T08:03:33.189858+00:00", "recall_count": 7, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "minor-lobe-spec-issue-64", "hash": "8b97d0eb6db4a548000aa5e6d06068f6173bc562f16c2f8693f0a4ec028da4bb", "content": "lobes issue #64 ('minor lobe') was specced via /think on 2026-06-26 -> docs/specs/2026-06-26-lobes-ships-a-minor-lobe-a-cheap-warm-co-resident.md. Scope (user-confirmed) is the FULL minor-role runtime, not just a catalog entry: add Qwen3.5-4B as catalog gear role_hint=minor, served bf16 WARM CO-RESIDENT behind the gateway, + new read-only verbs lobes run/route/eval with escalation + confidence governance. Key decisions: served checkpoint = bf16 Qwen/Qwen3.5-4B (chosen because unsloth LoRA needs the bf16 base; NVFP4 cosmicproc is doc-only untested); route v1 routes ONLY across lobes GEARS (minor vs primary vs candidate) + escalate, not tools/mesh agents; run/route/eval call the model via the gateway OpenAI endpoint reusing assess.py's stdlib urllib client. Deferred (not yet): actual fine-tuning/lobes train, eval-suite contents, MTP spec-decode for minor. Two code gotchas it requires: (1) runtime/_parser.py infer_parser returns 'hermes' for a qwen3.5 id but the model needs 'qwen3_coder' -> must extend _RULES (catalog test asserts tool_parser==infer_parser(id)); (2) catalog requires non-empty quantization for generate gears, so a bf16 gear needs a quantization-field convention (sentinel that switch/compose translate to omitting --quantization).", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "minor-lobe", "relates_to": "issue-64", "date": "2026-06-26"}, "created": "2026-06-26T09:44:18.175114+00:00", "last_recall": "2026-07-03T23:37:43.106121+00:00", "recall_count": 4, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "minor-lobe-spec-issue-64", "hash": "8b97d0eb6db4a548000aa5e6d06068f6173bc562f16c2f8693f0a4ec028da4bb", "content": "lobes issue #64 ('minor lobe') was specced via /think on 2026-06-26 -> docs/specs/2026-06-26-lobes-ships-a-minor-lobe-a-cheap-warm-co-resident.md. Scope (user-confirmed) is the FULL minor-role runtime, not just a catalog entry: add Qwen3.5-4B as catalog gear role_hint=minor, served bf16 WARM CO-RESIDENT behind the gateway, + new read-only verbs lobes run/route/eval with escalation + confidence governance. Key decisions: served checkpoint = bf16 Qwen/Qwen3.5-4B (chosen because unsloth LoRA needs the bf16 base; NVFP4 cosmicproc is doc-only untested); route v1 routes ONLY across lobes GEARS (minor vs primary vs candidate) + escalate, not tools/mesh agents; run/route/eval call the model via the gateway OpenAI endpoint reusing assess.py's stdlib urllib client. Deferred (not yet): actual fine-tuning/lobes train, eval-suite contents, MTP spec-decode for minor. Two code gotchas it requires: (1) runtime/_parser.py infer_parser returns 'hermes' for a qwen3.5 id but the model needs 'qwen3_coder' -> must extend _RULES (catalog test asserts tool_parser==infer_parser(id)); (2) catalog requires non-empty quantization for generate gears, so a bf16 gear needs a quantization-field convention (sentinel that switch/compose translate to omitting --quantization).", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "minor-lobe", "relates_to": "issue-64", "date": "2026-06-26"}, "created": "2026-06-26T09:44:18.175114+00:00", "last_recall": "2026-07-09T05:13:17.278668+00:00", "recall_count": 5, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "devague-blocking-hard-question-gotcha", "hash": "0b8151753dab00bb4c64218896d4cf470b2bb08e3c1a13b5ad000f6c89e13723", "content": "devague (the /think CLI) gotcha: a BLOCKING hard question (interrogate --hard-question --blocking) is NOT cleared by rejecting the claim it hangs on. convergence._missing_open_uncertainty iterates ALL frame.claims (including rejected) for q.blocking and not q.resolved. There is no CLI verb to resolve a hard question (interrogate has no --resolve; confirm/reject only take c*/h* ids; question --resolve is for the separate 'question' move's items). Workaround: edit the local working-state JSON at .devague/frames/.json and set the hard_question's resolved=true (honest when the question is actually answered). Also: the convergence gate requires a confirmed honesty condition on EVERY confirmed spec-affecting claim (audience/before_state/why_it_matters/after_state/boundary/success_signal/requirement/announcement), not just requirements; decisions/assumptions/non_goals are exempt.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "devague", "relates_to": "think-skill", "date": "2026-06-26"}, "created": "2026-06-26T09:44:18.175320+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "qwen3.5-vllm-0.19-fla-gibberish-gb10", "hash": "d038d33600580542974ea8dc412cce46f97641e2f6a77edc77f9633aa1c492a2", "content": "Serving Qwen3.5 (Qwen3_5ForConditionalGeneration; Gated DeltaNet linear-attention hybrid) on the lobes GB10 fleet's vLLM image nvcr.io/nvidia/vllm:26.04-py3 (engine 0.19.0): the architecture IS registered (verified via ModelRegistry: Qwen3_5ForConditionalGeneration + Qwen3_5MTP present) so it LOADS, BUT vLLM 0.19.0 carries a known FLA/Gated-DeltaNet tensor-format bug on Blackwell (sm_120/sm_121) that produces GIBBERISH output. The fix (vLLM PR #43961) shipped only in v0.23.0. So a live Qwen3.5-4B serve on the current production image likely emits garbled text -> NOT a meaningful live test. Also: V2 model runner crashes on the mixed-attention KV cache (workaround --enforce-v1); GDN needs --max-num-batched-tokens 2096 for cache alignment; sm_121 needs forward-compat (TORCH_CUDA_ARCH_LIST=12.0). --language-model-only correctly drops the vision tower for inference of the full multimodal checkpoint. A meaningful live 4B co-resident test needs vLLM >= 0.23.0 (newer image). The minor-lobe CODE (issue #64: catalog gear, qwen3_coder parser rule, run/route/eval verbs, governance, opt-in co-resident fleet service) is independent and CI-green. Fleet facts: gateway published on host :8001 (not :8000); deployment dir ~/.model-gear; HF cache /home/spark/.cache/huggingface; image runtime=runc.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "qwen3.5-serving", "relates_to": "minor-lobe", "date": "2026-06-26"}, "created": "2026-06-26T11:11:12.839972+00:00", "last_recall": "2026-06-30T16:08:24.560923+00:00", "recall_count": 3, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "qwen3.5-vllm-0.19-fla-gibberish-gb10", "hash": "d038d33600580542974ea8dc412cce46f97641e2f6a77edc77f9633aa1c492a2", "content": "Serving Qwen3.5 (Qwen3_5ForConditionalGeneration; Gated DeltaNet linear-attention hybrid) on the lobes GB10 fleet's vLLM image nvcr.io/nvidia/vllm:26.04-py3 (engine 0.19.0): the architecture IS registered (verified via ModelRegistry: Qwen3_5ForConditionalGeneration + Qwen3_5MTP present) so it LOADS, BUT vLLM 0.19.0 carries a known FLA/Gated-DeltaNet tensor-format bug on Blackwell (sm_120/sm_121) that produces GIBBERISH output. The fix (vLLM PR #43961) shipped only in v0.23.0. So a live Qwen3.5-4B serve on the current production image likely emits garbled text -> NOT a meaningful live test. Also: V2 model runner crashes on the mixed-attention KV cache (workaround --enforce-v1); GDN needs --max-num-batched-tokens 2096 for cache alignment; sm_121 needs forward-compat (TORCH_CUDA_ARCH_LIST=12.0). --language-model-only correctly drops the vision tower for inference of the full multimodal checkpoint. A meaningful live 4B co-resident test needs vLLM >= 0.23.0 (newer image). The minor-lobe CODE (issue #64: catalog gear, qwen3_coder parser rule, run/route/eval verbs, governance, opt-in co-resident fleet service) is independent and CI-green. Fleet facts: gateway published on host :8001 (not :8000); deployment dir ~/.model-gear; HF cache /home/spark/.cache/huggingface; image runtime=runc.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "qwen3.5-serving", "relates_to": "minor-lobe", "date": "2026-06-26"}, "created": "2026-06-26T11:11:12.839972+00:00", "last_recall": "2026-07-09T05:13:17.278668+00:00", "recall_count": 4, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "devague-no-resolve-park", "hash": "42eca84ddc6356f0a0d522b8953439304b28417a757f6ecab5b4f9d674a6a744", "content": "devague (the /think CLI) has NO resolve-park / edit-vagueness move. A park created with --kind unknown_blocking blocks `converge` by design, and there is no CLI verb to resolve it once the user decides the unknown \u2014 re-running `park` with the same text just creates a DUPLICATE vagueness item (v3), it does not update in place. To converge after a blocking unknown is genuinely decided, edit the frame JSON directly at .devague/frames/.json: reclassify the items \"kind\" from \"unknown_blocking\" to \"follow_up\" (or remove it) and record the resolution in its \"text\". Frame files live under .devague/frames/ (NOT .devague/*.json); spec exports land in docs/specs/-.md. Also: a capture with default origin lands as confirmed (treat as user-stated); --origin llm lands as proposed and needs user confirm. Honesty conditions are required on announcement/audience/before_state/after_state/why_it_matters/requirement/success_signal claims but NOT on decision/non_goal/assumption claims.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"area": "devague", "topic": "think-skill-gotcha"}, "created": "2026-06-26T14:54:28.559433+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "minor-lobe-live-spark-fleet", "hash": "e35daba9e905594346fe19cef3be94a89f381915ae79481a3904bdef2c66b7bb", "content": "GO-LIVE (2026-06-26): the minor lobe (Qwen/Qwen3.5-4B) is LIVE on the spark DGX fleet. Deployment dir is ~/.model-gear (legacy; no ~/.lobes/$LOBES_DIR set), compose project name model-gear, composed from BOTH docker-compose.yml + docker-compose.audio.yml (audio services are NOT behind a profile). Did a full re-scaffold to 0.30.0: overwrote docker-compose.yml + Dockerfile.gateway from the repo 0.30.0 templates (gateway now pip-installs lobes-cli==0.30.0, entrypoint lobes.gateway; was 0.26.0/model_gear pre-rename), and patched .env: MODEL_GEAR_VERSION=0.30.0, added MINOR_BASE_URL=http://vllm-minor:8000, MINOR_SERVED_NAME=Qwen/Qwen3.5-4B, COMPOSE_PROFILES=minor. Applied with: docker compose -f docker-compose.yml -f docker-compose.audio.yml --profile minor up -d --build --no-deps gateway vllm-minor (only gateway recreated + vllm-minor started; primary/embed/rerank/audio untouched). GPU utils: primary 0.6 + embed 0.06 + rerank 0.06 + minor 0.10 = 0.82 of 128GB unified \u2014 fits. Gateway host port 8001; /v1/models now lists Qwen/Qwen3.5-4B. AUTH is NOT in compose \u2014 it is a HOST cloudflared tunnel process (decoupled), so compose re-scaffold does not touch it; just keep gateway on host port 8001. Rollback files: ~/.model-gear/{docker-compose.yml,Dockerfile.gateway,.env}.preminor.bak. GOTCHAS: (1) gateway reads MINOR_BASE_URL (NOT MINOR_URL like the _URL convention for PRIMARY/EMBED/RERANK); empty default => minor silently unwired. (2) chatterbox TTS hit a poisoned CUDA context (torch.AcceleratorError cudaErrorUnknown, HTTP 500) \u2014 cleared by docker compose restart chatterbox; synthesis works fine WITH the 4B co-resident (the 4B did not cause GPU exhaustion).", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"area": "deployment", "host": "spark", "topic": "minor-lobe-golive"}, "created": "2026-06-26T15:35:48.071970+00:00", "last_recall": "2026-07-03T23:57:07.981592+00:00", "recall_count": 4, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "issue-68-tiered-routing-spec", "hash": "7d1af62c72622a18e11b0becc36982cf148e2c817a55242ebaf5bef1741673b5", "content": "Issue #68 (tiered model resource routing for Spark unified memory) specced via /think on 2026-06-29 -> docs/specs/2026-06-29-lobes-adds-a-third-middle-model-tier-and-pressure.md. User-confirmed decisions: (1) THIRD tier = a 14B-class NVFP4 checkpoint, INFERENCE-ONLY (~8GB weights) -- it is NOT a LoRA base; LoRA training stays on the existing 4B bf16 minor lobe (so 'scale work + train loras' = 3 inference tiers + 4B trainable base). (2) 27B primary served context trimmed 256K->128K (PRIMARY_MAX_MODEL_LEN=131072) to free ~half the KV cache for the co-resident middle gear. (3) Tier-request surface = gateway model-alias ONLY (caller sends model=cheap|normal|hard to the OpenAI endpoint; gateway resolves to 4B/14B/27B); NO 'lobes select --tier' verb, NO 'route --tier' field. (4) FULL pressure policy IS in scope: read-only swap%/iowait% sampling from /proc, #68 thresholds, degraded-mode state machine, model=hard downgrades with reason=pressure, manual override + downgrade reason cross the OpenAI boundary via HTTP headers, read-only 'lobes status --pressure' emits {tier,model,mode,reason,pressure}. Parked unknowns: exact 14B NVFP4 checkpoint id (same-gen Qwen3.6-14B vs dense Qwen3-14B-NVFP4); exact override/reason header names + whether a streaming response can carry the reason. Risk: 14B NVFP4 must load non-gibberish on the Blackwell vLLM image (dense Qwen3 NVFP4 lower-risk than a 3.5/3.6 hybrid w/ the pre-vLLM-0.23 FLA bug). Current fleet baseline: primary 0.6 + embed 0.06 + rerank 0.06 + minor 0.10. Next leg: /spec-to-plan.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "tiered-routing", "relates_to": "issue-68", "date": "2026-06-29"}, "created": "2026-06-29T19:44:26.978729+00:00", "last_recall": "2026-07-03T08:03:33.189858+00:00", "recall_count": 4, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "minor-lobe-live-spark-fleet", "hash": "e35daba9e905594346fe19cef3be94a89f381915ae79481a3904bdef2c66b7bb", "content": "GO-LIVE (2026-06-26): the minor lobe (Qwen/Qwen3.5-4B) is LIVE on the spark DGX fleet. Deployment dir is ~/.model-gear (legacy; no ~/.lobes/$LOBES_DIR set), compose project name model-gear, composed from BOTH docker-compose.yml + docker-compose.audio.yml (audio services are NOT behind a profile). Did a full re-scaffold to 0.30.0: overwrote docker-compose.yml + Dockerfile.gateway from the repo 0.30.0 templates (gateway now pip-installs lobes-cli==0.30.0, entrypoint lobes.gateway; was 0.26.0/model_gear pre-rename), and patched .env: MODEL_GEAR_VERSION=0.30.0, added MINOR_BASE_URL=http://vllm-minor:8000, MINOR_SERVED_NAME=Qwen/Qwen3.5-4B, COMPOSE_PROFILES=minor. Applied with: docker compose -f docker-compose.yml -f docker-compose.audio.yml --profile minor up -d --build --no-deps gateway vllm-minor (only gateway recreated + vllm-minor started; primary/embed/rerank/audio untouched). GPU utils: primary 0.6 + embed 0.06 + rerank 0.06 + minor 0.10 = 0.82 of 128GB unified \u2014 fits. Gateway host port 8001; /v1/models now lists Qwen/Qwen3.5-4B. AUTH is NOT in compose \u2014 it is a HOST cloudflared tunnel process (decoupled), so compose re-scaffold does not touch it; just keep gateway on host port 8001. Rollback files: ~/.model-gear/{docker-compose.yml,Dockerfile.gateway,.env}.preminor.bak. GOTCHAS: (1) gateway reads MINOR_BASE_URL (NOT MINOR_URL like the _URL convention for PRIMARY/EMBED/RERANK); empty default => minor silently unwired. (2) chatterbox TTS hit a poisoned CUDA context (torch.AcceleratorError cudaErrorUnknown, HTTP 500) \u2014 cleared by docker compose restart chatterbox; synthesis works fine WITH the 4B co-resident (the 4B did not cause GPU exhaustion).", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"area": "deployment", "host": "spark", "topic": "minor-lobe-golive"}, "created": "2026-06-26T15:35:48.071970+00:00", "last_recall": "2026-07-09T05:13:18.802890+00:00", "recall_count": 6, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "issue-68-tiered-routing-spec", "hash": "7d1af62c72622a18e11b0becc36982cf148e2c817a55242ebaf5bef1741673b5", "content": "Issue #68 (tiered model resource routing for Spark unified memory) specced via /think on 2026-06-29 -> docs/specs/2026-06-29-lobes-adds-a-third-middle-model-tier-and-pressure.md. User-confirmed decisions: (1) THIRD tier = a 14B-class NVFP4 checkpoint, INFERENCE-ONLY (~8GB weights) -- it is NOT a LoRA base; LoRA training stays on the existing 4B bf16 minor lobe (so 'scale work + train loras' = 3 inference tiers + 4B trainable base). (2) 27B primary served context trimmed 256K->128K (PRIMARY_MAX_MODEL_LEN=131072) to free ~half the KV cache for the co-resident middle gear. (3) Tier-request surface = gateway model-alias ONLY (caller sends model=cheap|normal|hard to the OpenAI endpoint; gateway resolves to 4B/14B/27B); NO 'lobes select --tier' verb, NO 'route --tier' field. (4) FULL pressure policy IS in scope: read-only swap%/iowait% sampling from /proc, #68 thresholds, degraded-mode state machine, model=hard downgrades with reason=pressure, manual override + downgrade reason cross the OpenAI boundary via HTTP headers, read-only 'lobes status --pressure' emits {tier,model,mode,reason,pressure}. Parked unknowns: exact 14B NVFP4 checkpoint id (same-gen Qwen3.6-14B vs dense Qwen3-14B-NVFP4); exact override/reason header names + whether a streaming response can carry the reason. Risk: 14B NVFP4 must load non-gibberish on the Blackwell vLLM image (dense Qwen3 NVFP4 lower-risk than a 3.5/3.6 hybrid w/ the pre-vLLM-0.23 FLA bug). Current fleet baseline: primary 0.6 + embed 0.06 + rerank 0.06 + minor 0.10. Next leg: /spec-to-plan.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "tiered-routing", "relates_to": "issue-68", "date": "2026-06-29"}, "created": "2026-06-29T19:44:26.978729+00:00", "last_recall": "2026-07-09T05:13:18.802890+00:00", "recall_count": 5, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "issue-69-gemma-duo-spec", "hash": "f0fc6928c15f0438a22ae71f7a4f8db22ed8831854c41a361db6f900b8f45b04", "content": "Issue #69 (make Spark default-serve the Qwen3.6-27B + Gemma4-12B duo) specced via /think 2026-06-30 -> docs/specs/2026-06-30-lobes-now-defaults-to-a-spark-duo-the-qwen3-6-27b.md (frame .devague/frames/lobes-now-defaults-to-a-spark-duo-the-qwen3-6-27b.json). USER-confirmed decisions: (1) 'lobes serve' STOPS being single-model -- with no flags it brings up BOTH main (27B MTP primary) + multimodal (Gemma4-12B); minor/14B/legacy need explicit profiles. (2) NEW tier vocabulary main/minor/multimodal (user reframed away from cheap/normal/hard) -- Gemma gets NEW role_hint='multimodal' in catalog.py; old 'middle' role + 'normal' alias DEPRECATED (normal->multimodal back-compat kept); nvidia/Qwen3-14B-NVFP4 demoted role_hint=middle->candidate (KEPT, cite-don't-delete, legacy profile only). (3) Default Gemma serves NVFP4 + NATIVE MTP ON; DeepSeek DSpark draft (deepseek-ai/dspark_gemma4_12b_block7, shipped 2026-06-27 via DeepSpec) = disabled-by-default experiment toggle. (4) Gemma serves FULL multimodal: vision + AUDIO (Gemma4Unified ingests image/video/audio; vLLM supports it; Gemma4-12B is first medium model with native audio-in/ASR) -- chat audio-in is DISTINCT from the /v1/audio/* Parakeet(STT)/Chatterbox(TTS) overlay (unchanged per boundary). Leading checkpoint candidate: sakamakismile/gemma-4-12B-coder-fable5-composer2.5-MTP-NVFP4 (SAME publisher as the lobes primary, NVFP4+MTP). Accepted plan risks (parked unknown_nonblocking): exact checkpoint pick; Gemma4 --tool-call-parser + runtime/_parser.py infer_parser rule (catalog test asserts tool_parser==infer_parser); loads-on-nv26.04-image (LOWER risk: Gemma4UnifiedForConditionalGeneration registered, auto-detects NVFP4, NOT the Qwen3.5 Gated-DeltaNet/FLA arch that gibbered on sm_120/121); native-MTP vs DSpark --speculative-config JSON; measured GPU util for multimodal Gemma (vision+audio embedders+KV vs 14B's 0.12). DESIGN SEAM for spec-to-plan: main/minor/multimodal breaks the linear cheap/normal/hard pressure ladder (issue #68) -- 'multimodal' is a different capability, not a capability rung, so the pressure-downgrade target needs redefining. TOOLING GOTCHA: devague spec_md exporter (render/spec_md.py:66 _follow_up) DROPS unknown_nonblocking vagueness from the exported spec.md (only renders follow_up/out_of_scope) -- had to append an 'Accepted plan risks' section by hand; the frame JSON retains all parked items, so /spec-to-plan (which reads the frame) still gets them. Next leg: /spec-to-plan.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "gemma-duo", "relates_to": "issue-69", "date": "2026-06-30"}, "created": "2026-06-30T12:37:29.031340+00:00", "last_recall": "2026-07-03T23:57:07.981592+00:00", "recall_count": 8, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "gemma4-unified-unsupported-released-vllm-images", "hash": "34c5ad26dedfe96cb163a6e3e4ad0348a6501f0f8d0be1450f24ebba3fa24840", "content": "t7 live validation (2026-06-30, DGX Spark) for the lobes Gemma 4 12B multimodal gear (issue #69): Gemma 4 12B's architecture is model_type=`gemma4_unified`, and ALL community NVFP4 12B checkpoints use it (sakamakismile/gemma-4-12B-coder-fable5-composer2.5-MTP-NVFP4, AxionML/Gemma-4-12B-NVFP4, coolthor/gemma-4-12B-it-NVFP4A16). NEITHER released NGC vLLM image registers gemma4_unified: nvcr.io/nvidia/vllm:26.04-py3 (vLLM 0.19.0, transformers 4.57.6) NOR nvcr.io/nvidia/vllm:26.05.post1-py3 (vLLM 0.21.0, transformers 5.6.0). vLLM crashes at config load: 'model type gemma4_unified but Transformers does not recognize this architecture ... install Transformers from source'. NOTE: 26.05/vLLM 0.21.0 DOES register Gemma4MTPModel + standard Gemma4ForConditionalGeneration (just not the Unified variant) \u2014 so a standard-arch Gemma4 would load, but the 12B unified multimodal needs nightly/source transformers. Fix path tracked in issue #71 (custom image: 26.05 base + `pip install --pre -U transformers` or git source, verify gemma4_unified registers, check vLLM 0.21.0 pin compat). The lobes catalog gear stays status=configured (correct) until a supporting image lands. Co-resident test approach = zero fleet disruption (run the multimodal container on a free host port alongside the running fleet; primary at util 0.51, +0.12 multimodal = 0.85 < 1.0, no primary bounce). Shipped in PR #72. The vLLM image's --gpus is DeviceRequests nvidia:-1 (=--gpus all), runtime=runc, HF cache /home/spark/.cache/huggingface, compose net model-gear_default, gateway host :8001, deploy dir ~/.model-gear.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "gemma4-unified-vllm", "relates_to": "issue-69", "date": "2026-06-30"}, "created": "2026-06-30T15:40:49.826739+00:00", "last_recall": "2026-07-03T23:26:39.010002+00:00", "recall_count": 5, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "gemma4-unified-unsupported-released-vllm-images", "hash": "34c5ad26dedfe96cb163a6e3e4ad0348a6501f0f8d0be1450f24ebba3fa24840", "content": "t7 live validation (2026-06-30, DGX Spark) for the lobes Gemma 4 12B multimodal gear (issue #69): Gemma 4 12B's architecture is model_type=`gemma4_unified`, and ALL community NVFP4 12B checkpoints use it (sakamakismile/gemma-4-12B-coder-fable5-composer2.5-MTP-NVFP4, AxionML/Gemma-4-12B-NVFP4, coolthor/gemma-4-12B-it-NVFP4A16). NEITHER released NGC vLLM image registers gemma4_unified: nvcr.io/nvidia/vllm:26.04-py3 (vLLM 0.19.0, transformers 4.57.6) NOR nvcr.io/nvidia/vllm:26.05.post1-py3 (vLLM 0.21.0, transformers 5.6.0). vLLM crashes at config load: 'model type gemma4_unified but Transformers does not recognize this architecture ... install Transformers from source'. NOTE: 26.05/vLLM 0.21.0 DOES register Gemma4MTPModel + standard Gemma4ForConditionalGeneration (just not the Unified variant) \u2014 so a standard-arch Gemma4 would load, but the 12B unified multimodal needs nightly/source transformers. Fix path tracked in issue #71 (custom image: 26.05 base + `pip install --pre -U transformers` or git source, verify gemma4_unified registers, check vLLM 0.21.0 pin compat). The lobes catalog gear stays status=configured (correct) until a supporting image lands. Co-resident test approach = zero fleet disruption (run the multimodal container on a free host port alongside the running fleet; primary at util 0.51, +0.12 multimodal = 0.85 < 1.0, no primary bounce). Shipped in PR #72. The vLLM image's --gpus is DeviceRequests nvidia:-1 (=--gpus all), runtime=runc, HF cache /home/spark/.cache/huggingface, compose net model-gear_default, gateway host :8001, deploy dir ~/.model-gear.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "gemma4-unified-vllm", "relates_to": "issue-69", "date": "2026-06-30"}, "created": "2026-06-30T15:40:49.826739+00:00", "last_recall": "2026-07-09T05:13:18.802890+00:00", "recall_count": 7, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "issue-71-gemma4-custom-image-spec", "hash": "167dd266f292a73237b77a2db62529cd14cb3664f363993eb7e34c3d4d1ad09e", "content": "Issue #71 (custom vLLM image for the Gemma 4 12B gemma4_unified multimodal gear) specced via /think 2026-06-30 -> docs/specs/2026-06-30-lobes-ships-a-custom-vllm-image-from-nvcr-io-nvidi.md. USER-confirmed decisions: (1) SCOPE: only vllm-multimodal moves to the custom 26.05-based image NOW; primary/embed/rerank stay on nvcr.io/nvidia/vllm:26.04-py3 (load-tested 27B primary not disturbed); if multimodal works well, open 3 follow-up issues to migrate primary/embed/rerank one-by-one. (2) DELIVERY: local compose build: directive on vllm-multimodal (matches gateway/chatterbox/parakeet/realtime Dockerfile pattern) is the DEFAULT; PLUS an optional MULTIMODAL_IMAGE env override so an operator can point at a ghcr.io/agentculture or local registry:2 tag (registry push feasible but not required). (3) TOOLING: Dockerfile.vllm-gemma4 FROM nvcr.io/nvidia/vllm:26.05.post1-py3 installs transformers with UV (uv pip install --system), NOT pip (user preference). (4) PIN: bake a PINNED transformers ref (version or git SHA) for reproducible rebuilds; exact ref discovered during validation (Q3 left to my default). (5) DELIVERABLE: validate+promote in ONE go on the Spark (spark-f8a9 IS the GB10 Spark, fleet live on 26.04, no vllm-multimodal up) -> the recipe PR merges regardless; status flips configured->load-tested + numbers in docs/gemma-4-12b-nvfp4.md ONLY if all pass (boots, image+text, audio+text, MTP accept>0, util in 0.69 budget); any fail -> stays configured, residual parked. Co-resident validation = zero fleet disruption (free host port alongside running fleet). Parked unknowns v1-v4: exact transformers ref; correct gemma4_mtp method string on vLLM 0.21.0 (r4); measured vision+audio util vs 0.12 (r5); native context (131072 default). Core risk: NO transformers ref may both register gemma4_unified AND keep vLLM 0.21.0 importing -> then await NGC release. Next leg: /spec-to-plan.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "gemma4-custom-image", "relates_to": "issue-71", "date": "2026-06-30"}, "created": "2026-06-30T16:19:55.749207+00:00", "last_recall": "2026-07-03T23:54:01.568905+00:00", "recall_count": 3, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "gemma4-unified-transformers-ref-resolved-r1", "hash": "7490113c6117e54c0ae4c8a4cec0d0accbd66d0cc4ff212ec58b4f893fdb875a", "content": "RESOLVED (2026-06-30, DGX Spark spark-f8a9, issue #71): the custom vLLM image for the Gemma 4 12B gemma4_unified gear BUILDS and the core risk r1 clears. WORKING RECIPE: FROM nvcr.io/nvidia/vllm:26.05.post1-py3 (vLLM 0.21.0+2325b6f0) + transformers from source pinned to commit 181beb3ba4c47098ed8cbc97ee250d1d45ae0107 (resolves to transformers==5.13.0.dev0). This ref BOTH registers gemma4_unified in transformers AutoConfig CONFIG_MAPPING AND leaves vLLM 0.21.0 importing/serving (the r1 question: does ANY ref do both -> YES). Verified in-image: AutoConfig.from_pretrained(sakamakismile/gemma-4-12B-coder-fable5-composer2.5-MTP-NVFP4, trust_remote_code=True) -> model_type gemma4_unified; vllm.ModelRegistry.get_supported_archs() shows [Gemma4ForCausalLM, Gemma4ForConditionalGeneration, Gemma4MTPModel] (Gemma4MTPModel present -> good for native MTP / r4); vllm serve --help exits 0 WITH --gpus (without GPU it errors RuntimeError: Failed to infer device type -- a no-GPU artifact, NOT a real break). TWO BUILD GOTCHAS on this base: (1) a multi-line RUN python3 -c \"...\" MUST use trailing-backslash continuations or Docker parses each body line as an instruction (unknown instruction: import); (2) uv pip install --system needs --break-system-packages because the base /usr Python carries a PEP 668 EXTERNALLY-MANAGED marker (the base sets PIP_BREAK_SYSTEM_PACKAGES=1 for pip only, and PIP_CONSTRAINT=/etc/pip/constraint.txt which uv ignores -- fine, we WANT to override the pinned transformers). uv 0.11.17 already ships in the 26.05 base (no bootstrap needed). Image tag built: lobes/vllm-gemma4:local. STILL OPEN (t4): live serve of the actual checkpoint with --speculative-config gemma4_mtp (r4 method string), measured GPU util (r5), native context (v4), image+text + audio+text functional validation.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "gemma4-unified-vllm", "relates_to": "issue-71", "date": "2026-06-30"}, "created": "2026-06-30T16:56:45.301751+00:00", "last_recall": "2026-07-01T05:32:46.663593+00:00", "recall_count": 2, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "gemma4-12b-serve-blocked-fp4-marlin-vllm021", "hash": "1070e80076271e9ac88ec328159202179f0ef71bc3daca0704697645a7ad1d4d", "content": "t4 live serve (2026-06-30, DGX Spark, issue #71) of sakamakismile/gemma-4-12B-coder-fable5-composer2.5-MTP-NVFP4 on the custom image (nv26.05.post1 vLLM 0.21.0 + transformers main 181beb3) is BLOCKED by a vLLM kernel bug -- the gear CANNOT serve on this image. Findings, in order hit: (1) QUANT: catalog/compose --quantization modelopt_fp4 is WRONG for this checkpoint; config.json quant_method=compressed-tensors, format nvfp4-pack-quantized (NVFP4A16). Must use --quantization compressed-tensors (or omit -> auto-detect). (2) MTP/r4 NEGATIVE: --speculative-config {method: gemma4_mtp} fails NotImplementedError: Unsupported speculative method mtp. vLLM 0.21.0 speculative.py only enables gemma4 MTP when a SEPARATE DRAFT model has model_type==gemma4_assistant (line 515 hf_config_override; use_gemma4_mtp() needs draft_model_config). There is NO auto-derivation from the gemma4_unified target (unlike deepseek_v3->deepseek_mtp), and THIS checkpoint config has no mtp/assistant/nextn keys despite the -MTP name. So native MTP needs a gemma4_assistant draft repo we do not have. Catalog speculative_config for the gemma gear is wrong -> remove/correct. (3) BLOCKER: with --quantization compressed-tensors + no spec-config, the model LOADS but CRASHES at warmup forward: RuntimeError: Shape mismatch: a.size(1)=4096, size_k=8192 in apply_fp4_marlin_linear (compressed_tensors_w4a16_nvfp4.py apply_weights -> marlin_utils_fp4) on Gemma4 attention o_proj. The CompressedTensorsW4A16Fp4 scheme is MARLIN-ONLY (no cutlass/native alt, no env override; VLLM_NVFP4_GEMM_BACKEND/CT_EMULATIONS dont apply). vLLM 0.21.0s W4A16 NVFP4 Marlin kernel mishandles Gemma4s o_proj shape. RESOLVED at config level: native context=131072 (text_config.max_position_embeddings; v4). NOT measurable: r5 GPU util, functional image/audio (blocked by the crash). Maintenance window: stopped primary(0.51)+minor(0.10) to free ~74GB (user-authorized), embed/rerank untouched, fleet restored after. NEXT OPTIONS: (A) try a different Gemma4-12B checkpoint that avoids W4A16 Marlin -- a W4A4 full-NVFP4 (cutlass path) or a nvidia modelopt_fp4 format (modelopt kernel) checkpoint (AxionML/Gemma-4-12B-NVFP4 fmt unknown; coolthor NVFP4A16 = same W4A16 problem likely). (B) await a vLLM that fixes Marlin W4A16 NVFP4 for gemma4 (or adds a non-marlin W4A16 path). Gear stays status=configured; recipe (image+wiring) + r1 win + quant/spec-config corrections still merge.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "gemma4-unified-vllm", "relates_to": "issue-71", "date": "2026-06-30"}, "created": "2026-06-30T17:17:56.428563+00:00", "last_recall": "2026-07-03T23:37:43.106121+00:00", "recall_count": 4, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "gemma4-serve-root-cause-nonsquare-attn-triton", "hash": "acc43da8b918f91ad0d9a68c3afe107e4a6209635253e269ec1a3314613b2200", "content": "t4 deep-dive (2026-06-30, DGX Spark, issue #71): the Gemma 4 12B serve blocker is PRECISELY the non-square attention, NOT a fundamental wall. config text_config: hidden_size=3840, head_dim=256, global_head_dim=512 (DOUBLE head_dim), num_attention_heads=16, attention_k_eq_v=true. The o_proj weight expects K = num_heads*global_head_dim = 16*512 = 8192, but FLASH_ATTN emits num_heads*head_dim = 16*256 = 4096 -> RuntimeError: Shape mismatch a.size(1)=4096 size_k=8192 in marlin_gemm (o_proj). Fix per ai-muninn.com blog: VLLM_ATTENTION_BACKEND=TRITON_ATTN (Triton handles non-square attn, emits 8192). BUT in my test the env DID NOT ENGAGE: vLLM logged Using FLASH_ATTN out of potential backends [FLASH_ATTN,FLASHINFER,TRITON_ATTN,FLEX_ATTENTION], because gemma4_unified runs via vLLMs Transformers modeling backend (no native vLLM Gemma4Unified impl; native Gemma4ForConditionalGeneration is the non-unified variant) and that path did not honor VLLM_ATTENTION_BACKEND=TRITON_ATTN as a docker -e var. OPEN: how to force TRITON_ATTN on the transformers-backend path (CLI flag? hf attn_implementation override? newer vLLM native unified impl?). Same crash on vLLM 0.21.0 AND 0.22.1. RUNTIME MATRIX TESTED: nv26.05.post1=vLLM0.21.0+torch2.12.0a0nv; nv26.06=vLLM0.22.1+torch2.13.0a0nv (BLOG-PROVEN version) -- both keep NGC torch when transformers overlaid (181beb3); host venv nightly=vLLM0.23.1rc1.dev+torch2.11.0+cu130 (stock torch arch_list sm_80..sm_120 only, sm_121 via fwd-compat, torch loads on GB10 but OOMs unless fleet stopped). NOTE 0.24.x does NOT exist yet (latest nightly 0.23.1rc1). BLOG-PROVEN WORKING RECIPE (untested by us): coolthor/gemma-4-12B-it-NVFP4A16 (7.7GB) + vLLM 0.22.1 + VLLM_ATTENTION_BACKEND=TRITON_ATTN -> image+audio+video all work. Our checkpoint (sakamakismile coder-fable5 variant) may also have a nonstandard quant; coolthor is the safer checkpoint. NEXT: (1) figure out TRITON_ATTN engagement on transformers backend; (2) test coolthor; (3) consider switching catalog default to coolthor.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "gemma4-unified-vllm", "relates_to": "issue-71", "date": "2026-06-30"}, "created": "2026-06-30T17:46:48.319291+00:00", "last_recall": "2026-07-01T05:32:46.663593+00:00", "recall_count": 1, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "gemma4-serve-root-cause-nonsquare-attn-triton", "hash": "acc43da8b918f91ad0d9a68c3afe107e4a6209635253e269ec1a3314613b2200", "content": "t4 deep-dive (2026-06-30, DGX Spark, issue #71): the Gemma 4 12B serve blocker is PRECISELY the non-square attention, NOT a fundamental wall. config text_config: hidden_size=3840, head_dim=256, global_head_dim=512 (DOUBLE head_dim), num_attention_heads=16, attention_k_eq_v=true. The o_proj weight expects K = num_heads*global_head_dim = 16*512 = 8192, but FLASH_ATTN emits num_heads*head_dim = 16*256 = 4096 -> RuntimeError: Shape mismatch a.size(1)=4096 size_k=8192 in marlin_gemm (o_proj). Fix per ai-muninn.com blog: VLLM_ATTENTION_BACKEND=TRITON_ATTN (Triton handles non-square attn, emits 8192). BUT in my test the env DID NOT ENGAGE: vLLM logged Using FLASH_ATTN out of potential backends [FLASH_ATTN,FLASHINFER,TRITON_ATTN,FLEX_ATTENTION], because gemma4_unified runs via vLLMs Transformers modeling backend (no native vLLM Gemma4Unified impl; native Gemma4ForConditionalGeneration is the non-unified variant) and that path did not honor VLLM_ATTENTION_BACKEND=TRITON_ATTN as a docker -e var. OPEN: how to force TRITON_ATTN on the transformers-backend path (CLI flag? hf attn_implementation override? newer vLLM native unified impl?). Same crash on vLLM 0.21.0 AND 0.22.1. RUNTIME MATRIX TESTED: nv26.05.post1=vLLM0.21.0+torch2.12.0a0nv; nv26.06=vLLM0.22.1+torch2.13.0a0nv (BLOG-PROVEN version) -- both keep NGC torch when transformers overlaid (181beb3); host venv nightly=vLLM0.23.1rc1.dev+torch2.11.0+cu130 (stock torch arch_list sm_80..sm_120 only, sm_121 via fwd-compat, torch loads on GB10 but OOMs unless fleet stopped). NOTE 0.24.x does NOT exist yet (latest nightly 0.23.1rc1). BLOG-PROVEN WORKING RECIPE (untested by us): coolthor/gemma-4-12B-it-NVFP4A16 (7.7GB) + vLLM 0.22.1 + VLLM_ATTENTION_BACKEND=TRITON_ATTN -> image+audio+video all work. Our checkpoint (sakamakismile coder-fable5 variant) may also have a nonstandard quant; coolthor is the safer checkpoint. NEXT: (1) figure out TRITON_ATTN engagement on transformers backend; (2) test coolthor; (3) consider switching catalog default to coolthor.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "gemma4-unified-vllm", "relates_to": "issue-71", "date": "2026-06-30"}, "created": "2026-06-30T17:46:48.319291+00:00", "last_recall": "2026-07-09T05:13:18.802890+00:00", "recall_count": 2, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "issue-75-gemma4-mtp-spec", "hash": "3730e1ff2f26b4d477684710af9197daf4f2f25348579c0ddc834c6e65e8f3d0", "content": "Issue #75 (Gemma4 native MTP: source/build a gemma4_assistant draft for speculative decoding) was specced via /think on 2026-07-01 -> docs/specs/2026-07-01-gemma-4-12b-gear-gets-speculative-decoding-draft.md (frame .devague/frames/the-gemma-4-12b-gear-gets-speculative-decoding-lob.json). USER-confirmed decisions (4 AskUserQuestion answers): (1) SCOPE = any speculative-decoding speedup counts, NOT strictly native gemma4_mtp -> measure the EXISTING DSpark draft_model route (deepseek-ai/dspark_gemma4_12b_block7) FIRST as the cheap path; pursue a native gemma4_assistant draft only if DSpark proves the win is real but insufficient. (2) BUILD APPETITE = NO -- training/distilling a gemma4_assistant draft head is a SEPARATE follow-up, not in #75; #75 stops at sourced-or-DSpark + documented verdict. (3) DONE = a MEASURED VERDICT (restore speculative_config in catalog+compose with recorded acceptance%/speedup beating baseline, OR a documented negative with the numbers that rule it out). (4) SERVE GATING = #75 is BLOCKED on #71 serve-enablement -- does NOT ship draft wiring ahead of serve; only desk-sourcing a candidate draft can start before #71 lands. GROUNDING: gemma4_mtp has NO self-speculation (vLLM 0.21/0.22 need a separate model_type==gemma4_assistant draft; deepseek_v3->deepseek_mtp auto-derivation has no gemma equivalent); {method:gemma4_mtp} is rejected Unsupported speculative method; the served -MTP checkpoint exposes no mtp/assistant/nextn keys. Catalog->compose wiring follows the 27B mtp_compose_command_items() pattern. Hard dependency #71: gear LOADS but does not SERVE (non-square attention global_head_dim 512 != head_dim 256 needs TRITON_ATTN, not honored on vLLM transformers backend -> o_proj GEMM 4096!=8192). EXPORTER GOTCHA confirmed again: devague spec_md drops unknown_nonblocking vagueness (whether a sourceable gemma4_assistant draft EXISTS on HF) -> restored by hand in the spec.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"topic": "gemma4-mtp-spec", "relates_to": "issue-75", "source": "claude-code-session", "date": "2026-07-01"}, "created": "2026-06-30T22:37:05.301533+00:00", "last_recall": "2026-07-01T18:13:02.194724+00:00", "recall_count": 2, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "issue-75-gemma4-mtp-workforce-findings", "hash": "2d373fe4374cd170fa61deb6a15e9435c41300f9d4862b5c494d257410bbaeb8", "content": "Issue #75 workforce run (2026-07-01, /assign-to-workforce, branch spec/issue-75-gemma4-mtp-draft). Wave 0 (t1,t2) built + merged as DOCS; t3 DEFERRED (re-binned behind #71). KEY FINDINGS: (1) t1 web-research RESOLVED the parked unknown: a NATIVE gemma4_assistant-family draft DOES exist for this checkpoint family -- google/gemma-4-12B-it-assistant (model_type gemma4_unified_assistant, Google DeepMind, Apache-2.0). vLLM hf_config_override recognizes BOTH gemma4_assistant (plain E2B/E4B/26B-A4B/31B) and gemma4_unified_assistant (the Unified/multimodal line = our checkpoint family); both normalize to internal gemma4_mtp with forced n_predict=1. Recorded as the ESCALATION candidate; chosen route stays DSpark draft_model (deepseek-ai/dspark_gemma4_12b_block7) per user measure-first. DSpark config.json (fetched): target_model_type gemma4_unified, num_target_layers 48, vocab_size 262144 -- purpose-built for this target. All 3 checkpoints share vocab_size 262144 + GemmaTokenizer (necessary-not-sufficient for acceptance; measured numbers are t4, gated on #71). Live behaviour on vLLM 0.22.1 flagged to-verify-at-serve. Doc: docs/gemma4-mtp-draft.md. (2) PLANNING CORRECTION: t3 (make gemma wiring catalog-driven) is NOT buildable-now -- the repo GUARDS the no-spec invariant with 3 tests naming #75 as the follow-up: test_gemma_has_no_speculative_config (asserts gemma.speculative_config==\"\"), test_fleet_compose_multimodal_vision_active_no_spec_decode (asserts vllm-multimodal has NO --speculative-config), + the MTP-items drift guard. Adding DSpark speculative_config + flipping those guards IS the restore action, gated behind measurement (t5) + #71. So only t1+t2 were buildable-now; t3->t5 resume when #71 serve-enablement lands. Docs PR opened for the #75 spec/plan/grounding. NOTE test_speculative_config_only_on_mtp_checkpoints passes for gemma (id contains MTP) so it is NOT a blocker -- the two blockers are the explicit gemma/no-spec assertions.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"topic": "gemma4-mtp-workforce", "relates_to": "issue-75", "source": "claude-code-session", "date": "2026-07-01"}, "created": "2026-07-01T05:07:24.807686+00:00", "last_recall": "2026-07-01T18:13:02.194724+00:00", "recall_count": 1, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "issue-75-gemma4-mtp-workforce-findings", "hash": "2d373fe4374cd170fa61deb6a15e9435c41300f9d4862b5c494d257410bbaeb8", "content": "Issue #75 workforce run (2026-07-01, /assign-to-workforce, branch spec/issue-75-gemma4-mtp-draft). Wave 0 (t1,t2) built + merged as DOCS; t3 DEFERRED (re-binned behind #71). KEY FINDINGS: (1) t1 web-research RESOLVED the parked unknown: a NATIVE gemma4_assistant-family draft DOES exist for this checkpoint family -- google/gemma-4-12B-it-assistant (model_type gemma4_unified_assistant, Google DeepMind, Apache-2.0). vLLM hf_config_override recognizes BOTH gemma4_assistant (plain E2B/E4B/26B-A4B/31B) and gemma4_unified_assistant (the Unified/multimodal line = our checkpoint family); both normalize to internal gemma4_mtp with forced n_predict=1. Recorded as the ESCALATION candidate; chosen route stays DSpark draft_model (deepseek-ai/dspark_gemma4_12b_block7) per user measure-first. DSpark config.json (fetched): target_model_type gemma4_unified, num_target_layers 48, vocab_size 262144 -- purpose-built for this target. All 3 checkpoints share vocab_size 262144 + GemmaTokenizer (necessary-not-sufficient for acceptance; measured numbers are t4, gated on #71). Live behaviour on vLLM 0.22.1 flagged to-verify-at-serve. Doc: docs/gemma4-mtp-draft.md. (2) PLANNING CORRECTION: t3 (make gemma wiring catalog-driven) is NOT buildable-now -- the repo GUARDS the no-spec invariant with 3 tests naming #75 as the follow-up: test_gemma_has_no_speculative_config (asserts gemma.speculative_config==\"\"), test_fleet_compose_multimodal_vision_active_no_spec_decode (asserts vllm-multimodal has NO --speculative-config), + the MTP-items drift guard. Adding DSpark speculative_config + flipping those guards IS the restore action, gated behind measurement (t5) + #71. So only t1+t2 were buildable-now; t3->t5 resume when #71 serve-enablement lands. Docs PR opened for the #75 spec/plan/grounding. NOTE test_speculative_config_only_on_mtp_checkpoints passes for gemma (id contains MTP) so it is NOT a blocker -- the two blockers are the explicit gemma/no-spec assertions.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"topic": "gemma4-mtp-workforce", "relates_to": "issue-75", "source": "claude-code-session", "date": "2026-07-01"}, "created": "2026-07-01T05:07:24.807686+00:00", "last_recall": "2026-07-09T05:13:18.802890+00:00", "recall_count": 2, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "gemma4-unified-needs-vllm-nightly-native-class", "hash": "7e085f2a0642b8f4e57b99db304ee50a8b4d77e61170630a49d14956f73fba03", "content": "DEFINITIVE (2026-07-01, DGX Spark GB10, issue #71/#73): the Gemma 4 12B gemma4_unified checkpoint (sakamakismile/gemma-4-12B-coder-fable5-composer2.5-MTP-NVFP4) CANNOT serve on our pinned custom image (NGC 26.06 = vLLM 0.22.1+7b9cb5b7.dev + transformers 181beb3). ROOT CAUSE proven by live test: gemma4_unified has HETEROGENEOUS PER-LAYER head sizes -- 40 sliding_attention layers at head_dim=256 (o_proj in=4096) + 8 full_attention layers at global_head_dim=512 (o_proj in=8192); verified from safetensors weight shapes (L5/L47 q_proj=[8192], o_proj in=8192; sliding layers q_proj=[4096], o_proj in=4096). vLLM 0.22.1 has NO native Gemma4UnifiedForConditionalGeneration class (only gemma4/gemma4_mm/gemma4_mtp), so it falls to the Transformers modeling backend (TransformersMultiModalForCausalLM), which builds EVERY layer's Attention with a SINGLE head_size=get_head_size()=256 -> the 8 full-attention layers emit 16*256=4096 but their o_proj wants 16*512=8192 -> RuntimeError: Shape mismatch a.size(1)=4096 size_k=8192 in torch.ops._C.marlin_gemm at profile_run/_dummy_run. PRIOR TRITON HYPOTHESIS DISPROVEN: I engaged TRITON via the untried --attention-backend TRITON_ATTN CLI FLAG (log: 'Using AttentionBackendEnum.TRITON_ATTN backend') -- a different code path than the env var the prior session tried -- and it CRASHED IDENTICALLY. The kernel was never the issue; the single head_size is. No attention-backend flag or head_size override fixes it (256 breaks full layers, 512 breaks sliding layers). Weights DO load fine on the transformers backend (7.97GiB, no missing-tower errors) -- gemma4_unified is ENCODER-FREE early-fusion: no vision_tower/audio_tower, just model.vision_embedder(patch Dense+LN+2D pos, 13 wts) + model.embed_audio/embed_vision projections. sakamakismile AND coolthor/gemma-4-12B-it-NVFP4A16 are BYTE-IDENTICAL structure (verified via HTTP-range read of coolthor's remote safetensors header: same 1335 tensors) -- #74's 'coolthor is a safer checkpoint' premise is FALSE; both are gemma4_unified and hit the same wall. THE FIX (evidence-backed, not yet run): vLLM NIGHTLY ships the native Gemma4UnifiedForConditionalGeneration class (docs.vllm.ai/en/latest/api/vllm/model_executor/models/gemma4_unified/) which unifies the physical page size by giving the two attention types DIFFERENT block_sizes; recipe = nightly vLLM + VLLM_ATTENTION_BACKEND=TRITON_ATTN (+ vllm[audio], --limit-mm-per-prompt '{\"image\":4,\"audio\":1}'). REFS: vllm-project/recipes Google/Gemma4.md (uv pip install -U vllm --pre); lna-lab/gemma4-12b-vllm-sm120 (Blackwell SM120 proven via stock vllm/vllm-openai:nightly image, single-GPU; released <=0.22.1 cannot load gemma4_unified). SO #73 serve-enablement = BUMP the custom image's vLLM to NIGHTLY (native unified class), NOT just add a flag or the transformers overlay. #71 gated on that. Nightly torch loads on GB10 (sm_121 via fwd-compat) but is memory-hungry (may need primary down). Serve validation used docker run standalone on a free port 8100, --max-model-len 4096 --gpu-mem-util 0.12 --quantization compressed-tensors, minor gear stopped to free ~13GB (user-authorized), primary kept up, minor restored after.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"topic": "gemma4-unified-vllm", "relates_to": "issue-71", "source": "claude-code-session", "date": "2026-07-01"}, "created": "2026-07-01T06:04:05.814462+00:00", "last_recall": "2026-07-01T18:13:02.194724+00:00", "recall_count": 1, "links": ["gemma4-serve-root-cause-nonsquare-attn-triton", "gemma4-12b-serve-blocked-fp4-marlin-vllm021"], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "gemma4-unified-serve-resolved-nightly", "hash": "d870f6ec640f5480ff61016667fac7632b3cb73778627a73d4874bc6ee0fe3bd", "content": "RESOLVED & SHIPPED (2026-07-01, lobes v0.34.0, branch issue-73-gemma4-serve-nightly): the Gemma 4 12B gemma4_unified gear now SERVES on the DGX Spark GB10. FIX = Dockerfile.vllm-gemma4 rebased FROM vllm/vllm-openai:nightly (pinned by digest sha256:7c5a10e9; vLLM 0.23.1rc1.dev, native Gemma4UnifiedForConditionalGeneration class + transformers 5.12.1) + the vllm[audio] extra (av/soundfile/librosa/soxr installed via uv; audio input RESAMPLES via av/PyAV -- text+image worked without it, audio needed `av` specifically). Live-validated standalone (docker run, port 8100): text OK (Tokyo/42), image+text OK (described red-circle+GEMMA test image), audio+text OK (transcribed a 24kHz chatterbox TTS clip verbatim). Footprint ~15.7GiB (weights 8.1 + cudagraph pool 0.46 actual + KV 7.2) ~= 0.12 of 128GB. TWO vLLM 0.23 config gotchas shipped in compose/env.example: (1) VLLM_MEMORY_PROFILER_ESTIMATE_CUDAGRAPHS=0 -- vLLM's cudagraph memory ESTIMATE (12.74GiB) >> actual (0.46GiB) starved KV so util 0.12 failed 'No available memory for the cache blocks'; (2) MULTIMODAL_MAX_MODEL_LEN default 131072->8192 -- at util 0.12 the KV holds only ~24k tokens, 128k native is unserveable at the co-resident lane budget. Catalog gemma status configured->load-tested; test_gemma4_dockerfile.py + test_catalog.py updated. Closes #71 + #73; #74 (coolthor) mooted -- coolthor is byte-identical gemma4_unified, same fix. See [[gemma4-unified-needs-vllm-nightly-native-class]] for root-cause detail. OPS NOTE: the nightly serve needed a PRIMARY-DOWN window -- the GB10 (128GB unified, ~113GB baseline used with the full fleet) cannot reliably co-host gemma's ~15GB even minor-down (freed 13GB gets reabsorbed); a memory-profiling race (free memory increasing mid-profile) fails startup, and repeated attempts OOM-restarted the primary once (it recovered).", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"topic": "gemma4-unified-vllm", "relates_to": "issue-71", "source": "claude-code-session", "date": "2026-07-01"}, "created": "2026-07-01T07:13:59.431728+00:00", "last_recall": null, "recall_count": 0, "links": ["gemma4-unified-needs-vllm-nightly-native-class"], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "gb10-cuda-free-and-gemma-image-2026-07-01", "hash": "a433d6112ec6c07ee256a8ef33e001655b56ef29015cf4f95e9b6cec0b33984a", "content": "Benchmarking a co-resident gear on the shared DGX Spark GB10: the box runs near-full (steady ~114/121 GiB used, swap often full). `free -h` MISLEADS for GPU work \u2014 on the unified LPDDR5X memory, other vLLM gears CUDA reservations count against a NEW process, so CUDA-visible free is far less than free shows (measured ~19 GiB free when free -h showed ~33 GiB available). Check the real number with: docker run --rm --gpus all --entrypoint python3 -c \"import torch;print(torch.cuda.mem_get_info())\". To bench the Gemma 4 12B multimodal gear standalone we freed room by stopping minor and reducing the 27B primary to 64K context + util 0.38 (edit PRIMARY_MAX_MODEL_LEN/PRIMARY_GPU_MEM_UTIL in ~/.model-gear/.env, then docker compose up -d vllm-primary). Also: the on-disk lobes/vllm-gemma4:local tag is STALE (vLLM 0.21, falls back to TransformersMultiModalForCausalLM and crashes/OOMs); the validated image is lobes/vllm-gemma4:nightly-audio (vLLM 0.23.1rc1.dev672, native Gemma4UnifiedForConditionalGeneration). lobes fleet up --build rebuilds :local fresh from Dockerfile.vllm-gemma4. Gemma 12B bench result: ~23 tok/s single-stream decode, no spec-decode.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "session", "repo": "lobes-cli", "topic": "gemma4-benchmark"}, "created": "2026-07-01T16:36:32.758857+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} @@ -24,7 +24,13 @@ {"id": "always-on-duo-budget-validated-2026-07-02", "hash": "166b20b11f5aa62b85d2712aa96eed3abd9329bbfcb6ffe29a665a504d2057be", "content": "ALWAYS-ON DUO serving config LIVE-VALIDATED (2026-07-02, DGX Spark GB10, branch spec/fleet-nightly-unification, wired into templates). User wanted: Gemma base always-on at MAX context + Qwen 27B always-on (lower context OK) + both co-resident. VALIDATED co-residence: Gemma base (coolthor/gemma-4-12B-it-NVFP4A16 + native MTP) at 128K context / util 0.22 (KV holds 128K at 4.67x concurrency; footprint ~26 GiB = weights 9.25 incl assistant draft + graph 4.17 + KV ~12.6) AND Qwen3.6-27B-Text-NVFP4-MTP at 64K context / util 0.35-measured (64K at 6.36x, KV 17.5 GiB) both HEALTHY + serving together (~108 GiB used / ~13 GiB free, with embed+rerank+co-tenant services eidetic/nova/realtime/gateway up). Answer to 'can Gemma-128K + Qwen-64K co-reside?' = YES. TEMPLATE DEFAULTS now: PRIMARY_MAX_MODEL_LEN=65536 (64K, trimmed from 128K), PRIMARY_GPU_MEM_UTIL=0.30 (shaved from validated 0.35 for headroom), MULTIMODAL_MAX_MODEL_LEN=131072 (128K native, up from co-resident 8192), MULTIMODAL_GPU_MEM_UTIL=0.22 (up from 0.12). New default-fleet budget = 0.30 + 0.22 + 0.06 (embed) + 0.06 (rerank) = 0.64. KEY LESSON: Gemma-128K is CHEAP -- at util 0.45 it got 42.88 GiB KV = 13.9x the 128K context, so 128K needs only ~util 0.20-0.22 (earlier 0.40 estimate was wrong; the #71 'util 0.12 holds 24K, 128K needs much larger util' claim is SUPERSEDED). The 27B KV is util-bound not context-bound: 32K vs 64K at same util just trades concurrency, same memory. NOTE: these are TEMPLATE changes; the RUNNING fleet (~/.model-gear) still has the old config -- a live redeploy (lobes init/serve with new template) is a SEPARATE disruptive step to actually run the duo. Recorded docs/vllm-nightly-migration.md section 8. Version 0.35.0.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"topic": "fleet-nightly-unification", "relates_to": "issue-75", "source": "claude-code-session", "date": "2026-07-02"}, "created": "2026-07-01T23:19:56.814616+00:00", "last_recall": "2026-07-03T08:03:33.189858+00:00", "recall_count": 1, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "claudemd-256k-context-stale-fleet-runs-64k", "hash": "d3732d72586287b2de6491267aaef6479142800831a9b486565654656e707132", "content": "CLAUDE.md's '256K native context served at the full 256K' for the Qwen 27B primary is STALE for the FLEET (duo) deployment (verified 2026-07-03, code map for issue #81). The 256K figure (VLLM_MAX_MODEL_LEN=262144) is ONLY the legacy single-model scaffold (lobes/templates/docker-compose.yml + env.example). The always-on fleet duo template (lobes/templates/fleet/) trims the 27B primary to 64K: PRIMARY_MAX_MODEL_LEN=65536 (util 0.30), so the Gemma 4 12B multimodal gear serves its full 128K native (MULTIMODAL_MAX_MODEL_LEN=131072, util 0.22). Live-validated co-resident on the DGX Spark GB10 2026-07-02 (PR #80). Issue #81 REBALANCES this: cortex(Qwen)=128K, senses(Gemma)=32K. Don't trust the 256K line when reasoning about the fleet.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "fleet-context-window", "relates_to": "issue-81", "date": "2026-07-03"}, "created": "2026-07-03T08:22:08.843433+00:00", "last_recall": "2026-07-03T23:54:01.568905+00:00", "recall_count": 1, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "fleet-containers-omits-gemma-multimodal", "hash": "a87d77b4a8f3e681b57f2edbf43d8b83e4b9563f71713d306675ec5f7e17ae09", "content": "GAP (found 2026-07-03, #81 code map): FLEET_CONTAINERS in lobes/runtime/_compose.py:42 = (FLEET_PRIMARY, FLEET_EMBED, FLEET_RERANK, FLEET_GATEWAY) OMITS the Gemma/multimodal container (no FLEET_MULTIMODAL constant), even though vllm-multimodal is default-on in the duo. So 'lobes fleet status' (fleet.py iterates _compose.fleet_containers()) does NOT report model-gear-vllm-multimodal's container state at all. Making 'senses' a first-class discoverable role (#81) REQUIRES adding this container to FLEET_CONTAINERS. Captured as decision c26 in docs/specs/2026-07-03-lobes-exposes-the-full-colleague-runtime-stack-as.md.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "claude-code-session", "topic": "fleet-status-gap", "relates_to": "issue-81", "date": "2026-07-03"}, "created": "2026-07-03T08:22:08.845251+00:00", "last_recall": "2026-07-03T23:37:43.106121+00:00", "recall_count": 2, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "issue-81-cortex-senses-roles-specced", "hash": "37fc19bb17bdcac85e4d013d2815fafeb6d8a5bbe624f01c592feb3aedaed6de", "content": "Issue #81 (expose full Colleague stack as cortex/senses role-based lobes) specced via /think 2026-07-03 -> docs/specs/2026-07-03-lobes-exposes-the-full-colleague-runtime-stack-as.md. USER-confirmed FULL-#81 scope: six first-class roles cortex/senses/stt/tts/embedder/reranker; REBALANCE cortex(Qwen3.6-27B-MTP)=128K (up from fleet's 64K), senses(Gemma4-12B)=32K (down from 128K), util retuned, pressure degrades cortex->minor (senses is a distinct capability not a rung); 'lobes capabilities'/'lobes endpoint ' + gateway GET /capabilities JSON contract for Colleague; 'lobes up ' + 'colleague-stack' profile; per-role RUNTIME metrics only (Colleague owns task-quality); cortex-only-vs-cortex+senses bench profiles. KEY DECISIONS: (c24) ROLE LAYER not rename -- cortex->primary, senses->multimodal via catalog.TIER_ROLE + mirrors (_pressure_policy._TIER_ROLE, _tier_request); internal service/env/container names (vllm-primary/vllm-multimodal, PRIMARY_*/MULTIMODAL_*) NOT renamed; main|multimodal|hard|normal kept as back-compat aliases. (c25) contract transport = CLI 'lobes capabilities --json' AND gateway GET /capabilities. (c26) fix FLEET_CONTAINERS gap. 'brain' FORBIDDEN as role name/alias. Parked (nonblocking): exact utils, canonical responsibilities word-lists, 'lobes up' verb shape, whether colleague-stack bundles audio overlay. Next leg: /spec-to-plan.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "cortex-senses-roles", "relates_to": "issue-81", "date": "2026-07-03"}, "created": "2026-07-03T08:22:08.845819+00:00", "last_recall": "2026-07-03T23:54:01.568905+00:00", "recall_count": 2, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} -{"id": "pr-90-fleet-reachability-truth-review-reflex", "hash": "91a2aafd09c1fb6859b760c68c8417e2866db446a86a7edc7c3c87dadb80c26c", "content": "PR #90 (agentculture/lobes-cli, 0.38.0, 2026-07-04) shipped fleet reachability truth as ONE PR fixing #84/#87/#89: (#84) lobes status fleet-aware via _compose.is_fleet() Dockerfile.gateway marker, single-model output byte-identical; (#87) GET /capabilities + every role endpoint advertises a client-reachable origin from the request Host header with a GATEWAY_PUBLIC_URL override for tunnels; (#89) stt/tts ready is a LIVE probe of the realtime bridge GET /v1/health/ready aggregate, gateway returns 503-warming vs 502-unreachable for /v1/audio/*, Chatterbox reports honest 503 cuda_context_poisoned. Built via the devague pipeline (/think spec -> /spec-to-plan 6 tasks/3 waves decomposed by FILE for disjointness -> /assign-to-workforce) with colleague local-vLLM as the wave-0 workforce and opus for the gateway integration. LESSON \u2014 the diverse-review reflex earns its keep: the ask-colleague review AND the Qodo agentic review each independently caught real semantic bugs in the SAME audio loaded/ready area. colleague caught loaded/ready conflation (a warming backend read as not-deployed). Qodo caught two more: probe_audio_ready caught only OSError so a malformed AUDIO_URL (non-numeric port -> urlsplit(...).port raises ValueError) crashed the handler uncaught; and build_role_registry did not clamp ready on audio_configured, so a caller passing audio_ready=True with AUDIO_URL unset got ready=True/loaded=False/endpoint=empty. Fix patterns: catch ValueError in URL-parsing probes exactly like open_upstream does; make public builders self-enforce the invariants their own docstrings promise. Run BOTH review passes on a committed diff before merge.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "pr-90-fleet-reachability", "relates_to": "issues-84-87-89", "date": "2026-07-04"}, "created": "2026-07-04T04:03:43.104710+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "issue-81-cortex-senses-roles-specced", "hash": "37fc19bb17bdcac85e4d013d2815fafeb6d8a5bbe624f01c592feb3aedaed6de", "content": "Issue #81 (expose full Colleague stack as cortex/senses role-based lobes) specced via /think 2026-07-03 -> docs/specs/2026-07-03-lobes-exposes-the-full-colleague-runtime-stack-as.md. USER-confirmed FULL-#81 scope: six first-class roles cortex/senses/stt/tts/embedder/reranker; REBALANCE cortex(Qwen3.6-27B-MTP)=128K (up from fleet's 64K), senses(Gemma4-12B)=32K (down from 128K), util retuned, pressure degrades cortex->minor (senses is a distinct capability not a rung); 'lobes capabilities'/'lobes endpoint ' + gateway GET /capabilities JSON contract for Colleague; 'lobes up ' + 'colleague-stack' profile; per-role RUNTIME metrics only (Colleague owns task-quality); cortex-only-vs-cortex+senses bench profiles. KEY DECISIONS: (c24) ROLE LAYER not rename -- cortex->primary, senses->multimodal via catalog.TIER_ROLE + mirrors (_pressure_policy._TIER_ROLE, _tier_request); internal service/env/container names (vllm-primary/vllm-multimodal, PRIMARY_*/MULTIMODAL_*) NOT renamed; main|multimodal|hard|normal kept as back-compat aliases. (c25) contract transport = CLI 'lobes capabilities --json' AND gateway GET /capabilities. (c26) fix FLEET_CONTAINERS gap. 'brain' FORBIDDEN as role name/alias. Parked (nonblocking): exact utils, canonical responsibilities word-lists, 'lobes up' verb shape, whether colleague-stack bundles audio overlay. Next leg: /spec-to-plan.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "cortex-senses-roles", "relates_to": "issue-81", "date": "2026-07-03"}, "created": "2026-07-03T08:22:08.845819+00:00", "last_recall": "2026-07-09T05:13:17.278668+00:00", "recall_count": 3, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "pr-90-fleet-reachability-truth-review-reflex", "hash": "91a2aafd09c1fb6859b760c68c8417e2866db446a86a7edc7c3c87dadb80c26c", "content": "PR #90 (agentculture/lobes-cli, 0.38.0, 2026-07-04) shipped fleet reachability truth as ONE PR fixing #84/#87/#89: (#84) lobes status fleet-aware via _compose.is_fleet() Dockerfile.gateway marker, single-model output byte-identical; (#87) GET /capabilities + every role endpoint advertises a client-reachable origin from the request Host header with a GATEWAY_PUBLIC_URL override for tunnels; (#89) stt/tts ready is a LIVE probe of the realtime bridge GET /v1/health/ready aggregate, gateway returns 503-warming vs 502-unreachable for /v1/audio/*, Chatterbox reports honest 503 cuda_context_poisoned. Built via the devague pipeline (/think spec -> /spec-to-plan 6 tasks/3 waves decomposed by FILE for disjointness -> /assign-to-workforce) with colleague local-vLLM as the wave-0 workforce and opus for the gateway integration. LESSON \u2014 the diverse-review reflex earns its keep: the ask-colleague review AND the Qodo agentic review each independently caught real semantic bugs in the SAME audio loaded/ready area. colleague caught loaded/ready conflation (a warming backend read as not-deployed). Qodo caught two more: probe_audio_ready caught only OSError so a malformed AUDIO_URL (non-numeric port -> urlsplit(...).port raises ValueError) crashed the handler uncaught; and build_role_registry did not clamp ready on audio_configured, so a caller passing audio_ready=True with AUDIO_URL unset got ready=True/loaded=False/endpoint=empty. Fix patterns: catch ValueError in URL-parsing probes exactly like open_upstream does; make public builders self-enforce the invariants their own docstrings promise. Run BOTH review passes on a committed diff before merge.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "project", "record_metadata": {"source": "claude-code-session", "topic": "pr-90-fleet-reachability", "relates_to": "issues-84-87-89", "date": "2026-07-04"}, "created": "2026-07-04T04:03:43.104710+00:00", "last_recall": "2026-07-09T05:13:18.802890+00:00", "recall_count": 2, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "vllm-preserve-thinking-qwen36", "hash": "9aea7c52e41a22b743843280df65110b2d74c238bece4d66fff194a48dac1617", "content": "vLLM preserve_thinking for Qwen3.6 cortex is real and supported: --default-chat-template-kwargs '{\"preserve_thinking\": true}' is a stable vLLM serve flag since 0.9.0 (both pinned images \u2014 nvcr.io/nvidia/vllm:26.04-py3 and the 2026 vllm/vllm-openai nightly \u2014 are newer). preserve_thinking is a genuine variable in the SERVED chat template (cortex serves with --tokenizer=mmangkad/Qwen3.6-27B-NVFP4), gating historical retention via `preserve_thinking is true or loop.index0 > ns.last_query_index`. Default keeps only for turns after the last user query; preserve_thinking=true keeps ALL. Older Qwen3-30B template lacks the var \u2014 it is Qwen3.6-specific. Request-level chat_template_kwargs override the server default. Open risk is the INPUT side: whether vLLM re-injects an assistant-history reasoning field back into the template \u2014 that is what issue #93's token-delta diagnostic proves. Spec: docs/specs/2026-07-07-lobes-preserves-qwen-thinking-traces-across-multi.md (issue #93).", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "issue-93-think", "issue": "93"}, "created": "2026-07-07T05:22:37.832377+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} {"id": "devague-no-resolve-verb-for-blocking-items", "hash": "7ed18e35e93824f581b137e6fd605f7e9193eadd3e201e9d259b4e62f0cc4b29", "content": "devague (/think) tooling gap: there is NO verb to resolve a blocking hard question or an unknown_blocking vagueness. `reject` only accepts claim (c*) / honesty (h*) ids \u2014 not q* hard-questions or v* vagueness. `question --resolve` only handles standalone question-move items, not hard questions on claims. Re-parking a vagueness spawns a NEW v2, it does not flip the original. The convergence gate blocks ONLY on hard_questions with blocking:true and open_vagueness kind unknown_blocking; non-blocking ones ship as caveats in the exported spec. Practical rule: record hard questions/risks as NON-blocking from the start, or hand-edit .devague/frames/.json to set blocking:false / kind:unknown_nonblocking to converge.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "reference", "record_metadata": {"source": "issue-93-think"}, "created": "2026-07-07T05:22:37.836620+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "lobes-advertised-implies-reachable-2026-07-09", "hash": "4095aa5c41e5be9ecc1273f6a11bbd8de1161d6e9d81974e75f156fdb7242d9c", "content": "lobes-cli 2026-07-09 live-rig investigation unifying #92/#91/#95/#74/#69 into ONE spec (docs/specs/2026-07-09-lobes-never-advertises-a-capability-it-cannot-serv.md, branch spec/advertised-implies-reachable-92-91-74-69). THE ISSUES NAMED THE SYMPTOMS RIGHT AND THE CAUSES WRONG. (1) #92 is NOT a regression of #87: reachable_origin shipped in 0.38.0/PR#90 (merged 2026-07-04), but the deployed gateway image was built 2026-07-03 carrying lobes 0.36.0 (verified: docker exec model-gear-gateway python -c \"import lobes; lobes.__version__\" -> 0.36.0; hasattr(server,\"reachable_origin\") -> False). Pre-fix code falls back to _gateway_base_url() = the gateway INTERNAL container port :8000, while the published host port is VLLM_PORT=8001. NOTHING DETECTS GATEWAY CONTAINER VERSION SKEW \u2014 that absence is the real defect. Extra hazard: host :8000 is NOT dead, it is reachy-mini-dae (an unrelated uvicorn app) answering {\"detail\":\"Not Found\"}, so the advertised endpoint points at a foreign live service. (2) #91 is NOT a backend-reload window: handle_post calls rewrite_model ONCE before the failover loop, and order_backends returns EVERY same-task generate backend as a failover candidate \u2014 for the cortex model that is [primary, multimodal, multimodal-coder, middle]. Cortex returns >=500 (vllm.v1.engine.exceptions.EngineDeadError 04:08:34) -> gateway retries the SAME body, still naming the Qwen id, against the Gemma backend -> Gemma correctly 404s NotFoundError -> handle_post rule \"2xx or 4xx -> commit to this backend (4xx is a client error; no failover)\" relays it as a TERMINAL 404. PROOF, not theory: model-gear-vllm-primary (the Qwen container) logs \"The model `coolthor/gemma-4-12B-it-NVFP4A16` does not exist.\" at 04:34:01 and 04:39:01 \u2014 the symmetric direction. tests/test_gateway_routing.py::test_order_backends_generate_still_failovers_between_generate_backends ASSERTS THE BUG AS INTENDED BEHAVIOUR. (3) TWO UNFILED BUGS of the same shape, both found here, now #96 and #97. #96: AUDIO_URL reaches the gateway ONLY via docker-compose.audio.yml:144, never the base fleet template, so ServerConfig.audio_url is empty and POST /v1/audio/speech 404s \"audio endpoints are not configured\" \u2014 yet `lobes capabilities` reads the merged .env (which HAS AUDIO_URL) and reports stt/tts ready=true/loaded=true. #97: _optional_backend wires a backend on *_BASE_URL OR *_SERVED_NAME with a default_url naming a container that need not exist -> /v1/models lists 6 models against 4 running containers. (4) KEY INSIGHT: NEITHER contract surface is authoritative and they are wrong in OPPOSITE directions \u2014 for the generate roles the gateway JSON is wrong and the CLI is right; for the audio roles the CLI is wrong and the gateway is right. Both derive readiness from CONFIGURATION and neither dials anything. build_role_registry is genuinely one builder, but it is fed two different gateway_url values (CLI: .env VLLM_PORT; gateway: its own GATEWAY_PORT), so \"one source of truth\" holds for the SHAPE and fails for the ORIGIN. (5) EVIDENCE GATHERED: coolthor/gemma-4-12B-it-NVFP4A16 genuinely PERCEIVES images \u2014 a stdlib-generated solid-red PNG -> \"Red\", solid-blue -> \"Blue\" (ground-truth checked). tests/test_smoke_duo.py:287-347 only asserts HTTP 200 + non-empty content on a 1x1 placeholder, i.e. it proves the WIRE not the PERCEPTION; \"image+text confirmed\" in the docs was that weaker claim. Audio perception could NOT be proven because generating real speech needs /v1/audio/speech, which is bug #96. (6) USER DECISIONS in the frame: NO cross-backend failover at all (one backend per request; dead owner -> 503 + Retry-After; order_backends collapses to a single-element list) \u2014 stricter than model-exact-with-tier-rewrite, chosen because a caller asking for cortex must never silently receive Gemma; readiness becomes a BACKGROUND cached probe mirroring PressureCache; phantom backends stopped by BOTH a config gate (require *_BASE_URL) and the readiness filter; #69 DSpark disabled-entry criterion closed ANSWERED-NEGATIVE (#75 proved Gemma4DSparkModel does not load on vLLM 0.23) with docs/gemma4-mtp-draft.md to be corrected; EngineDeadError root cause OUT of scope -> filed as #98. (7) NEW REQUIREMENT from user: a LOCAL, single-trigger, UNATTENDED pre-PR live test of the capabilities contract (not CI \u2014 no GPU there), which must FAIL not skip. It must also compare the deployed gateway container lobes.__version__ against the CLI wheel. (8) TOOL GAP: devague has NO move to un-park a blocking vagueness \u2014 once parked as unknown_blocking, converge can never pass. Had to hand-edit .devague/frames/.json to re-kind v3 to follow_up. Worth an upstream issue on agentculture/devague. (9) #69 audit: 6/8 acceptance criteria DONE; README.md:52-68 still documents `lobes init` as scaffolding the SINGLE-model deployment though fleet is the default since e51ffc8 (stale, contradicts init.py:106-111). #74 audit: the coolthor default switched in 0.35.0 justified by MTP decode throughput, NOT by the image/audio precondition #74 itself set.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "investigation", "record_metadata": {"source": "claude-code-session", "topic": "advertised-implies-reachable", "relates_to": "issues-92-91-95-74-69-96-97-98", "date": "2026-07-09"}, "created": "2026-07-09T05:37:04.254050+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "gateway-public-url-must-not-default-2026-07-09", "hash": "b00294589b18a9dcf624e32a58ee3c93699fa5c87323463f7b86d30e100d62d0", "content": "lobes-cli gateway origin advertising: NEVER default GATEWAY_PUBLIC_URL to a localhost URL in the compose template. reachable_origin(host_header, public_url) checks `if public_url: return public_url` BEFORE `if host_header: ...`, so a defaulted public_url (e.g. GATEWAY_PUBLIC_URL=${GATEWAY_PUBLIC_URL:-http://localhost:${VLLM_PORT:-8000}}) is ALWAYS set and the Host header is never consulted. A LAN or tunnel client then GETs /capabilities and is told to dial http://localhost:8001 \u2014 which on ITS machine is a different service. That is the exact #92 defect (advertised endpoint points at a foreign daemon) reintroduced by the fix for it. Correct precedence: explicit operator override (GATEWAY_PUBLIC_URL, for a tunnel / Host-rewriting reverse proxy) > the origin the client actually dialed (Host header) > NOTHING (empty endpoint). Never fabricate an absolute URL from the internal GATEWAY_PORT. This was caught only because the spec claim (c11) was tested against a simulated LAN Host header before merging the subagent's faithful implementation of it \u2014 the REQUIREMENT was wrong, not the code. Amended to c29 in the frame lobes-never-advertises-a-capability-it-cannot-serv. LESSON: when a spec says 'configured truth beats inference', check whether the 'configured truth' is actually per-deployment truth or per-CALLER truth. An origin is per-caller.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "gotcha", "record_metadata": {"source": "claude-code-session", "topic": "gateway-origin-advertising", "relates_to": "issues-92-96", "date": "2026-07-09"}, "created": "2026-07-09T07:07:45.762815+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "devague-cli-gaps-2026-07-09", "hash": "6493960c999eb7f5248596d5c45c83c2daaed559149de80320a916a4de6cd53a", "content": "devague CLI gaps hit while running /think -> /spec-to-plan -> /assign-to-workforce on lobes-cli (2026-07-09), worth upstream issues on agentculture/devague: (1) NO MOVE TO UN-PARK A BLOCKING VAGUENESS. Once `devague park \"...\" --kind unknown_blocking` is recorded, converge can never pass \u2014 `confirm`/`reject` accept only claim ids (c*) and honesty ids (h*), not vagueness ids (v*). Workaround: hand-edit .devague/frames/.json to re-kind the item to follow_up. A decided unknown is no longer vagueness, so there should be a `resolve` move. (2) `devague plan confirm` takes ONE task id, unlike `devague confirm` which is transactional over many ids \u2014 `plan confirm t1 t2 t3` errors with 'unrecognized arguments'. Loop instead. (3) After amending the source frame (reject a claim, capture a replacement), `devague plan cover --target ` FAILS with 'run devague plan show to see targets' until you first run `devague plan status` (or converge), which re-derives and persists the plan's targets from the live frame. Run `plan status` immediately after any frame amendment. (4) The plan JSON key is `targets`, not `coverage_targets`. Also: `devague plan converge` correctly refuses when the source frame has regressed below convergence \u2014 re-converge the frame first.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "gotcha", "record_metadata": {"source": "claude-code-session", "topic": "devague-cli", "relates_to": "think-spec-to-plan-workforce", "date": "2026-07-09"}, "created": "2026-07-09T07:07:45.847201+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "senses-gemma-cannot-hear-2026-07-09", "hash": "93dee119b623b8d35a77422ef56604c09119e3fb72196f1433f91c04a8514dbf", "content": "lobes `senses` role (coolthor/gemma-4-12B-it-NVFP4A16 on vLLM 0.23.1rc1.dev672+g93d8f834d) CANNOT HEAR \u2014 vLLM silently DROPS `input_audio` OpenAI content parts (filed as issue #101). Proof by prompt-token accounting via model=multimodal: text-only = 15 prompt_tokens; text+image (96x96 solid PNG) = 273 (+258, and the model correctly answers \"Red\"/\"Blue\"); text+audio (0.68s WAV @24kHz from the rig's own Chatterbox) = 34 (+19 placeholder tokens) with empty content; same clip resampled to 16kHz = 48 tokens and the model replies \"I cannot hear any audio because you haven't provided a file or a link\". Sample rate is NOT the cause. The checkpoint is fine: AutoConfig shows audio_config + audio_token_id + vision_config. So vLLM's gemma4_unified path wires the vision encoder and not the audio encoder, and DROPS rather than REJECTS the audio part \u2014 caller gets HTTP 200 and a fluent answer that ignored the audio. HOW IT WENT UNNOTICED FOR SO LONG: tests/test_smoke_duo.py's live layer asserted only `assert content.strip()` (HTTP 200 + non-empty) against a 1x1 placeholder PNG and a tiny WAV. That proves the WIRE, not the PERCEPTION. The docs' \"image+text \u2713 / audio+text \u2713\" for the sakamakismile coder checkpoint rest on the same weak check. METHOD THAT WORKS: generate a solid-colour PNG with stdlib zlib+struct, ask \"What single colour fills this image? Answer with one word.\", assert the colour name appears; ALWAYS run a negative control (feed blue, assert \"red\" fails) or the test is vacuous. For audio, synthesize a known word with the rig's own Chatterbox TTS (POST /v1/audio/speech) and assert the transcription contains it. The audio probe is now @pytest.mark.xfail(strict=True) citing #101, so it XPASSes and fails the suite the day audio starts working. NOTE the `stt` role (Parakeet, POST /v1/audio/transcriptions) is a separate purpose-built path and WORKS: tts(\"banana\") -> stt -> \"Banana.\"; tts(\"the quick brown fox\") -> \"The Quick Brown Fox\".", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "investigation", "record_metadata": {"source": "claude-code-session", "topic": "senses-audio", "relates_to": "issues-101-74", "date": "2026-07-09"}, "created": "2026-07-09T09:38:15.311269+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "lobes-pressure-swap-occupancy-sticky-2026-07-09", "hash": "7f666a9ea25527288cadc52f8b87e5499164ea353ffed40d2e3eda9aa7868e6b", "content": "lobes pressure policy sheds 100% of generate traffic FOREVER on a box whose swap has ever filled (filed as issue #100). lobes/runtime/_pressure.py computes swap_used_percent = (SwapTotal - SwapFree)/SwapTotal from /proc/meminfo \u2014 an OCCUPANCY metric. Swap occupancy is STICKY: the kernel does not reclaim swapped pages just because RAM freed, so once it crosses the 75% default threshold it stays there until swapoff/swapon. On the DGX Spark GB10 (2026-07-09): swap 16322/16383 MB = 99.6% occupancy, BUT `vmstat` si=0 so=0 (nothing being swapped) and /proc/pressure/memory shows `full avg10=0.00 avg60=0.00` (ZERO memory stall), with 26 GB RAM available. Since #88 (0.37.0) the policy SHEDS with 429 server_busy instead of degrading, so every cortex/senses request 429s while `X-Lobes-Override: 1` still returns 200. The rig had been fine on gateway 0.36.0 (pre-shed) and broke the instant it was upgraded to 0.39.0 \u2014 the wire had been armed for weeks. WORKAROUND (documented, but it DISABLES the policy rather than fixing it): set LOBES_SWAP_DEGRADED_THRESHOLD=100 and LOBES_IOWAIT_DEGRADED_THRESHOLD=100 in the deployment .env and recreate the gateway. Note LOBES_IOWAIT_DEGRADED_THRESHOLD=100 was ALREADY set by a previous operator for the same class of reason (0.36.2 CHANGELOG documents \"phantom high iowait on an idle disk, e.g. the DGX Spark GB10\") \u2014 half the policy had already been switched off, which is the tell that the METRIC is wrong, not the threshold. RIGHT FIX: read PSI (/proc/pressure/memory, /proc/pressure/io avg10) or swap RATE (pswpin/pswpout deltas from /proc/vmstat), not occupancy; keep occupancy at most as a secondary AND-condition. Also: a gateway that has shed 100% of requests for an hour is misconfigured, not busy \u2014 nothing currently warns.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "gotcha", "record_metadata": {"source": "claude-code-session", "topic": "pressure-policy", "relates_to": "issues-100-85-86-88", "date": "2026-07-09"}, "created": "2026-07-09T09:38:15.369909+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} +{"id": "lobes-fleet-frozen-image-pin-2026-07-09", "hash": "08f35f73d1ad1f81fc3dd0a6d348a2d8f7f6c2ef4581caa92dfba6c7a56b14c5", "content": "lobes-cli fleet deployments FREEZE their lobes version and nothing bumps it (issue #99). Dockerfile.gateway/realtime/stt/chatterbox all run: pip install --no-cache-dir lobes-cli==${MODEL_GEAR_VERSION}. `lobes init` writes MODEL_GEAR_VERSION into .env ONCE at scaffold time and no verb ever rewrites it, so `docker compose up -d --build gateway` faithfully reinstalls the OLD pinned version. On 2026-07-09 the reference rig ran THREE lobes versions at once: gateway 0.36.0 (image built 07-03), realtime 0.34.1 (07-02), stt/chatterbox 0.34.1-era (07-02) \u2014 against a CLI at 0.39.0, with 0.36.1/0.36.2/0.37.0/0.38.0/0.39.0 ALL published on PyPI. CONSEQUENCE: three bugs were reported against lobes CODE that were merely undeployed fixes. (a) #92 'advertised endpoint :8000 404s' \u2014 reachable_origin shipped in 0.38.0/PR#90 on 07-04; the gateway image predated it by one day, so it fell back to _gateway_base_url() = its INTERNAL container port 8000 while the published host port is 8001. Worse, host :8000 was an unrelated uvicorn service (reachy-mini-dae), so the advertised endpoint pointed at a foreign daemon. (b) the stt sidecar answered /v1/health/ready with 200 status=ready while its CUDA context was poisoned and transcription 500d \u2014 #89 (07-04) had replaced the old liveness-only handler with a model_loaded+cuda_ok probe, but the stt image was built 07-02. (c) chatterbox likewise. CHEAP FIX (no model reload): edit ~/.lobes/.env MODEL_GEAR_VERSION=, then from ~/.lobes run: docker compose -f docker-compose.yml -f docker-compose.audio.yml up -d --build --no-deps gateway realtime stt chatterbox. GOTCHA 1: the compose PROJECT is named lobes (from the dir basename) while containers are named model-gear-* via container_name \u2014 check the com.docker.compose.project label before invoking compose or you create a duplicate project. GOTCHA 2: AUDIO_URL reaches the gateway ONLY via docker-compose.audio.yml, so a gateway recreated without the -f audio overlay silently loses the audio lane (issue #96). GOTCHA 3: a from-source checkout can be AHEAD of PyPI, in which case the gateway image cannot be built from the pin at all \u2014 the dev path needs a locally built wheel COPYd in.", "scope": {"name": "lobes", "visibility": "public"}, "metadata": {"type": "gotcha", "record_metadata": {"source": "claude-code-session", "topic": "fleet-version-pin", "relates_to": "issues-99-92-89-96", "date": "2026-07-09"}, "created": "2026-07-09T09:38:41.844728+00:00", "last_recall": null, "recall_count": 0, "links": [], "supersedes": null, "lifecycle": "active", "added_by": null}} diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bf9e6e..2e16dec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,36 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.40.0] - 2026-07-09 + +### Added + +- `scripts/live-check.sh` + `tests/test_live_capabilities.py` — a LOCAL, single-trigger, unattended pre-PR gate that dials every advertised role endpoint+path, every id in `/v1/models`, checks CLI/gateway agreement, reproduces Colleague's role-discovery path, and fails on deployed-gateway version skew. It FAILS rather than skips when armed. A 429 (pressure shed) and a 503 carrying `Retry-After` count as reachable; only a 404, a connection failure, a `Retry-After`-less 503, or a bare 5xx are faults. +- `lobes/gateway/_readiness.py` — a bounded background probe of each backend's `/health`, mirroring `PressureCache`. Tri-state (`True` healthy / `False` reached-but-unhealthy / `None` unreachable), daemon thread, socket-free `.current()`, and a probe that degrades to `None` on `OSError`, `http.client.HTTPException` and `ValueError`. +- `GET /health` now reports `{"version": ...}`, and `lobes doctor` gains a `gateway_version_match` check that fails on skew between the deployed gateway and the CLI wheel (issue #99). +- `lobes/gateway/_routing.py::is_unknown_model` — a pure predicate separating "unknown model id" from "unspecified model". +- Ground-truth perception probes for the `senses` role: a stdlib-generated solid-colour PNG whose colour the model must name, and a Chatterbox-synthesized word the model must transcribe. Both carry negative controls; the old placeholder-media tests are relabelled as wire checks. + +### Changed + +- **No cross-backend failover.** `order_backends` now returns at most one backend. A request naming the cortex model can never be answered by the Gemma backend, which protects the `final_authority` role contract from #81. +- `GET /v1/models` and `GET /capabilities.ready` are backed by the live readiness cache rather than by configuration. A wired-but-dead backend is no longer advertised. +- `RoleInfo.ready` is no longer an alias of `loaded` for `cortex`/`senses`/`embedder`/`reranker`. `build_role_registry` self-enforces the invariant: a supplied `backend_ready` map is authoritative, and a present `None`, a present `False`, and a missing key all mean not-ready. +- `lobes capabilities` / `lobes endpoint` now render the running gateway's `GET /capabilities` when it is reachable, falling back to an offline `.env`-derived view tagged `"source": "offline"` with `ready=false` on every role. The CLI and the gateway can no longer disagree, because there is now one derivation instead of two. +- A backend is wired only when its `*_BASE_URL` is set; the `or *_SERVED_NAME` clause is gone (issue #97). +- `senses` is documented as vision-only intake. The checkpoint declares audio support but vLLM's `gemma4_unified` path does not serve it (issue #101); `stt` remains the supported speech path. +- `docs/gemma4-mtp-draft.md` carries a superseded banner: DSpark does not load on vLLM 0.23 (#75). Issue #69's disabled-experiment-entry criterion is closed answered-negative. +- README's quickstart no longer claims `lobes init` scaffolds the single-model deployment; the fleet duo has been the default since #69. + +### Fixed + +- **#91** — a dead cortex backend no longer surfaces as a terminal `404 model does not exist`. `handle_post` rewrote the model id once, before the failover loop, then retried the same body against a backend serving a different model, which correctly 404'd; the `4xx = client error, no failover` rule relayed that as terminal and killed multi-step agent loops. A dead, unreachable or warming owner now yields **503 + `Retry-After`** with `type: backend_unavailable`. +- **#92 / #95** — the gateway never advertises an origin built from its internal listen port. Precedence is `GATEWAY_PUBLIC_URL` (an operator override for a tunnel or Host-rewriting proxy) > the request `Host` header > an empty endpoint. `GATEWAY_PUBLIC_URL` is deliberately NOT defaulted: a defaulted `public_url` outranks `Host` and would advertise loopback to every remote client. +- **#96** — `AUDIO_URL` now reaches the gateway from the base fleet compose, so `stt`/`tts` stop advertising `ready=true` on a path that 404s when the audio overlay is not composed in. +- **#97** — `GET /v1/models` no longer advertises phantom backends wired from a `*_SERVED_NAME` alone against a `default_url` naming a container that need not exist. +- An unknown model id returns `404 model_not_found` instead of being silently served by the default backend under a different model's weights. Unknown-ness is decided against the routing table, never against the readiness-filtered `/v1/models` list, so a wired-but-dead backend still yields 503 rather than 404. +- `test_live_main_text_returns_nonempty_content` no longer fails on a thinking model: `max_tokens=16` was consumed entirely by the reasoning trace, leaving `content=None` and `finish_reason=length` (issue #93). + ## [0.39.0] - 2026-07-07 ### Added diff --git a/pyproject.toml b/pyproject.toml index bcd4b06..55b325b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "lobes-cli" -version = "0.39.0" +version = "0.40.0" description = "lobes — run, assess, and switch the local vLLM model." readme = "README.md" license = "Apache-2.0" diff --git a/uv.lock b/uv.lock index 79d8fce..9acbd78 100644 --- a/uv.lock +++ b/uv.lock @@ -440,7 +440,7 @@ wheels = [ [[package]] name = "lobes-cli" -version = "0.39.0" +version = "0.40.0" source = { editable = "." } [package.optional-dependencies] From d8884e70a7f0fad8078a4844aa94dcedfca1b32b Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Thu, 9 Jul 2026 13:59:54 +0300 Subject: [PATCH 18/23] fix: validate Host header before reflecting it in /capabilities (S5131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit reachable_origin echoed the request Host header into every role's endpoint field unsanitised — a client-controlled value (path traversal, script markup, userinfo-style credential injection like 127.0.0.1:8001@attacker.test) reflected straight into the /capabilities JSON body, flagged as a SonarCloud BLOCKER (pythonsecurity:S5131). Add a strict host-authority allowlist regex (hostname/IPv4 or bracketed IPv6, optional :port) and gate the Host-header echo behind it. A Host that fails validation degrades to None, same as no Host header at all, so the endpoint comes back empty rather than reflecting attacker input. GATEWAY_PUBLIC_URL (trusted operator config) is untouched and keeps winning first. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01MrghbUdRuVa2Y98EV8RHQg --- lobes/gateway/server.py | 52 ++++++++++++++++++++++- tests/test_gateway_capabilities.py | 68 ++++++++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 1 deletion(-) diff --git a/lobes/gateway/server.py b/lobes/gateway/server.py index f1405de..2eb4072 100644 --- a/lobes/gateway/server.py +++ b/lobes/gateway/server.py @@ -38,6 +38,7 @@ import http.client import json import os +import re import sys from collections.abc import Mapping from concurrent.futures import ThreadPoolExecutor @@ -685,6 +686,32 @@ def fleet_status_payload( # its config-derived defaults so the CLI/unit path is unchanged. +# A legitimate HTTP ``Host`` header is a bare authority: a DNS hostname or IPv4 +# literal (dot-separated alphanumeric/hyphen labels — RFC 1123, which an IPv4 +# literal's digit-only labels already satisfy) or a bracketed IPv6 literal, +# each optionally followed by ``:`` (1-5 digits). Nothing in that grammar +# permits ``/``, ``@``, whitespace, control characters, ``<``/``>``, ``?``, +# ``#``, or backslashes, so a single allowlist regex both recognises a +# well-formed host AND excludes every character class a path-traversal, +# userinfo-credential-injection, header-injection (CRLF), XSS, or +# query-string payload needs. See :func:`reachable_origin` for why this +# exists (SonarCloud S5131). +_HOST_LABEL = r"[A-Za-z0-9](?:[A-Za-z0-9-]{0,61}[A-Za-z0-9])?" +_HOSTNAME = rf"{_HOST_LABEL}(?:\.{_HOST_LABEL})*" +_IPV6_LITERAL = r"\[[0-9A-Fa-f:]+\]" +_PORT = r"(?::[0-9]{1,5})?" +_VALID_HOST_HEADER_RE = re.compile(rf"(?:{_HOSTNAME}|{_IPV6_LITERAL}){_PORT}") + + +def _is_valid_host_header(host: str) -> bool: + """True when ``host`` is a well-formed ``hostname[:port]`` authority. + + Used to gate :func:`reachable_origin`'s Host-header echo — see there for + the reflection risk this guards against. + """ + return _VALID_HOST_HEADER_RE.fullmatch(host) is not None + + def reachable_origin( host_header: str | None, public_url: str | None, scheme: str = "http" ) -> str | None: @@ -696,10 +723,33 @@ def reachable_origin( ``host:port`` in the right shape, IPv6 brackets included). Returns ``None`` when neither is available, so the caller falls back to the config-derived origin (unchanged behaviour). + + ``public_url`` is trusted operator config (set via the deployment's own + ``.env``, never attacker-reachable) so it is never validated and always + wins first, unchanged — that precedence is #92 target c29/h25 and is + covered by ``test_reachable_origin_public_url_wins_over_host`` / + ``test_capabilities_public_url_wins_over_host_end_to_end``. + + ``host_header``, by contrast, is fully attacker-controlled: any client can + set an arbitrary ``Host:`` value, and this function's return value is + reflected verbatim into every role's ``endpoint`` in the JSON response + (:func:`capabilities_payload`). Echoing it unsanitised is exactly + SonarCloud rule ``pythonsecurity:S5131`` ("Change this code to not reflect + unsanitized user-controlled data") — a scraping client could be handed an + attacker's origin to dial, or a payload (path traversal, script markup, a + userinfo-style credential-injection host like + ``127.0.0.1:8001@attacker.test``) smuggled through an otherwise-trusted + contract. The remediation is the standard S5131 fix: constrain the tainted + value to a strict allowlist (:func:`_is_valid_host_header`, a bare + ``hostname[:port]``/``[ipv6][:port]`` authority) before it can reach the + response. A ``Host`` header that fails validation is treated exactly like + a missing one — it falls through to ``None``, and the caller advertises an + empty endpoint (never a fabricated or attacker-supplied one) rather than + guessing at a "sanitised" rewrite of untrusted input. """ if public_url: return public_url.rstrip("/") - if host_header: + if host_header and _is_valid_host_header(host_header): return f"{scheme}://{host_header}" return None diff --git a/tests/test_gateway_capabilities.py b/tests/test_gateway_capabilities.py index 5069243..63a844f 100644 --- a/tests/test_gateway_capabilities.py +++ b/tests/test_gateway_capabilities.py @@ -333,6 +333,74 @@ def test_capabilities_public_url_wins_over_host_end_to_end() -> None: assert payload[role]["endpoint"] == "https://tunnel.example" +# --- S5131: the Host header is attacker-controlled and must be validated +# before it is reflected into /capabilities (SonarCloud BLOCKER, PR #102) ------ + + +@pytest.mark.parametrize( + "host", + [ + "localhost:8001", + "gw.example", + "10.0.0.5:8000", + "[::1]:8000", + "spark.local", + ], +) +def test_reachable_origin_accepts_well_formed_hosts(host: str) -> None: + # A well-formed authority (hostname or IPv4/IPv6 literal, optional port) is + # echoed unchanged, prefixed with the scheme. + assert S.reachable_origin(host, None) == f"http://{host}" + + +@pytest.mark.parametrize( + "host", + [ + "evil.example/../