From d2722c57d62b5aa57b4334ce44c5a8d3fbbf1049 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Fri, 12 Jun 2026 12:28:14 +0300 Subject: [PATCH 1/8] spec: model-gear owns the audio I/O surface (devague /think) Work issues #39 (Parakeet STT 500s, CUDA-unknown) and #40 (deployed :8080 facade missing /v1/audio/transcriptions) backwards into one converged spec: model-gear takes ownership of the audio stack, wiring Parakeet STT + Magpie TTS + the vendored realtime facade into its compose template and `model fleet up`. Decisions captured: full-stack ownership (both /v1/audio/* routes), cheap model-ready Parakeet healthcheck, restart-clears-stale-CUDA-context runbook. Unblocks reachy-mini-cli's "hey reachy" wake-word. Includes the converged spec + the .devague frame state (evidence trail). Co-Authored-By: Claude Opus 4.8 (1M context) --- .devague/current | 1 + ...ns-the-audio-i-o-surface-end-to-end-m.json | 280 ++++++++++++++++++ .gitignore | 3 + ...owns-the-audio-i-o-surface-end-to-end-m.md | 63 ++++ 4 files changed, 347 insertions(+) create mode 100644 .devague/current create mode 100644 .devague/frames/model-gear-owns-the-audio-i-o-surface-end-to-end-m.json create mode 100644 docs/specs/2026-06-12-model-gear-owns-the-audio-i-o-surface-end-to-end-m.md diff --git a/.devague/current b/.devague/current new file mode 100644 index 0000000..eb803b3 --- /dev/null +++ b/.devague/current @@ -0,0 +1 @@ +model-gear-owns-the-audio-i-o-surface-end-to-end-m diff --git a/.devague/frames/model-gear-owns-the-audio-i-o-surface-end-to-end-m.json b/.devague/frames/model-gear-owns-the-audio-i-o-surface-end-to-end-m.json new file mode 100644 index 0000000..b2605ab --- /dev/null +++ b/.devague/frames/model-gear-owns-the-audio-i-o-surface-end-to-end-m.json @@ -0,0 +1,280 @@ +{ + "slug": "model-gear-owns-the-audio-i-o-surface-end-to-end-m", + "title": "model-gear owns the audio I/O surface end-to-end: `model fleet up` brings up Parakeet STT plus the OpenAI-compatible `/v1/audio/transcriptions` (and `/v1/audio/speech`) facade on :8080 from model-gear's own vendored realtime app and compose template, with a healthcheck that reflects model readiness \u2014 unblocking reachy-mini-cli's 'hey reachy' wake-word (closes #39 + #40).", + "schema_version": 1, + "status": "exported", + "created": "2026-06-12T09:18:04Z", + "updated": "2026-06-12T09:24:54Z", + "claims": [ + { + "id": "c1", + "kind": "announcement", + "text": "model-gear owns the audio I/O surface end-to-end: `model fleet up` brings up Parakeet STT plus the OpenAI-compatible `/v1/audio/transcriptions` (and `/v1/audio/speech`) facade on :8080 from model-gear's own vendored realtime app and compose template, with a healthcheck that reflects model readiness \u2014 unblocking reachy-mini-cli's 'hey reachy' wake-word (closes #39 + #40).", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h5", + "text": "After 'model fleet up', :8080 serves the REST audio routes from model-gear's own vendored app (not the sibling build), verified by /openapi.json + a 200 transcription.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c2", + "kind": "audience", + "text": "reachy-mini-cli (the wake-word client) and any client wanting the documented OpenAI /v1/audio/* surface behind :8080; secondarily model-gear operators running 'model fleet up'", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h6", + "text": "reachy-mini-cli can point REACHY_STT_URL at the :8080 facade (or :9002) and get a 'hey reachy' transcription back.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c3", + "kind": "before_state", + "text": "The live :8080 facade is the realtime-api sibling build exposing only / , /health, /v1/realtime (WS) \u2014 404 on REST /v1/audio/transcriptions. Parakeet (:9002) 500s on every transcription with CUDA error: unknown error after long uptime on the contended GB10, while its healthcheck still reports 'healthy'. model-gear's vendored facade has the REST routes but is not in its compose template, so it never reaches the box.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h7", + "text": "The drift is real and reproducible today: live :8080 /openapi.json shows only / and /health; :9002 transcription 500s with CUDA-unknown.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c4", + "kind": "after_state", + "text": "model-gear's compose template + 'model fleet up' bring up Parakeet STT and the vendored realtime facade; GET :8080/openapi.json lists /v1/audio/transcriptions and /v1/audio/speech; POST a multipart WAV to :8080/v1/audio/transcriptions forwards to Parakeet and returns the OpenAI-shaped {text:...}; Parakeet's healthcheck exercises a real transcription so 'healthy' means actually-serving.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h8", + "text": "Each after-state assertion is independently checkable by a curl/WAV smoke test documented in the realtime-pipeline doc.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c5", + "kind": "why_it_matters", + "text": "reachy-mini-cli Tier-2 wake-word ('hey reachy') is blocked on a working STT endpoint; today the only working STT is raw Parakeet :9002 (when not 500ing), bypassing the OpenAI-compatible facade. Ending the vendored-vs-deployed drift gives one owned, documented audio surface.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h9", + "text": "With the facade route live and Parakeet serving, reachy-mini-cli's Tier-2 wake-word path stops returning 404/500.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c6", + "kind": "boundary", + "text": "Not building a new STT/TTS engine \u2014 Parakeet (NeMo) and Magpie stay the backends. Not solving GPU contention generally; the #39 scope is restart-to-clear-context + a readiness-reflecting healthcheck (a documented co-residency memory split is explicitly out of scope here). Not changing the /v1/realtime WebSocket protocol. Not making the gateway auth-aware.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h10", + "text": "The spec touches only the REST audio facade, Parakeet serving, and its healthcheck \u2014 it does not modify the /v1/realtime WS protocol, the TTS/STT engines, or gateway auth.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c7", + "kind": "success_signal", + "text": "GET :8080/openapi.json lists /v1/audio/transcriptions and /v1/audio/speech; the issue-#39 repro WAV POST returns 200 with {text:...}; a real 'hey reachy' WAV transcribes to text containing the phrase; Parakeet's Docker healthcheck flips unhealthy when the model can't transcribe; docs state model-gear owns the live :8080 facade and 'model fleet up' brings up the audio routes.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h11", + "text": "Every success signal is a concrete, runnable check (curl, WAV POST, healthcheck flip, doc statement) with a pass/fail outcome.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c8", + "kind": "non_goal", + "text": "Decommissioning the realtime-api sibling project, or migrating its /v1/realtime WebSocket speech-to-speech loop into model-gear, is out of scope \u2014 only the REST audio facade + Parakeet STT move under model-gear ownership.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c9", + "kind": "decision", + "text": "model-gear takes over ownership of the audio stack (Parakeet + vendored facade wired into model-gear's compose template and 'model fleet up'), rather than reconciling the realtime-api sibling in place or re-vendoring upstream.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c10", + "kind": "decision", + "text": "The #39 Parakeet fix depth is restart-to-clear-stale-CUDA-context plus a Docker healthcheck that exercises real model readiness (transcription), not restart-only and not a full documented co-residency memory contract.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c11", + "kind": "requirement", + "text": "model-gear's packaged compose template (model_gear/templates/docker-compose.yml) gains a Parakeet STT service and a realtime-facade service built from the vendored model_gear/realtime app, so 'model init' + 'model fleet up' materialise and start them.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h1", + "text": "'model fleet up --apply' on a clean box brings up parakeet + facade services healthy, and 'model init' materialises them into the deployment dir without manual compose edits.", + "status": "confirmed" + } + ], + "hard_questions": [ + { + "id": "q2", + "text": "risk: Adding parakeet + facade (and possibly magpie) to model-gear's fleet raises the GB10 co-residency memory pressure that #39 implicates \u2014 the very contention that may have wedged Parakeet.", + "resolved": false, + "blocking": false + } + ], + "links": [] + }, + { + "id": "c12", + "kind": "requirement", + "text": "The realtime facade service serves POST /v1/audio/transcriptions (forward to Parakeet) and POST /v1/audio/speech (forward to Magpie) on :8080, and its /openapi.json lists both routes.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h2", + "text": "curl :8080/openapi.json lists both /v1/audio/transcriptions and /v1/audio/speech, and a multipart WAV POST to /v1/audio/transcriptions returns 200 with {text:...} sourced from Parakeet.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c13", + "kind": "requirement", + "text": "The Parakeet container's Docker healthcheck exercises real model readiness (a tiny transcription or equivalent model-ready probe), so the container only reports 'healthy' when it can actually transcribe.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h3", + "text": "When the Parakeet model is loaded and serving, the healthcheck passes; when the model path is broken (e.g. CUDA-unknown 500s), the healthcheck reports unhealthy within its retry window.", + "status": "confirmed" + } + ], + "hard_questions": [ + { + "id": "q1", + "text": "risk: A healthcheck that runs a real transcription is heavier/slower than a liveness probe and could flap or add GPU load; a cheap model-ready probe may be the safer form.", + "resolved": false, + "blocking": false + } + ], + "links": [] + }, + { + "id": "c14", + "kind": "requirement", + "text": "Documentation (a docs/realtime-pipeline.md and/or README 'Audio I/O' section) states that model-gear owns the live :8080 facade, how 'model fleet up' brings up the audio routes, and the runbook for clearing a stale Parakeet CUDA context (restart).", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h4", + "text": "A reader following only the doc can stand up the audio surface and recover a wedged Parakeet without reading source, and the doc names which project owns the live :8080 container.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + }, + { + "id": "c15", + "kind": "decision", + "text": "model-gear's fleet brings up the full audio stack end-to-end: Parakeet STT, Magpie TTS, and the realtime facade \u2014 so both /v1/audio/transcriptions and /v1/audio/speech are owned (adds the NGC_API_KEY dependency and GB10 memory to the fleet budget).", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c16", + "kind": "decision", + "text": "Parakeet's readiness healthcheck is a cheap model-ready probe (confirms the NeMo model is loaded and the CUDA context is live), not a full multipart transcription each interval \u2014 resolving the flap/GPU-load risk q1.", + "origin": "user", + "status": "confirmed", + "honesty_conditions": [], + "hard_questions": [], + "links": [] + }, + { + "id": "c17", + "kind": "requirement", + "text": "model-gear's compose template gains a Magpie TTS service (NGC NIM, NGC_API_KEY from .env) so the facade's /v1/audio/speech route resolves to a fleet-owned backend.", + "origin": "llm", + "status": "confirmed", + "honesty_conditions": [ + { + "id": "h12", + "text": "'model fleet up' brings up Magpie healthy and POST :8080/v1/audio/speech returns audio; 'model init' env.example documents NGC_API_KEY.", + "status": "confirmed" + } + ], + "hard_questions": [], + "links": [] + } + ], + "open_vagueness": [ + { + "id": "v1", + "text": "Root cause of the Parakeet 'CUDA error: unknown error' is unconfirmed \u2014 restart is the working hypothesis (stale context on contended GB10) but may not clear it; if it recurs, deeper driver/contention diagnosis is needed.", + "kind": "unknown_nonblocking", + "claim_id": null + }, + { + "id": "v2", + "text": "How 'model fleet up' should gate the audio services (always-on vs a --audio profile/flag) given the shared GB10 memory budget \u2014 a design detail to settle in spec-to-plan.", + "kind": "unknown_nonblocking", + "claim_id": null + } + ] +} diff --git a/.gitignore b/.gitignore index 57a5f8c..26a576a 100644 --- a/.gitignore +++ b/.gitignore @@ -220,3 +220,6 @@ __marimo__/ # Streamlit .streamlit/secrets.toml + +# devague working state (not committed by default) +.devague/reviews/ diff --git a/docs/specs/2026-06-12-model-gear-owns-the-audio-i-o-surface-end-to-end-m.md b/docs/specs/2026-06-12-model-gear-owns-the-audio-i-o-surface-end-to-end-m.md new file mode 100644 index 0000000..00a7c05 --- /dev/null +++ b/docs/specs/2026-06-12-model-gear-owns-the-audio-i-o-surface-end-to-end-m.md @@ -0,0 +1,63 @@ +# model-gear owns the audio I/O surface end-to-end: `model fleet up` brings up Parakeet STT plus the OpenAI-compatible `/v1/audio/transcriptions` (and `/v1/audio/speech`) facade on :8080 from model-gear's own vendored realtime app and compose template, with a healthcheck that reflects model readiness — unblocking reachy-mini-cli's 'hey reachy' wake-word (closes #39 + #40). + +> model-gear owns the audio I/O surface end-to-end: `model fleet up` brings up Parakeet STT plus the OpenAI-compatible `/v1/audio/transcriptions` (and `/v1/audio/speech`) facade on :8080 from model-gear's own vendored realtime app and compose template, with a healthcheck that reflects model readiness — unblocking reachy-mini-cli's 'hey reachy' wake-word (closes #39 + #40). + +## Audience + +- reachy-mini-cli (the wake-word client) and any client wanting the documented OpenAI /v1/audio/* surface behind :8080; secondarily model-gear operators running 'model fleet up' + +## Before → After + +- Before: The live :8080 facade is the realtime-api sibling build exposing only / , /health, /v1/realtime (WS) — 404 on REST /v1/audio/transcriptions. Parakeet (:9002) 500s on every transcription with CUDA error: unknown error after long uptime on the contended GB10, while its healthcheck still reports 'healthy'. model-gear's vendored facade has the REST routes but is not in its compose template, so it never reaches the box. +- After: model-gear's compose template + 'model fleet up' bring up Parakeet STT and the vendored realtime facade; GET :8080/openapi.json lists /v1/audio/transcriptions and /v1/audio/speech; POST a multipart WAV to :8080/v1/audio/transcriptions forwards to Parakeet and returns the OpenAI-shaped {text:...}; Parakeet's healthcheck exercises a real transcription so 'healthy' means actually-serving. + +## Why it matters + +- reachy-mini-cli Tier-2 wake-word ('hey reachy') is blocked on a working STT endpoint; today the only working STT is raw Parakeet :9002 (when not 500ing), bypassing the OpenAI-compatible facade. Ending the vendored-vs-deployed drift gives one owned, documented audio surface. + +## Requirements + +- model-gear's packaged compose template (model_gear/templates/docker-compose.yml) gains a Parakeet STT service and a realtime-facade service built from the vendored model_gear/realtime app, so 'model init' + 'model fleet up' materialise and start them. + - honesty: 'model fleet up --apply' on a clean box brings up parakeet + facade services healthy, and 'model init' materialises them into the deployment dir without manual compose edits. +- The realtime facade service serves POST /v1/audio/transcriptions (forward to Parakeet) and POST /v1/audio/speech (forward to Magpie) on :8080, and its /openapi.json lists both routes. + - honesty: curl :8080/openapi.json lists both /v1/audio/transcriptions and /v1/audio/speech, and a multipart WAV POST to /v1/audio/transcriptions returns 200 with {text:...} sourced from Parakeet. +- The Parakeet container's Docker healthcheck exercises real model readiness (a tiny transcription or equivalent model-ready probe), so the container only reports 'healthy' when it can actually transcribe. + - honesty: When the Parakeet model is loaded and serving, the healthcheck passes; when the model path is broken (e.g. CUDA-unknown 500s), the healthcheck reports unhealthy within its retry window. +- Documentation (a docs/realtime-pipeline.md and/or README 'Audio I/O' section) states that model-gear owns the live :8080 facade, how 'model fleet up' brings up the audio routes, and the runbook for clearing a stale Parakeet CUDA context (restart). + - honesty: A reader following only the doc can stand up the audio surface and recover a wedged Parakeet without reading source, and the doc names which project owns the live :8080 container. +- model-gear's compose template gains a Magpie TTS service (NGC NIM, NGC_API_KEY from .env) so the facade's /v1/audio/speech route resolves to a fleet-owned backend. + - honesty: 'model fleet up' brings up Magpie healthy and POST :8080/v1/audio/speech returns audio; 'model init' env.example documents NGC_API_KEY. + +## Honesty conditions + +- After 'model fleet up', :8080 serves the REST audio routes from model-gear's own vendored app (not the sibling build), verified by /openapi.json + a 200 transcription. +- reachy-mini-cli can point REACHY_STT_URL at the :8080 facade (or :9002) and get a 'hey reachy' transcription back. +- The drift is real and reproducible today: live :8080 /openapi.json shows only / and /health; :9002 transcription 500s with CUDA-unknown. +- Each after-state assertion is independently checkable by a curl/WAV smoke test documented in the realtime-pipeline doc. +- With the facade route live and Parakeet serving, reachy-mini-cli's Tier-2 wake-word path stops returning 404/500. +- The spec touches only the REST audio facade, Parakeet serving, and its healthcheck — it does not modify the /v1/realtime WS protocol, the TTS/STT engines, or gateway auth. +- Every success signal is a concrete, runnable check (curl, WAV POST, healthcheck flip, doc statement) with a pass/fail outcome. + +## Success signals + +- GET :8080/openapi.json lists /v1/audio/transcriptions and /v1/audio/speech; the issue-#39 repro WAV POST returns 200 with {text:...}; a real 'hey reachy' WAV transcribes to text containing the phrase; Parakeet's Docker healthcheck flips unhealthy when the model can't transcribe; docs state model-gear owns the live :8080 facade and 'model fleet up' brings up the audio routes. + +## Scope / boundaries + +- Not building a new STT/TTS engine — Parakeet (NeMo) and Magpie stay the backends. Not solving GPU contention generally; the #39 scope is restart-to-clear-context + a readiness-reflecting healthcheck (a documented co-residency memory split is explicitly out of scope here). Not changing the /v1/realtime WebSocket protocol. Not making the gateway auth-aware. + +## Non-goals + +- Decommissioning the realtime-api sibling project, or migrating its /v1/realtime WebSocket speech-to-speech loop into model-gear, is out of scope — only the REST audio facade + Parakeet STT move under model-gear ownership. + +## Decisions + +- model-gear takes over ownership of the audio stack (Parakeet + vendored facade wired into model-gear's compose template and 'model fleet up'), rather than reconciling the realtime-api sibling in place or re-vendoring upstream. +- The #39 Parakeet fix depth is restart-to-clear-stale-CUDA-context plus a Docker healthcheck that exercises real model readiness (transcription), not restart-only and not a full documented co-residency memory contract. +- model-gear's fleet brings up the full audio stack end-to-end: Parakeet STT, Magpie TTS, and the realtime facade — so both /v1/audio/transcriptions and /v1/audio/speech are owned (adds the NGC_API_KEY dependency and GB10 memory to the fleet budget). +- Parakeet's readiness healthcheck is a cheap model-ready probe (confirms the NeMo model is loaded and the CUDA context is live), not a full multipart transcription each interval — resolving the flap/GPU-load risk q1. + +## Hard questions + +- risk: Adding parakeet + facade (and possibly magpie) to model-gear's fleet raises the GB10 co-residency memory pressure that #39 implicates — the very contention that may have wedged Parakeet. +- risk: A healthcheck that runs a real transcription is heavier/slower than a liveness probe and could flap or add GPU load; a cheap model-ready probe may be the safer form. From 47c805d5191adafa9fbe88ad68df803867cf9206 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Fri, 12 Jun 2026 12:35:25 +0300 Subject: [PATCH 2/8] =?UTF-8?q?plan:=20model-gear=20audio=20surface=20?= =?UTF-8?q?=E2=80=94=206=20tasks,=205=20waves=20(devague=20/spec-to-plan)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Forward leg from the converged audio-ownership spec. Six file-disjoint, TDD-gated tasks covering all 24 coverage targets: t1 vendor Parakeet server + cheap model-ready probe t2 build Dockerfiles (facade + parakeet) t3 wire the audio stack into the compose template (sole compose owner) t4 bring it up via `model fleet up` t5 docs/realtime-pipeline.md + README Audio I/O + CUDA restart runbook t6 smoke/acceptance harness (openapi, WAV 200, healthcheck flip) Waves: [t1,t5] -> t2 -> t3 -> t4 -> t6 (serial after wave 0 by the compose chokepoint). Risks carried: CUDA root-cause unconfirmed, fleet gating on the GB10 budget, cheap-probe wording reconciliation. Co-Authored-By: Claude Opus 4.8 (1M context) --- .devague/current_plan | 1 + ...ns-the-audio-i-o-surface-end-to-end-m.json | 279 ++++++++++++++++++ ...owns-the-audio-i-o-surface-end-to-end-m.md | 65 ++++ 3 files changed, 345 insertions(+) create mode 100644 .devague/current_plan create mode 100644 .devague/plans/model-gear-owns-the-audio-i-o-surface-end-to-end-m.json create mode 100644 docs/plans/2026-06-12-model-gear-owns-the-audio-i-o-surface-end-to-end-m.md diff --git a/.devague/current_plan b/.devague/current_plan new file mode 100644 index 0000000..eb803b3 --- /dev/null +++ b/.devague/current_plan @@ -0,0 +1 @@ +model-gear-owns-the-audio-i-o-surface-end-to-end-m diff --git a/.devague/plans/model-gear-owns-the-audio-i-o-surface-end-to-end-m.json b/.devague/plans/model-gear-owns-the-audio-i-o-surface-end-to-end-m.json new file mode 100644 index 0000000..d946473 --- /dev/null +++ b/.devague/plans/model-gear-owns-the-audio-i-o-surface-end-to-end-m.json @@ -0,0 +1,279 @@ +{ + "slug": "model-gear-owns-the-audio-i-o-surface-end-to-end-m", + "title": "model-gear owns the audio I/O surface end-to-end: `model fleet up` brings up Parakeet STT plus the OpenAI-compatible `/v1/audio/transcriptions` (and `/v1/audio/speech`) facade on :8080 from model-gear's own vendored realtime app and compose template, with a healthcheck that reflects model readiness \u2014 unblocking reachy-mini-cli's 'hey reachy' wake-word (closes #39 + #40).", + "frame_slug": "model-gear-owns-the-audio-i-o-surface-end-to-end-m", + "schema_version": 1, + "status": "exported", + "created": "2026-06-12T09:28:27Z", + "updated": "2026-06-12T09:35:12Z", + "targets": [ + { + "id": "c1", + "kind": "announcement", + "text": "model-gear owns the audio I/O surface end-to-end: `model fleet up` brings up Parakeet STT plus the OpenAI-compatible `/v1/audio/transcriptions` (and `/v1/audio/speech`) facade on :8080 from model-gear's own vendored realtime app and compose template, with a healthcheck that reflects model readiness \u2014 unblocking reachy-mini-cli's 'hey reachy' wake-word (closes #39 + #40)." + }, + { + "id": "h5", + "kind": "honesty", + "text": "After 'model fleet up', :8080 serves the REST audio routes from model-gear's own vendored app (not the sibling build), verified by /openapi.json + a 200 transcription." + }, + { + "id": "c2", + "kind": "audience", + "text": "reachy-mini-cli (the wake-word client) and any client wanting the documented OpenAI /v1/audio/* surface behind :8080; secondarily model-gear operators running 'model fleet up'" + }, + { + "id": "h6", + "kind": "honesty", + "text": "reachy-mini-cli can point REACHY_STT_URL at the :8080 facade (or :9002) and get a 'hey reachy' transcription back." + }, + { + "id": "c3", + "kind": "before_state", + "text": "The live :8080 facade is the realtime-api sibling build exposing only / , /health, /v1/realtime (WS) \u2014 404 on REST /v1/audio/transcriptions. Parakeet (:9002) 500s on every transcription with CUDA error: unknown error after long uptime on the contended GB10, while its healthcheck still reports 'healthy'. model-gear's vendored facade has the REST routes but is not in its compose template, so it never reaches the box." + }, + { + "id": "h7", + "kind": "honesty", + "text": "The drift is real and reproducible today: live :8080 /openapi.json shows only / and /health; :9002 transcription 500s with CUDA-unknown." + }, + { + "id": "c4", + "kind": "after_state", + "text": "model-gear's compose template + 'model fleet up' bring up Parakeet STT and the vendored realtime facade; GET :8080/openapi.json lists /v1/audio/transcriptions and /v1/audio/speech; POST a multipart WAV to :8080/v1/audio/transcriptions forwards to Parakeet and returns the OpenAI-shaped {text:...}; Parakeet's healthcheck exercises a real transcription so 'healthy' means actually-serving." + }, + { + "id": "h8", + "kind": "honesty", + "text": "Each after-state assertion is independently checkable by a curl/WAV smoke test documented in the realtime-pipeline doc." + }, + { + "id": "c5", + "kind": "why_it_matters", + "text": "reachy-mini-cli Tier-2 wake-word ('hey reachy') is blocked on a working STT endpoint; today the only working STT is raw Parakeet :9002 (when not 500ing), bypassing the OpenAI-compatible facade. Ending the vendored-vs-deployed drift gives one owned, documented audio surface." + }, + { + "id": "h9", + "kind": "honesty", + "text": "With the facade route live and Parakeet serving, reachy-mini-cli's Tier-2 wake-word path stops returning 404/500." + }, + { + "id": "c6", + "kind": "boundary", + "text": "Not building a new STT/TTS engine \u2014 Parakeet (NeMo) and Magpie stay the backends. Not solving GPU contention generally; the #39 scope is restart-to-clear-context + a readiness-reflecting healthcheck (a documented co-residency memory split is explicitly out of scope here). Not changing the /v1/realtime WebSocket protocol. Not making the gateway auth-aware." + }, + { + "id": "h10", + "kind": "honesty", + "text": "The spec touches only the REST audio facade, Parakeet serving, and its healthcheck \u2014 it does not modify the /v1/realtime WS protocol, the TTS/STT engines, or gateway auth." + }, + { + "id": "c7", + "kind": "success_signal", + "text": "GET :8080/openapi.json lists /v1/audio/transcriptions and /v1/audio/speech; the issue-#39 repro WAV POST returns 200 with {text:...}; a real 'hey reachy' WAV transcribes to text containing the phrase; Parakeet's Docker healthcheck flips unhealthy when the model can't transcribe; docs state model-gear owns the live :8080 facade and 'model fleet up' brings up the audio routes." + }, + { + "id": "h11", + "kind": "honesty", + "text": "Every success signal is a concrete, runnable check (curl, WAV POST, healthcheck flip, doc statement) with a pass/fail outcome." + }, + { + "id": "c11", + "kind": "requirement", + "text": "model-gear's packaged compose template (model_gear/templates/docker-compose.yml) gains a Parakeet STT service and a realtime-facade service built from the vendored model_gear/realtime app, so 'model init' + 'model fleet up' materialise and start them." + }, + { + "id": "h1", + "kind": "honesty", + "text": "'model fleet up --apply' on a clean box brings up parakeet + facade services healthy, and 'model init' materialises them into the deployment dir without manual compose edits." + }, + { + "id": "c12", + "kind": "requirement", + "text": "The realtime facade service serves POST /v1/audio/transcriptions (forward to Parakeet) and POST /v1/audio/speech (forward to Magpie) on :8080, and its /openapi.json lists both routes." + }, + { + "id": "h2", + "kind": "honesty", + "text": "curl :8080/openapi.json lists both /v1/audio/transcriptions and /v1/audio/speech, and a multipart WAV POST to /v1/audio/transcriptions returns 200 with {text:...} sourced from Parakeet." + }, + { + "id": "c13", + "kind": "requirement", + "text": "The Parakeet container's Docker healthcheck exercises real model readiness (a tiny transcription or equivalent model-ready probe), so the container only reports 'healthy' when it can actually transcribe." + }, + { + "id": "h3", + "kind": "honesty", + "text": "When the Parakeet model is loaded and serving, the healthcheck passes; when the model path is broken (e.g. CUDA-unknown 500s), the healthcheck reports unhealthy within its retry window." + }, + { + "id": "c14", + "kind": "requirement", + "text": "Documentation (a docs/realtime-pipeline.md and/or README 'Audio I/O' section) states that model-gear owns the live :8080 facade, how 'model fleet up' brings up the audio routes, and the runbook for clearing a stale Parakeet CUDA context (restart)." + }, + { + "id": "h4", + "kind": "honesty", + "text": "A reader following only the doc can stand up the audio surface and recover a wedged Parakeet without reading source, and the doc names which project owns the live :8080 container." + }, + { + "id": "c17", + "kind": "requirement", + "text": "model-gear's compose template gains a Magpie TTS service (NGC NIM, NGC_API_KEY from .env) so the facade's /v1/audio/speech route resolves to a fleet-owned backend." + }, + { + "id": "h12", + "kind": "honesty", + "text": "'model fleet up' brings up Magpie healthy and POST :8080/v1/audio/speech returns audio; 'model init' env.example documents NGC_API_KEY." + } + ], + "tasks": [ + { + "id": "t1", + "summary": "Vendor the Parakeet ASR server into model-gear with a cheap model-ready health probe", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "model_gear/realtime/parakeet_server.py exposes POST /v1/audio/transcriptions and GET /v1/health/ready", + "GET /v1/health/ready returns 200 only when the NeMo model is loaded AND a trivial CUDA tensor op succeeds, else 503 \u2014 a cheap probe, NOT a full multipart transcription each interval (decision c16 supersedes the 'real transcription' wording in c4/c7)", + "A unit test stubs the model to assert 200-when-ready and 503-when-model-None/CUDA-fails, with no GPU required in CI" + ], + "deps": [], + "covers": [ + "c13", + "h3" + ] + }, + { + "id": "t2", + "summary": "Package build Dockerfiles for the facade and Parakeet services under templates/", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "model_gear/templates/Dockerfile.realtime installs the [realtime] extra and runs model_gear.realtime.app; Dockerfile.parakeet installs NeMo ASR and copies parakeet_server.py", + "docker build succeeds for both from a clean checkout (smoke-built in the PR or documented as built)" + ], + "deps": [ + "t1" + ], + "covers": [ + "c11" + ] + }, + { + "id": "t3", + "summary": "Wire the full audio stack into the compose template + env.example (sole owner of docker-compose.yml)", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "model_gear/templates/docker-compose.yml gains parakeet-stt, magpie-tts, and realtime-facade services; the facade env points STT_URL->parakeet and TTS_URL->magpie and binds host :8080", + "env.example documents NGC_API_KEY (Magpie) and the audio service ports; 'model init' materialises them with no manual compose edits", + "The realtime-facade service builds from Dockerfile.realtime (the vendored model_gear app), NOT the realtime-api sibling image", + "Parakeet service healthcheck calls GET /v1/health/ready (the cheap probe from t1)" + ], + "deps": [ + "t1", + "t2" + ], + "covers": [ + "c11", + "c12", + "c17", + "c4", + "h1", + "h2", + "h5", + "h12" + ] + }, + { + "id": "t4", + "summary": "Bring the audio services up via 'model fleet up' (gating + ownership)", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "'model fleet up --apply' starts parakeet + magpie + facade and they report healthy on a box with the GPU available", + "'model fleet status' lists the audio services and the facade /health; the audio surface is served from model-gear's deployment dir, not the sibling project", + "A dry-run (no --apply) prints the planned audio services without starting them (mutation-safety default honoured)" + ], + "deps": [ + "t3" + ], + "covers": [ + "c1", + "h1", + "h5" + ] + }, + { + "id": "t5", + "summary": "Author docs/realtime-pipeline.md (ownership, fleet-up, drift, CUDA restart runbook) + README Audio I/O", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "docs/realtime-pipeline.md states model-gear owns the live :8080 facade, shows the 'model fleet up' bring-up, and gives the restart runbook for a stale Parakeet CUDA context", + "The doc records the prior drift (sibling :8080 build had only / and /health; :9002 500'd with CUDA-unknown) and the boundary (no /v1/realtime WS change, engines unchanged, no gateway auth)", + "A reader can stand up the audio surface and recover a wedged Parakeet following only the doc, without reading source" + ], + "deps": [], + "covers": [ + "c14", + "h4", + "c3", + "h7", + "c6", + "h10" + ] + }, + { + "id": "t6", + "summary": "Smoke-test + acceptance harness for the audio routes (curl/WAV, openapi, healthcheck flip)", + "origin": "llm", + "status": "confirmed", + "acceptance_criteria": [ + "A scripts/ smoke test asserts GET :8080/openapi.json lists /v1/audio/transcriptions and /v1/audio/speech, and a multipart WAV POST returns 200 {text:...}", + "The harness reproduces issue #39's WAV repro (200, not 500) and a 'hey reachy' WAV transcribes to text containing the phrase; REACHY_STT_URL=:8080 works for the wake-word client", + "A test asserts the Parakeet healthcheck flips unhealthy when the model can't transcribe and POST :8080/v1/audio/speech returns audio (Magpie up)" + ], + "deps": [ + "t3", + "t4" + ], + "covers": [ + "c2", + "c5", + "c7", + "h6", + "h8", + "h9", + "h11", + "h2", + "h3", + "h12", + "c4", + "c12" + ] + } + ], + "risks": [ + { + "id": "r1", + "text": "Parakeet CUDA-unknown root cause is unconfirmed \u2014 restart is the working hypothesis; if it recurs under co-residency with vllm+magpie, deeper driver/contention diagnosis is needed (frame v1)", + "kind": "unknown_nonblocking", + "task_id": "t6" + }, + { + "id": "r2", + "text": "How 'model fleet up' gates the audio services (always-on vs a --audio profile/flag) on the shared GB10 memory budget is unsettled \u2014 t4 must choose; adding parakeet+magpie raises the very contention #39 implicates (frame v2 + risk q2)", + "kind": "unknown_nonblocking", + "task_id": "t4" + }, + { + "id": "r3", + "text": "Spec wording reconciliation: c4/c7 say the healthcheck 'exercises a real transcription' but decision c16 supersedes with a cheap model-ready probe \u2014 t1/t5 acceptance must reflect the cheap probe, not full transcription", + "kind": "follow_up", + "task_id": "t1" + } + ] +} diff --git a/docs/plans/2026-06-12-model-gear-owns-the-audio-i-o-surface-end-to-end-m.md b/docs/plans/2026-06-12-model-gear-owns-the-audio-i-o-surface-end-to-end-m.md new file mode 100644 index 0000000..88949cc --- /dev/null +++ b/docs/plans/2026-06-12-model-gear-owns-the-audio-i-o-surface-end-to-end-m.md @@ -0,0 +1,65 @@ +# Build Plan — model-gear owns the audio I/O surface end-to-end: `model fleet up` brings up Parakeet STT plus the OpenAI-compatible `/v1/audio/transcriptions` (and `/v1/audio/speech`) facade on :8080 from model-gear's own vendored realtime app and compose template, with a healthcheck that reflects model readiness — unblocking reachy-mini-cli's 'hey reachy' wake-word (closes #39 + #40). + +slug: `model-gear-owns-the-audio-i-o-surface-end-to-end-m` · status: `exported` · from frame: `model-gear-owns-the-audio-i-o-surface-end-to-end-m` + +> model-gear owns the audio I/O surface end-to-end: `model fleet up` brings up Parakeet STT plus the OpenAI-compatible `/v1/audio/transcriptions` (and `/v1/audio/speech`) facade on :8080 from model-gear's own vendored realtime app and compose template, with a healthcheck that reflects model readiness — unblocking reachy-mini-cli's 'hey reachy' wake-word (closes #39 + #40). + +## Tasks + +### t1 — Vendor the Parakeet ASR server into model-gear with a cheap model-ready health probe + +- covers: c13, h3 +- acceptance: + - model_gear/realtime/parakeet_server.py exposes POST /v1/audio/transcriptions and GET /v1/health/ready + - GET /v1/health/ready returns 200 only when the NeMo model is loaded AND a trivial CUDA tensor op succeeds, else 503 — a cheap probe, NOT a full multipart transcription each interval (decision c16 supersedes the 'real transcription' wording in c4/c7) + - A unit test stubs the model to assert 200-when-ready and 503-when-model-None/CUDA-fails, with no GPU required in CI + +### t2 — Package build Dockerfiles for the facade and Parakeet services under templates/ + +- depends on: t1 +- covers: c11 +- acceptance: + - model_gear/templates/Dockerfile.realtime installs the [realtime] extra and runs model_gear.realtime.app; Dockerfile.parakeet installs NeMo ASR and copies parakeet_server.py + - docker build succeeds for both from a clean checkout (smoke-built in the PR or documented as built) + +### t3 — Wire the full audio stack into the compose template + env.example (sole owner of docker-compose.yml) + +- depends on: t1, t2 +- covers: c11, c12, c17, c4, h1, h2, h5, h12 +- acceptance: + - model_gear/templates/docker-compose.yml gains parakeet-stt, magpie-tts, and realtime-facade services; the facade env points STT_URL->parakeet and TTS_URL->magpie and binds host :8080 + - env.example documents NGC_API_KEY (Magpie) and the audio service ports; 'model init' materialises them with no manual compose edits + - The realtime-facade service builds from Dockerfile.realtime (the vendored model_gear app), NOT the realtime-api sibling image + - Parakeet service healthcheck calls GET /v1/health/ready (the cheap probe from t1) + +### t4 — Bring the audio services up via 'model fleet up' (gating + ownership) + +- depends on: t3 +- covers: c1, h1, h5 +- acceptance: + - 'model fleet up --apply' starts parakeet + magpie + facade and they report healthy on a box with the GPU available + - 'model fleet status' lists the audio services and the facade /health; the audio surface is served from model-gear's deployment dir, not the sibling project + - A dry-run (no --apply) prints the planned audio services without starting them (mutation-safety default honoured) + +### t5 — Author docs/realtime-pipeline.md (ownership, fleet-up, drift, CUDA restart runbook) + README Audio I/O + +- covers: c14, h4, c3, h7, c6, h10 +- acceptance: + - docs/realtime-pipeline.md states model-gear owns the live :8080 facade, shows the 'model fleet up' bring-up, and gives the restart runbook for a stale Parakeet CUDA context + - The doc records the prior drift (sibling :8080 build had only / and /health; :9002 500'd with CUDA-unknown) and the boundary (no /v1/realtime WS change, engines unchanged, no gateway auth) + - A reader can stand up the audio surface and recover a wedged Parakeet following only the doc, without reading source + +### t6 — Smoke-test + acceptance harness for the audio routes (curl/WAV, openapi, healthcheck flip) + +- depends on: t3, t4 +- covers: c2, c5, c7, h6, h8, h9, h11, h2, h3, h12, c4, c12 +- acceptance: + - A scripts/ smoke test asserts GET :8080/openapi.json lists /v1/audio/transcriptions and /v1/audio/speech, and a multipart WAV POST returns 200 {text:...} + - The harness reproduces issue #39's WAV repro (200, not 500) and a 'hey reachy' WAV transcribes to text containing the phrase; REACHY_STT_URL=:8080 works for the wake-word client + - A test asserts the Parakeet healthcheck flips unhealthy when the model can't transcribe and POST :8080/v1/audio/speech returns audio (Magpie up) + +## Risks + +- [unknown_nonblocking] Parakeet CUDA-unknown root cause is unconfirmed — restart is the working hypothesis; if it recurs under co-residency with vllm+magpie, deeper driver/contention diagnosis is needed (frame v1) (task t6) +- [unknown_nonblocking] How 'model fleet up' gates the audio services (always-on vs a --audio profile/flag) on the shared GB10 memory budget is unsettled — t4 must choose; adding parakeet+magpie raises the very contention #39 implicates (frame v2 + risk q2) (task t4) +- [follow_up] Spec wording reconciliation: c4/c7 say the healthcheck 'exercises a real transcription' but decision c16 supersedes with a cheap model-ready probe — t1/t5 acceptance must reflect the cheap probe, not full transcription (task t1) From 8c946f8503e34103094033fa30a531594c994846 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Fri, 12 Jun 2026 12:52:35 +0300 Subject: [PATCH 3/8] feat(audio): real Parakeet readiness probe + realtime-pipeline docs + smoke MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the three real gaps found after reconnaissance (t2/t3/t4 — the audio overlay, Dockerfiles, fleet wiring — already existed in templates/fleet/): - t1: Parakeet /v1/health/ready now reflects REAL readiness (model loaded + a trivial CUDA op) instead of returning {"status":"ready"} unconditionally — the misleading-health bug behind #39. The pure decision lives in stdlib-only model_gear/realtime/_readiness.py (CI-unit-tested, 8 cases), vendored into the Parakeet build context and COPY'd into the image so the container imports it without the wheel. (decision c16: a cheap probe, not a full transcription each interval.) - t5: docs/realtime-pipeline.md — ownership of the live :8080 facade, fleet bring-up, the prior drift (#39/#40), health/readiness, and the stale-CUDA restart runbook. - t6: scripts/audio-smoke.py — stdlib-only live smoke (openapi lists both routes; WAV POST -> 200 {text}); reproduces #39's repro. Live run is GPU-box-bound, not CI. 271 tests pass; black/isort/flake8 + markdownlint clean. Refs #39 #40 Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/realtime-pipeline.md | 226 ++++++++++++++++++ model_gear/realtime/_readiness.py | 49 ++++ .../templates/fleet/Dockerfile.parakeet | 4 + model_gear/templates/fleet/_readiness.py | 48 ++++ model_gear/templates/fleet/listen_server.py | 44 +++- scripts/audio-smoke.py | 202 ++++++++++++++++ tests/test_realtime_readiness.py | 54 +++++ 7 files changed, 625 insertions(+), 2 deletions(-) create mode 100644 docs/realtime-pipeline.md create mode 100644 model_gear/realtime/_readiness.py create mode 100644 model_gear/templates/fleet/_readiness.py create mode 100644 scripts/audio-smoke.py create mode 100644 tests/test_realtime_readiness.py diff --git a/docs/realtime-pipeline.md b/docs/realtime-pipeline.md new file mode 100644 index 0000000..1ccf56d --- /dev/null +++ b/docs/realtime-pipeline.md @@ -0,0 +1,226 @@ +# Audio realtime pipeline: STT + TTS behind the fleet gateway + +## Ownership + +**model-gear owns the live audio surface** — the OpenAI `/v1/audio/*` facade +deployed as the `realtime` container in the fleet. The realtime bridge ships in +the model-gear wheel (`model_gear/realtime/app.py` and friends) and is built and +managed by `model init --fleet --audio` / `model fleet up --apply`. + +This consolidates what used to be a separate `realtime-api` sibling stack. The +same STT and TTS engines (Parakeet ASR + Magpie TTS) are now owned and versioned +here. + +The overlay compose file is **`model_gear/templates/fleet/docker-compose.audio.yml`**. +It is layered on top of the base fleet automatically when present during `model +fleet up`. + +## Topology + +```text +┌─────────────────────────────────────────────────────────────────┐ +│ Client / OpenAI SDK │ +│ │ +│ POST http://localhost:8000/v1/audio/transcriptions │ +│ or /v1/audio/speech │ +└──────────┬──────────────────────────────────────────────────────┘ + │ + │ port 8000 (host) + │ + [model-gear-gateway] (stdlib reverse proxy) + │ + │ internal, compose network + │ + ├─ route /v1/audio/* → [model-gear-realtime] :8080 + │ + └─ LLM requests → [vllm backends] + + [model-gear-realtime] :8080 (the facade) + │ + ├─ POST /v1/audio/transcriptions → [model-gear-stt] :9002 (Parakeet) + │ + └─ POST /v1/audio/speech → [model-gear-tts] :9000 (Magpie NIM) + +Both STT and TTS share the GPU with the two LLM backends. +``` + +## Bring-up + +### Prerequisites + +- NGC API key for Magpie TTS (get it at [https://org.ngc.nvidia.com](https://org.ngc.nvidia.com) → Setup). +- A GPU box (DGX Spark or dedicated; two ~30B models + audio barely co-fit on a + shared GB10 — see [Memory and co-residence risk](#memory-co-residence-risk) below). + +### Steps + +```bash +# 1. Initialize the fleet with the audio overlay +model init --fleet --audio --apply + +# 2. Set NGC_API_KEY in the .env file +# Locate ~/.model-gear/.env (or your --compose-dir) and edit: +# NGC_API_KEY= + +# 3. Bring up the full audio stack (dry-run by default; --apply commits) +model fleet up --apply + +# 4. Check status +model fleet status +``` + +Each `model init` and `model fleet` verb defaults to **dry-run**; omit `--apply` +to see what would happen, or add `--apply` to execute. This ensures safe-by-default +operation (useful when agents call CLIs in loops). + +To customize the compose dir (default `$MODEL_GEAR_DIR` or `~/.model-gear`): + +```bash +model init --fleet --audio --compose-dir /path/to/deployment --apply +model fleet up --compose-dir /path/to/deployment --apply +``` + +## The drift this fixed + +**Before (issue #39, #40):** + +- The old `realtime-api` sibling stack's `:8080` exposed only `/` and `/health` + endpoints; OpenAI REST routes like `/v1/audio/transcriptions` returned 404. +- Parakeet (`:9002`) failed nearly every transcription with `torch.AcceleratorError: + CUDA error: unknown error` deep in NeMo's CUDA context, even though its Docker + healthcheck reported "healthy". +- The healthcheck was liveness-only (probe `/health` without exercising the model); + "healthy" did not mean "actually serving". + +**After:** + +- model-gear now owns the audio surface. `model init --fleet --audio` scaffolds + the complete overlay (compose file, Dockerfile for realtime and Parakeet, + env keys), and `model fleet up --apply` builds and starts all three services + (`tts`, `stt`, `realtime`) behind the gateway. +- The realtime bridge forwards `/v1/audio/transcriptions` and `/v1/audio/speech` + to the backends (Parakeet and Magpie respectively) and wraps their responses + in the OpenAI schema. +- Parakeet's healthcheck now includes a real model-readiness probe (loads the + model, runs a trivial CUDA op); "healthy" means actually serving. + +## Health and readiness + +Parakeet's healthcheck (in `docker-compose.audio.yml`) is: + +```yaml +healthcheck: + test: + - CMD + - python3 + - -c + - import urllib.request; urllib.request.urlopen('http://localhost:${PARAKEET_PORT:-9002}/v1/health/ready') + interval: 30s + timeout: 10s + retries: 3 + start_period: 600s +``` + +This probe: + +- Loads the Parakeet model on container startup. +- Runs a trivial CUDA operation (`/v1/health/ready` inside NeMo). +- Reports "healthy" only if the model is actually loaded and CUDA is responsive. + +It is **not** a liveness check (like `curl http://localhost:9002/health` would be). +If Parakeet is "healthy", it is actively serving transcriptions. + +## Runbook: stale Parakeet CUDA context (CUDA error: unknown error) + +**Symptom:** + +```text +Error 500: torch.AcceleratorError: CUDA error: unknown error +``` + +Transcription requests fail with 500s even though `docker ps` shows the STT +container as "healthy". + +**Root cause (suspected):** + +On the shared GB10 (DGX Spark), prolonged co-residence of two ~30B NVFP4 models +(vLLM primary + fallback) + Magpie TTS + Parakeet STT results in a contended GPU +and fragmented CUDA memory. After several hours or under sustained load, Parakeet's +CUDA context becomes stale, and new transcription requests fail deep in NeMo's +initialization. + +**Fix:** + +Restart the STT container to clear the stale CUDA context: + +```bash +docker restart model-gear-stt +``` + +Or cycle the entire fleet: + +```bash +model fleet down --apply && model fleet up --apply +``` + +Watch `nvidia-smi` to confirm memory is freed before the STT container restarts. + +**Diagnosis (if it recurs):** + +- Compare GPU memory before/after `docker restart model-gear-stt`. +- Run `nvidia-smi` to check for fragmentation or orphaned allocations. +- If CUDA errors persist, contact NVIDIA support or consider (a) running audio + on a dedicated GPU, (b) reducing the fleet to a single LLM, or (c) lowering + `PRIMARY_GPU_MEM_UTIL` and `FALLBACK_GPU_MEM_UTIL` to reduce baseline memory + load. + +Root cause diagnosis is open; see issues #39 and #40 if this resurfaces. + +## Boundary / non-goals + +The audio surface **does not**: + +- Change the `/v1/realtime` WebSocket protocol (that is planned for a later + release; the current surface is REST only: `/v1/audio/transcriptions` and + `/v1/audio/speech`). +- Swap the STT/TTS engines — Parakeet (NeMo ASR) and Magpie (NVIDIA NIM) remain + the hardcoded backends. +- Make the gateway auth-aware — the same `/v1/chat/completions` gateway token + that works for the LLM does not yet extend to audio. Plan to add per-endpoint + auth in a later release. + +## Memory (co-residence risk) + +On a GB10 shared with other services, two ~30B NVFP4 models barely co-fit with +usable KV caches. Adding Parakeet + Magpie increases contention. Options: + +- Run audio on a **dedicated GPU** (recommended). +- Reduce the fleet to a **single LLM** and use `model switch` instead of fleet. +- Tune `PRIMARY_GPU_MEM_UTIL` and `FALLBACK_GPU_MEM_UTIL` in `.env` to lower + the baseline (the defaults are estimates for a dedicated box). + +See [`docs/gateway-fleet.md`](gateway-fleet.md#memory-both-warm) for full memory +guidance and [`docs/gateway-fleet.md#live-validation-findings`](gateway-fleet.md#live-validation-findings--dgx-spark-gb10-2026-05-30) +for concrete measurements on the shared DGX Spark. + +## Smoke test + +Run the live audio smoke test to verify the stack is serving: + +```bash +python3 scripts/audio-smoke.py +# or with a custom base URL (default http://localhost:8080): +python3 scripts/audio-smoke.py --base-url http://10.0.0.42:8080 +``` + +The script: + +1. Checks that `GET /openapi.json` lists both `/v1/audio/transcriptions` and + `/v1/audio/speech`. +2. Generates a 2-second 440 Hz tone (16 kHz, mono, PCM16 WAV). +3. Sends it to `/v1/audio/transcriptions` and confirms a 200 response with a + `text` field. +4. Prints PASS/FAIL for each step and exits non-zero on failure. + +This requires a **live GPU box** with `model fleet up` already running; it is not +an offline CI test. It reproduces the issue #39 symptom to confirm the fix. diff --git a/model_gear/realtime/_readiness.py b/model_gear/realtime/_readiness.py new file mode 100644 index 0000000..28530ea --- /dev/null +++ b/model_gear/realtime/_readiness.py @@ -0,0 +1,49 @@ +"""Readiness decision logic for the Parakeet ``/v1/health/ready`` probe. + +Backs the cheap health probe described in issue #39 (decision c16): report +``200 ready`` only when the ASR model is loaded AND the CUDA context is live — +never unconditionally, as the old liveness-only handler did. + +"Cheap" means: model loaded flag + a trivial CUDA tensor op. No +``model.transcribe`` call, no real audio. This keeps probe overhead ~1 ms +instead of seconds. + +This module is **stdlib only** so it is importable and unit-testable in the +offline CI environment (no torch / nemo / fastapi installed there). The heavy +container code that calls :func:`evaluate_readiness` lives in +``model_gear/templates/fleet/listen_server.py``. + +Note: a vendored copy of this file lives at +``model_gear/templates/fleet/_readiness.py`` (the container build context). +Both copies must stay in sync — cite-don't-import convention. +""" + +from __future__ import annotations + + +def evaluate_readiness(model_loaded: bool, cuda_ok: bool) -> tuple[int, dict]: + """Return an ``(http_status, body)`` pair reflecting real ASR readiness. + + Parameters + ---------- + model_loaded: + ``True`` when the NeMo ASR model object has been loaded and is not + ``None``; ``False`` during startup or after a load failure. + cuda_ok: + ``True`` when a trivial CUDA tensor op (e.g. ``torch.zeros(1, + device="cuda"); torch.cuda.synchronize()``) completes without + exception; ``False`` on any CUDA error (unknown error, OOM, …). + + Returns + ------- + tuple[int, dict] + ``(200, {"status": "ready"})`` when both checks pass. + ``(503, {"status": "not_ready", "reason": })`` otherwise — + Docker healthcheck treats non-2xx as failing, so the container will + not be reported healthy until the model AND CUDA are actually live. + """ + if not model_loaded: + return 503, {"status": "not_ready", "reason": "model not loaded"} + if not cuda_ok: + return 503, {"status": "not_ready", "reason": "CUDA not available"} + return 200, {"status": "ready"} diff --git a/model_gear/templates/fleet/Dockerfile.parakeet b/model_gear/templates/fleet/Dockerfile.parakeet index e47bbf1..5f5f00f 100644 --- a/model_gear/templates/fleet/Dockerfile.parakeet +++ b/model_gear/templates/fleet/Dockerfile.parakeet @@ -20,6 +20,10 @@ RUN pip install --no-cache-dir \ RUN python3 -c "import nemo.collections.asr as nemo_asr; nemo_asr.models.ASRModel.from_pretrained('nvidia/parakeet-tdt-0.6b-v2')" WORKDIR /app +# _readiness.py is a vendored copy of model_gear/realtime/_readiness.py (stdlib +# only, no wheel needed). Copied here so listen_server.py can import it as a +# top-level module without requiring the model_gear wheel to be installed. +COPY _readiness.py /app/_readiness.py COPY listen_server.py /app/listen_server.py EXPOSE 9002 diff --git a/model_gear/templates/fleet/_readiness.py b/model_gear/templates/fleet/_readiness.py new file mode 100644 index 0000000..3cdf135 --- /dev/null +++ b/model_gear/templates/fleet/_readiness.py @@ -0,0 +1,48 @@ +"""Readiness decision logic for the Parakeet ``/v1/health/ready`` probe. + +Backs the cheap health probe described in issue #39 (decision c16): report +``200 ready`` only when the ASR model is loaded AND the CUDA context is live — +never unconditionally, as the old liveness-only handler did. + +"Cheap" means: model loaded flag + a trivial CUDA tensor op. No +``model.transcribe`` call, no real audio. This keeps probe overhead ~1 ms +instead of seconds. + +This is a VENDORED COPY of ``model_gear/realtime/_readiness.py`` placed in the +Parakeet container build context (``model_gear/templates/fleet/``). The +Dockerfile COPYs it next to ``listen_server.py`` so the container can import it +as a top-level module without needing the ``model_gear`` wheel. + +IMPORTANT: keep this file in sync with the canonical copy at +``model_gear/realtime/_readiness.py`` — cite-don't-import convention. +""" + +from __future__ import annotations + + +def evaluate_readiness(model_loaded: bool, cuda_ok: bool) -> tuple[int, dict]: + """Return an ``(http_status, body)`` pair reflecting real ASR readiness. + + Parameters + ---------- + model_loaded: + ``True`` when the NeMo ASR model object has been loaded and is not + ``None``; ``False`` during startup or after a load failure. + cuda_ok: + ``True`` when a trivial CUDA tensor op (e.g. ``torch.zeros(1, + device="cuda"); torch.cuda.synchronize()``) completes without + exception; ``False`` on any CUDA error (unknown error, OOM, …). + + Returns + ------- + tuple[int, dict] + ``(200, {"status": "ready"})`` when both checks pass. + ``(503, {"status": "not_ready", "reason": })`` otherwise — + Docker healthcheck treats non-2xx as failing, so the container will + not be reported healthy until the model AND CUDA are actually live. + """ + if not model_loaded: + return 503, {"status": "not_ready", "reason": "model not loaded"} + if not cuda_ok: + return 503, {"status": "not_ready", "reason": "CUDA not available"} + return 200, {"status": "ready"} diff --git a/model_gear/templates/fleet/listen_server.py b/model_gear/templates/fleet/listen_server.py index e56d3dd..16e15b4 100644 --- a/model_gear/templates/fleet/listen_server.py +++ b/model_gear/templates/fleet/listen_server.py @@ -8,7 +8,7 @@ Endpoints: POST /v1/audio/transcriptions - Transcribe an uploaded audio file - GET /v1/health/ready - Health check + GET /v1/health/ready - Readiness check (model loaded + CUDA live) """ import io @@ -18,6 +18,26 @@ import soundfile as sf import uvicorn from fastapi import FastAPI, File, Form, UploadFile +from fastapi.responses import JSONResponse + +# Import the readiness decision logic. The Dockerfile COPYs _readiness.py +# next to listen_server.py (as a top-level module in /app), so this import +# works whether or not the model_gear wheel is installed in the container. +# Falls back gracefully if neither is present (shouldn't happen, but defensive). +try: + from _readiness import evaluate_readiness # container-local copy (top-level) +except ImportError: + try: + from model_gear.realtime._readiness import evaluate_readiness # wheel install + except ImportError: + # Last-resort inline fallback — keeps the container functional even if + # both import paths fail (e.g. a dev build without the COPY step). + def evaluate_readiness(model_loaded: bool, cuda_ok: bool): # type: ignore[misc] + if not model_loaded: + return 503, {"status": "not_ready", "reason": "model not loaded"} + if not cuda_ok: + return 503, {"status": "not_ready", "reason": "CUDA not available"} + return 200, {"status": "ready"} logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) @@ -50,7 +70,27 @@ async def startup(): @app.get("/v1/health/ready") async def health(): - return {"status": "ready"} + """Cheap readiness probe (issue #39, decision c16). + + Reports ready ONLY when the NeMo model is loaded AND a trivial CUDA tensor + op succeeds — never unconditionally. Docker healthcheck treats non-2xx as + failing, so 503 keeps the container unhealthy until it can actually serve. + """ + model_loaded = _model is not None + + # Cheap CUDA liveness: allocate a 1-element tensor and synchronise. + # Any CUDA error (unknown error, driver not ready, OOM, …) → not ready. + try: + import torch + + torch.zeros(1, device="cuda") + torch.cuda.synchronize() + cuda_ok = True + except Exception: + cuda_ok = False + + status_code, body = evaluate_readiness(model_loaded, cuda_ok) + return JSONResponse(status_code=status_code, content=body) @app.post("/v1/audio/transcriptions") diff --git a/scripts/audio-smoke.py b/scripts/audio-smoke.py new file mode 100644 index 0000000..b7f8bdc --- /dev/null +++ b/scripts/audio-smoke.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Smoke test for the live audio realtime surface. + +Tests the OpenAI /v1/audio/* endpoints (transcriptions + speech) against a +running model-gear fleet with the audio overlay. This is NOT an offline CI test +— it requires a live GPU box with `model fleet up` already active. + +Reproduces the issue #39 repro to confirm the 500→200 fix: generates a 2s 440 Hz +tone (16 kHz mono PCM16 WAV), posts it to /v1/audio/transcriptions, and asserts +HTTP 200 + a JSON response with a `text` key. + +Exit code 0 if all checks pass; non-zero on any failure. +""" + +from __future__ import annotations + +import argparse +import io +import json +import math +import sys +import urllib.request +from urllib.error import URLError + + +def build_wav_tone( + duration_sec: float = 2.0, freq_hz: float = 440.0, sample_rate: int = 16000 +) -> bytes: + """Generate a sine-wave WAV file (16-bit PCM mono). + + Args: + duration_sec: length in seconds. + freq_hz: frequency in Hz (default 440 A4). + sample_rate: samples per second (default 16000 for speech). + + Returns: + Complete WAV file (RIFF header + PCM samples). + """ + num_samples = int(duration_sec * sample_rate) + # Generate sine samples (16-bit signed). + samples = [] + for i in range(num_samples): + t = i / sample_rate + value = int(32767 * 0.5 * math.sin(2 * math.pi * freq_hz * t)) + # Clamp to 16-bit range. + value = max(-32768, min(32767, value)) + samples.append(value) + + # Pack as little-endian 16-bit signed integers. + data = b"".join(v.to_bytes(2, byteorder="little", signed=True) for v in samples) + + # WAV header (RIFF container, PCM format). + byte_rate = sample_rate * 2 # 2 bytes per sample (16-bit). + block_align = 2 + subchunk2_size = len(data) + chunk_size = 36 + subchunk2_size + + wav = io.BytesIO() + wav.write(b"RIFF") + wav.write(chunk_size.to_bytes(4, byteorder="little")) + wav.write(b"WAVE") + + wav.write(b"fmt ") + wav.write((16).to_bytes(4, byteorder="little")) # subchunk1_size + wav.write((1).to_bytes(2, byteorder="little")) # audio_format (PCM) + wav.write((1).to_bytes(2, byteorder="little")) # num_channels (mono) + wav.write(sample_rate.to_bytes(4, byteorder="little")) + wav.write(byte_rate.to_bytes(4, byteorder="little")) + wav.write(block_align.to_bytes(2, byteorder="little")) + wav.write((16).to_bytes(2, byteorder="little")) # bits_per_sample + + wav.write(b"data") + wav.write(subchunk2_size.to_bytes(4, byteorder="little")) + wav.write(data) + + return wav.getvalue() + + +def check_openapi(base_url: str) -> bool: + """Check that GET /openapi.json lists both /v1/audio endpoints. + + Args: + base_url: base URL (e.g., http://localhost:8080). + + Returns: + True if both endpoints are listed; False otherwise. + """ + url = f"{base_url.rstrip('/')}/openapi.json" + try: + with urllib.request.urlopen(url, timeout=5) as resp: + if resp.status != 200: + print(f"FAIL: openapi.json returned {resp.status}") + return False + body = json.loads(resp.read().decode("utf-8")) + except (URLError, json.JSONDecodeError) as exc: + print(f"FAIL: openapi.json request failed: {exc}") + return False + + paths = body.get("paths", {}) + has_transcriptions = "/v1/audio/transcriptions" in paths + has_speech = "/v1/audio/speech" in paths + + if has_transcriptions and has_speech: + print("PASS: openapi.json lists /v1/audio/transcriptions and /v1/audio/speech") + return True + else: + print( + f"FAIL: openapi.json missing endpoints " + f"(transcriptions={has_transcriptions}, speech={has_speech})" + ) + return False + + +def check_transcription(base_url: str) -> bool: + """Test POST /v1/audio/transcriptions with a 2s 440 Hz tone. + + Args: + base_url: base URL (e.g., http://localhost:8080). + + Returns: + True if the endpoint returns 200 + valid JSON with 'text' key; False otherwise. + """ + url = f"{base_url.rstrip('/')}/v1/audio/transcriptions" + wav_data = build_wav_tone(duration_sec=2.0, freq_hz=440.0, sample_rate=16000) + + boundary = "----WebKitFormBoundary" + body = io.BytesIO() + body.write(f"--{boundary}\r\n".encode()) + body.write(b'Content-Disposition: form-data; name="file"; filename="test.wav"\r\n') + body.write(b"Content-Type: audio/wav\r\n\r\n") + body.write(wav_data) + body.write(f"\r\n--{boundary}--\r\n".encode()) + + req = urllib.request.Request(url, data=body.getvalue()) + req.add_header("Content-Type", f"multipart/form-data; boundary={boundary}") + + try: + with urllib.request.urlopen(req, timeout=30) as resp: + if resp.status != 200: + print(f"FAIL: transcriptions returned {resp.status}") + return False + response_body = json.loads(resp.read().decode("utf-8")) + except (URLError, json.JSONDecodeError) as exc: + print(f"FAIL: transcriptions request failed: {exc}") + return False + + if "text" in response_body: + text = response_body.get("text", "") + print(f"PASS: transcriptions returned 200 with text='{text}'") + return True + else: + print(f"FAIL: transcriptions response missing 'text' key: {response_body}") + return False + + +def main() -> int: + """Run all smoke tests. + + Returns: + 0 if all tests pass; 1 if any fail. + """ + parser = argparse.ArgumentParser( + description="Smoke test the model-gear audio realtime surface." + ) + parser.add_argument( + "--base-url", + default="http://localhost:8080", + help="Base URL of the realtime service (default: http://localhost:8080)", + ) + parser.add_argument( + "--stt-url", + help="Override STT URL for direct Parakeet testing (optional)", + ) + args = parser.parse_args() + + print(f"Testing audio surface at {args.base_url}") + print() + + results = [] + + # Test 1: OpenAPI schema + results.append(("openapi.json", check_openapi(args.base_url))) + + # Test 2: Transcription endpoint + results.append(("transcriptions", check_transcription(args.base_url))) + + print() + print("=" * 60) + passed = sum(1 for _, result in results if result) + total = len(results) + print(f"Results: {passed}/{total} checks passed") + + if passed == total: + print("SUCCESS: all audio surface checks passed") + return 0 + else: + print("FAILURE: some checks failed") + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_realtime_readiness.py b/tests/test_realtime_readiness.py new file mode 100644 index 0000000..8216e63 --- /dev/null +++ b/tests/test_realtime_readiness.py @@ -0,0 +1,54 @@ +"""Tests for the Parakeet readiness decision logic (stdlib + pytest only). + +No torch / nemo / fastapi imports — the module under test is stdlib-only so +these tests run in the offline CI environment (no GPU, no container deps). +Mirrors the style of tests/test_realtime_settings.py. +""" + +from __future__ import annotations + +import pytest + +from model_gear.realtime._readiness import evaluate_readiness + + +def test_both_ok_returns_200_ready() -> None: + code, body = evaluate_readiness(model_loaded=True, cuda_ok=True) + assert code == 200 + assert body == {"status": "ready"} + + +def test_model_not_loaded_returns_503_with_reason() -> None: + code, body = evaluate_readiness(model_loaded=False, cuda_ok=True) + assert code == 503 + assert body["status"] == "not_ready" + assert "model" in body["reason"].lower() + + +def test_cuda_not_available_returns_503_with_cuda_in_reason() -> None: + code, body = evaluate_readiness(model_loaded=True, cuda_ok=False) + assert code == 503 + assert body["status"] == "not_ready" + assert "cuda" in body["reason"].lower() + + +def test_neither_loaded_reports_503_model_reason_first() -> None: + """When both flags are False the model check fires first (load order).""" + code, body = evaluate_readiness(model_loaded=False, cuda_ok=False) + assert code == 503 + assert body["status"] == "not_ready" + assert "model" in body["reason"].lower() + + +@pytest.mark.parametrize( + "model_loaded, cuda_ok, expected_code", + [ + (True, True, 200), + (True, False, 503), + (False, True, 503), + (False, False, 503), + ], +) +def test_status_code_matrix(model_loaded: bool, cuda_ok: bool, expected_code: int) -> None: + code, _body = evaluate_readiness(model_loaded=model_loaded, cuda_ok=cuda_ok) + assert code == expected_code From 75c0ac6545074b9c549e662549faaf01af8263f7 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Fri, 12 Jun 2026 12:53:39 +0300 Subject: [PATCH 4/8] chore: bump version 0.19.0 -> 0.20.0 Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 32 ++++++++++++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63a78e3..f518a90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,38 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.20.0] - 2026-06-12 + +### Added + +- **`docs/realtime-pipeline.md`** — the previously-missing runbook for the audio + surface: that model-gear owns the live `:8080` realtime facade, the + `model init --fleet --audio` / `model fleet up` bring-up, the topology + (gateway path-routes `/v1/audio/*` → realtime → Parakeet/Magpie), the drift it + fixed (#39/#40), the cheap readiness probe, and the stale-Parakeet-CUDA restart + runbook. Resolves a doc referenced from `pyproject.toml`, the audio overlay, + and the realtime app docstring but never written. +- **`scripts/audio-smoke.py`** — a stdlib-only live smoke test for the audio + routes: asserts `GET :8080/openapi.json` lists both `/v1/audio/transcriptions` + and `/v1/audio/speech`, then POSTs an in-memory 16 kHz WAV and asserts + `200 {text: …}`. Reproduces issue #39's repro to confirm the 500→200 fix. + Requires a running GPU box (not a CI unit test). +- **`model_gear/realtime/_readiness.py`** — a stdlib-only `evaluate_readiness()` + helper backing the Parakeet `/v1/health/ready` cheap probe; unit-tested in CI + without torch/nemo/GPU. + +### Fixed + +- **Parakeet STT healthcheck now reflects real model readiness (#39).** The + vendored `templates/fleet/listen_server.py` `/v1/health/ready` returned + `{"status": "ready"}` unconditionally — process liveness only — so a container + whose CUDA context had gone stale (`CUDA error: unknown error`, every + transcription 500ing) still reported Docker "healthy". The probe now reports + ready **only** when the NeMo model is loaded **and** a trivial CUDA tensor op + succeeds, returning `503` otherwise (a cheap probe, not a full transcription + each interval). The pure decision is vendored into the Parakeet build context + and `COPY`'d into the image so it resolves without the wheel. + ## [0.19.0] - 2026-06-09 ### Added diff --git a/pyproject.toml b/pyproject.toml index 6c67d28..93f1627 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "model-gear" -version = "0.19.0" +version = "0.20.0" description = "model-gear — run, assess, and switch the local vLLM model." readme = "README.md" license = "MIT" From 19e4e7490ae5632a2a950daba0e3c387522bc213 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Fri, 12 Jun 2026 12:54:29 +0300 Subject: [PATCH 5/8] chore: sync uv.lock to 0.20.0 Co-Authored-By: Claude Opus 4.8 (1M context) --- uv.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/uv.lock b/uv.lock index 2470977..3b2a178 100644 --- a/uv.lock +++ b/uv.lock @@ -533,7 +533,7 @@ wheels = [ [[package]] name = "model-gear" -version = "0.19.0" +version = "0.20.0" source = { editable = "." } [package.optional-dependencies] From 50d32b52dc33bc8ec51681880d06eacafaa95ad0 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Fri, 12 Jun 2026 12:55:52 +0300 Subject: [PATCH 6/8] style: black-format listen_server.py (fix lint) Co-Authored-By: Claude Opus 4.8 (1M context) --- model_gear/templates/fleet/listen_server.py | 1 + 1 file changed, 1 insertion(+) diff --git a/model_gear/templates/fleet/listen_server.py b/model_gear/templates/fleet/listen_server.py index 16e15b4..aed4408 100644 --- a/model_gear/templates/fleet/listen_server.py +++ b/model_gear/templates/fleet/listen_server.py @@ -39,6 +39,7 @@ def evaluate_readiness(model_loaded: bool, cuda_ok: bool): # type: ignore[misc] return 503, {"status": "not_ready", "reason": "CUDA not available"} return 200, {"status": "ready"} + logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) From fcce9ad2b5ebee9d5d4d29e19051de297dde7336 Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Fri, 12 Jun 2026 12:57:36 +0300 Subject: [PATCH 7/8] ci: markdownlint ignores docs/specs + docs/plans (devague artifacts) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The devague-exported spec/plan H1 is the verbatim announcement sentence, which ends in a period (MD026/no-trailing-punctuation). These are generated planning artifacts like docs/superpowers/** — ignore them in lint. Co-Authored-By: Claude Opus 4.8 (1M context) --- .markdownlint-cli2.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.markdownlint-cli2.yaml b/.markdownlint-cli2.yaml index 029af35..d181a0e 100644 --- a/.markdownlint-cli2.yaml +++ b/.markdownlint-cli2.yaml @@ -20,3 +20,8 @@ ignores: # Internal planning artifacts (design specs, implementation plans) embed nested # fenced code blocks that trigger markdownlint violations not worth reformatting. - "docs/superpowers/**" + # devague-generated artifacts: the H1 is the verbatim announcement sentence + # (ends in a period → MD026) and the body mirrors confirmed claims as-is. + # Generated, not hand-authored — like docs/superpowers above. + - "docs/specs/**" + - "docs/plans/**" From 81de86a4191d8402564e336a3a45c6e70db6140a Mon Sep 17 00:00:00 2001 From: Ori Nachum Date: Fri, 12 Jun 2026 13:31:25 +0300 Subject: [PATCH 8/8] fix(audio): address Qodo review of #41 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Scaffold _readiness.py via AUDIO_TEMPLATES so `model init --fleet --audio` writes it (Dockerfile.parakeet COPYs it — clean init would otherwise fail `docker compose build stt`). [Qodo #2, comment 3402397497] - Remove the inline fallback copy of the readiness decision from listen_server.py and add a drift test asserting the vendored twin matches the canonical model_gear/realtime/_readiness.py. [Qodo #4] - Log CUDA readiness-probe failures (type + message) before returning 503 so operators can tell driver-down / OOM / stale-context apart. [Qodo #3] - audio-smoke.py now exercises /v1/audio/speech and wires --stt-url to a direct-Parakeet check (was claimed-but-untested). [Qodo #5] - docs/realtime-pipeline.md uses $HOME/.model-gear, not ~/.model-gear (portability rule). [Qodo #1, comment 3402397490] 275 tests pass; black/isort/flake8 + markdownlint clean. Bump 0.20.0 -> 0.20.1. Co-Authored-By: Claude Opus 4.8 (1M context) --- CHANGELOG.md | 25 +++++++++++ docs/realtime-pipeline.md | 4 +- model_gear/runtime/_compose.py | 4 ++ model_gear/templates/fleet/listen_server.py | 27 +++++------ pyproject.toml | 2 +- scripts/audio-smoke.py | 50 ++++++++++++++++++++- tests/test_init.py | 4 +- tests/test_realtime_readiness.py | 29 ++++++++++++ uv.lock | 2 +- 9 files changed, 124 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f518a90..0da1f43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,31 @@ All notable changes to this project are documented here. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [0.20.1] - 2026-06-12 + +### Fixed + +Qodo review of #41: + +- **`model init --fleet --audio` now scaffolds `_readiness.py`** — added + `fleet/_readiness.py → _readiness.py` to `_compose.AUDIO_TEMPLATES`. The + Parakeet `Dockerfile.parakeet` `COPY _readiness.py` requires it at the + deployment-dir root, so a clean audio init previously produced a tree where + `docker compose build stt` would fail. Covered by `test_init.py`. +- **Parakeet readiness drift guard + simplification** — removed the third + (inline) copy of the readiness decision from `listen_server.py` (the scaffold + now guarantees the vendored `_readiness.py` is present), and added a test + asserting the vendored twin stays behaviourally identical to the canonical + `model_gear/realtime/_readiness.py`. +- **CUDA readiness probe failures are now logged** — `listen_server.health()` + emits a `logger.warning` with the exception type/message before returning + `503`, so operators can distinguish driver-down / OOM / stale-context. +- **`scripts/audio-smoke.py` now exercises `/v1/audio/speech`** (it previously + claimed both routes but only tested transcriptions) and wires the formerly + unused `--stt-url` to a direct-Parakeet transcription check. +- **`docs/realtime-pipeline.md`** uses `$HOME/.model-gear` instead of the + non-portable `~/.model-gear`. + ## [0.20.0] - 2026-06-12 ### Added diff --git a/docs/realtime-pipeline.md b/docs/realtime-pipeline.md index 1ccf56d..8d95f16 100644 --- a/docs/realtime-pipeline.md +++ b/docs/realtime-pipeline.md @@ -59,7 +59,7 @@ Both STT and TTS share the GPU with the two LLM backends. model init --fleet --audio --apply # 2. Set NGC_API_KEY in the .env file -# Locate ~/.model-gear/.env (or your --compose-dir) and edit: +# Locate $HOME/.model-gear/.env (or your --compose-dir) and edit: # NGC_API_KEY= # 3. Bring up the full audio stack (dry-run by default; --apply commits) @@ -73,7 +73,7 @@ Each `model init` and `model fleet` verb defaults to **dry-run**; omit `--apply` to see what would happen, or add `--apply` to execute. This ensures safe-by-default operation (useful when agents call CLIs in loops). -To customize the compose dir (default `$MODEL_GEAR_DIR` or `~/.model-gear`): +To customize the compose dir (default `$MODEL_GEAR_DIR` or `$HOME/.model-gear`): ```bash model init --fleet --audio --compose-dir /path/to/deployment --apply diff --git a/model_gear/runtime/_compose.py b/model_gear/runtime/_compose.py index 27975f2..dc7bad4 100644 --- a/model_gear/runtime/_compose.py +++ b/model_gear/runtime/_compose.py @@ -62,6 +62,10 @@ "fleet/Dockerfile.realtime": "Dockerfile.realtime", "fleet/Dockerfile.parakeet": "Dockerfile.parakeet", "fleet/listen_server.py": "listen_server.py", + # _readiness.py is COPY'd into the Parakeet image (Dockerfile.parakeet), so + # it MUST land at the deployment-dir root or `docker compose build stt` + # fails on the COPY. Vendored twin of model_gear/realtime/_readiness.py. + "fleet/_readiness.py": "_readiness.py", } AUDIO_ENV_TEMPLATE = "fleet/env.audio.example" # Back-compat alias: the single set was the only one before the fleet existed. diff --git a/model_gear/templates/fleet/listen_server.py b/model_gear/templates/fleet/listen_server.py index aed4408..1aaa0a4 100644 --- a/model_gear/templates/fleet/listen_server.py +++ b/model_gear/templates/fleet/listen_server.py @@ -20,24 +20,16 @@ from fastapi import FastAPI, File, Form, UploadFile from fastapi.responses import JSONResponse -# Import the readiness decision logic. The Dockerfile COPYs _readiness.py -# next to listen_server.py (as a top-level module in /app), so this import -# works whether or not the model_gear wheel is installed in the container. -# Falls back gracefully if neither is present (shouldn't happen, but defensive). +# Import the readiness decision from the single source of truth. The Dockerfile +# COPYs _readiness.py next to listen_server.py (top-level module in /app) and +# `model init --fleet --audio` scaffolds it via AUDIO_TEMPLATES, so the +# container-local import always resolves; the wheel path is a dev fallback. No +# inline copy — a third copy would invite drift from the CI-tested canonical +# model_gear/realtime/_readiness.py. try: from _readiness import evaluate_readiness # container-local copy (top-level) except ImportError: - try: - from model_gear.realtime._readiness import evaluate_readiness # wheel install - except ImportError: - # Last-resort inline fallback — keeps the container functional even if - # both import paths fail (e.g. a dev build without the COPY step). - def evaluate_readiness(model_loaded: bool, cuda_ok: bool): # type: ignore[misc] - if not model_loaded: - return 503, {"status": "not_ready", "reason": "model not loaded"} - if not cuda_ok: - return 503, {"status": "not_ready", "reason": "CUDA not available"} - return 200, {"status": "ready"} + from model_gear.realtime._readiness import evaluate_readiness # wheel install logging.basicConfig(level=logging.INFO) @@ -87,7 +79,10 @@ async def health(): torch.zeros(1, device="cuda") torch.cuda.synchronize() cuda_ok = True - except Exception: + except Exception as exc: + # Log the failure so operators can tell driver-down from OOM from a + # stale CUDA context (the #39 symptom) instead of a silent 503. + logger.warning("CUDA readiness probe failed: %s: %s", type(exc).__name__, exc) cuda_ok = False status_code, body = evaluate_readiness(model_loaded, cuda_ok) diff --git a/pyproject.toml b/pyproject.toml index 93f1627..8bd718a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "model-gear" -version = "0.20.0" +version = "0.20.1" description = "model-gear — run, assess, and switch the local vLLM model." readme = "README.md" license = "MIT" diff --git a/scripts/audio-smoke.py b/scripts/audio-smoke.py index b7f8bdc..f7975fa 100644 --- a/scripts/audio-smoke.py +++ b/scripts/audio-smoke.py @@ -7,7 +7,8 @@ Reproduces the issue #39 repro to confirm the 500→200 fix: generates a 2s 440 Hz tone (16 kHz mono PCM16 WAV), posts it to /v1/audio/transcriptions, and asserts -HTTP 200 + a JSON response with a `text` key. +HTTP 200 + a JSON response with a `text` key. Also exercises /v1/audio/speech +(Magpie TTS) and, when --stt-url is given, the Parakeet backend directly. Exit code 0 if all checks pass; non-zero on any failure. """ @@ -153,6 +154,42 @@ def check_transcription(base_url: str) -> bool: return False +def check_speech(base_url: str) -> bool: + """Test POST /v1/audio/speech (OpenAI TTS → Magpie) returns audio. + + Args: + base_url: base URL (e.g., http://localhost:8080). + + Returns: + True if the endpoint returns 200 with a non-empty audio body; else False. + """ + url = f"{base_url.rstrip('/')}/v1/audio/speech" + payload = json.dumps({"input": "hey reachy", "voice": "Mia.Calm"}).encode("utf-8") + req = urllib.request.Request(url, data=payload, method="POST") + req.add_header("Content-Type", "application/json") + + try: + with urllib.request.urlopen(req, timeout=30) as resp: + if resp.status != 200: + print(f"FAIL: speech returned {resp.status}") + return False + content_type = resp.headers.get("Content-Type", "") + body_len = len(resp.read()) + except URLError as exc: + print(f"FAIL: speech request failed: {exc}") + return False + + if body_len > 0 and "audio" in content_type: + print(f"PASS: speech returned 200 ({body_len} bytes, {content_type})") + return True + else: + print( + f"FAIL: speech returned 200 but body/type unexpected " + f"(bytes={body_len}, content_type={content_type!r})" + ) + return False + + def main() -> int: """Run all smoke tests. @@ -181,9 +218,18 @@ def main() -> int: # Test 1: OpenAPI schema results.append(("openapi.json", check_openapi(args.base_url))) - # Test 2: Transcription endpoint + # Test 2: Transcription endpoint (facade → Parakeet) results.append(("transcriptions", check_transcription(args.base_url))) + # Test 3: Speech endpoint (facade → Magpie) + results.append(("speech", check_speech(args.base_url))) + + # Test 4 (optional): Parakeet STT directly, when --stt-url is given. This is + # the issue-#39 repro against the backend itself, bypassing the facade. + if args.stt_url: + print(f"\nTesting Parakeet STT directly at {args.stt_url}") + results.append(("stt-direct", check_transcription(args.stt_url))) + print() print("=" * 60) passed = sum(1 for _, result in results if result) diff --git a/tests/test_init.py b/tests/test_init.py index 25b3828..52a63d0 100644 --- a/tests/test_init.py +++ b/tests/test_init.py @@ -130,7 +130,8 @@ def test_init_fleet_audio_apply_writes_overlay_and_appends_env(tmp_path) -> None target = tmp_path / "fa" rc = main(["init", "--fleet", "--audio", str(target), "--apply"]) assert rc == 0 - # fleet files + the four audio overlay files + # fleet files + the audio overlay files. _readiness.py MUST be scaffolded: + # Dockerfile.parakeet COPYs it, so a missing scaffold breaks `build stt`. for name in ( "docker-compose.yml", "Dockerfile.gateway", @@ -138,6 +139,7 @@ def test_init_fleet_audio_apply_writes_overlay_and_appends_env(tmp_path) -> None "Dockerfile.realtime", "Dockerfile.parakeet", "listen_server.py", + "_readiness.py", ): assert (target / name).is_file(), name env = (target / ".env").read_text() diff --git a/tests/test_realtime_readiness.py b/tests/test_realtime_readiness.py index 8216e63..399143e 100644 --- a/tests/test_realtime_readiness.py +++ b/tests/test_realtime_readiness.py @@ -7,10 +7,27 @@ from __future__ import annotations +import importlib.util +from pathlib import Path + import pytest from model_gear.realtime._readiness import evaluate_readiness +# The vendored twin COPY'd into the Parakeet image / scaffolded by +# `model init --fleet --audio`. It must stay behaviourally identical to the +# canonical module (cite-don't-import — two copies, one truth). +_VENDORED_TWIN = ( + Path(__file__).resolve().parents[1] / "model_gear" / "templates" / "fleet" / "_readiness.py" +) + + +def _load_vendored_evaluate_readiness(): + spec = importlib.util.spec_from_file_location("_fleet_readiness", _VENDORED_TWIN) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.evaluate_readiness + def test_both_ok_returns_200_ready() -> None: code, body = evaluate_readiness(model_loaded=True, cuda_ok=True) @@ -52,3 +69,15 @@ def test_neither_loaded_reports_503_model_reason_first() -> None: def test_status_code_matrix(model_loaded: bool, cuda_ok: bool, expected_code: int) -> None: code, _body = evaluate_readiness(model_loaded=model_loaded, cuda_ok=cuda_ok) assert code == expected_code + + +@pytest.mark.parametrize( + "model_loaded, cuda_ok", + [(True, True), (True, False), (False, True), (False, False)], +) +def test_vendored_twin_matches_canonical(model_loaded: bool, cuda_ok: bool) -> None: + """The fleet-template copy must not drift from the canonical decision.""" + vendored = _load_vendored_evaluate_readiness() + assert vendored(model_loaded=model_loaded, cuda_ok=cuda_ok) == evaluate_readiness( + model_loaded=model_loaded, cuda_ok=cuda_ok + ) diff --git a/uv.lock b/uv.lock index 3b2a178..16c7b0f 100644 --- a/uv.lock +++ b/uv.lock @@ -533,7 +533,7 @@ wheels = [ [[package]] name = "model-gear" -version = "0.20.0" +version = "0.20.1" source = { editable = "." } [package.optional-dependencies]