feat: keep models in memory, offload them when idle, and give whisper.cpp a resident worker

.env.example

@@ -180,6 +180,13 @@ VOCAGATEWAY_PUBLISH_PORT=8765
 # choose in the WebUI unless you specifically want an environment-level lock.
 #VOCAGATEWAY_ENGINE=auto
 
+# How hard whisper.cpp searches while decoding. `quality` keeps the narrowed
+# beam search (beam 2, best-of 2); `fast` decodes greedily, which cuts decoder
+# work on a CPU-only host and can cost accuracy on accented or noisy audio.
+# Temperature fallback stays on either way. Compare word error rate on your own
+# recordings before keeping `fast`.
+#VOCAGATEWAY_WHISPER_DECODER_PRESET=quality
+
 # How long a failed session's audio is kept so the phone can retry, and whether
 # audio is deleted immediately after a successful transcription.
 #VOCAGATEWAY_RETENTION_HOURS=24
@@ -222,11 +229,12 @@ VOCAGATEWAY_PUBLISH_PORT=8765
 #       remapping the `vocagateway-data` volume, not by setting these.
 #
 #   VOCAGATEWAY_WHISPER_BINARY, VOCAGATEWAY_WHISPER_MODEL,
+#   VOCAGATEWAY_WHISPER_SERVER_BINARY,
 #   VOCAGATEWAY_WHISPERKIT_BINARY, VOCAGATEWAY_VOCAMAC_APP,
 #   VOCAGATEWAY_VOCAMAC_MODEL, VOCAGATEWAY_HANDY_BINARY,
 #   VOCAGATEWAY_HANDY_MODEL, VOCAGATEWAY_HANDY_FALLBACK_MODEL
-#       Native-only. The image pins its own whisper.cpp CLI, and the macOS
-#       engines do not exist in a Linux container.
+#       Native-only. The image pins its own whisper.cpp CLI and resident
+#       worker, and the macOS engines do not exist in a Linux container.
 #
 # And one that must not go here even though Compose would use it:
 #

Dockerfile

@@ -117,6 +117,7 @@ RUN curl --fail --location --show-error \
 # The default target is built rather than `--target whisper-cli`: the variant
 # backends are standalone dlopen'd modules that nothing links against, so a
 # whisper-cli-only build would produce a binary with no CPU backend to load.
+# It also yields whisper-server, the resident worker the gateway prefers.
 RUN --mount=type=cache,id=ccache-${ACCEL},target=/root/.cache/ccache \
     cmake -S . -B build \
       -DCMAKE_BUILD_TYPE=Release \
@@ -137,11 +138,14 @@ RUN --mount=type=cache,id=ccache-${ACCEL},target=/root/.cache/ccache \
 
 # Sort the build output the way the runtime expects it: the linked libraries
 # onto the loader path, the dlopen'd backends into GGML_BACKEND_DIR. Only
-# whisper-cli is taken from the examples; the rest of the default target is
-# build fallout.
+# whisper-cli and whisper-server are taken from the examples; the rest of the
+# default target is build fallout. whisper-server is what keeps the model and
+# the accelerator context resident between requests — the gateway starts it on
+# a private loopback port and falls back to whisper-cli when it is missing, so
+# both have to come from this one build.
 RUN set -eu; \
     mkdir -p /out/bin /out/lib/ggml; \
-    cp build/bin/whisper-cli /out/bin/; \
+    cp build/bin/whisper-cli build/bin/whisper-server /out/bin/; \
     for lib in build/bin/*.so*; do \
       case "${lib##*/}" in \
         libggml-cpu*|libggml-blas*|libggml-cuda*|libggml-vulkan*) \

README.md

@@ -148,7 +148,10 @@ MLX Audio and WhisperKit are recommended on Apple silicon. The `apple` extra
 installs MLX only on an arm64 Mac. It is left out of Linux and Docker on
 purpose. The standalone `whisper.cpp` engine uses the `whisper-cli` binary
 installed above (override its location with `VOCAGATEWAY_WHISPER_BINARY`). On a
-native Linux host it is optional and can be built from source instead.
+native Linux host it is optional and can be built from source instead. When the
+same build also ships `whisper-server` next to `whisper-cli`, the engine keeps
+the model resident in that worker instead of reloading it for every clip; see
+[Health and readiness](#health-and-readiness).
 
 ## Native Linux quick start
 
@@ -631,6 +634,8 @@ uv run vocagateway
 | `VOCAGATEWAY_ENGINE` | `auto` | `auto` | `auto`, `vocamac`, `handy`, `mlx-audio`, `whisperkit`, `sherpa-onnx`, `faster-whisper`, `moonshine`, or `whisper.cpp` |
 | `VOCAGATEWAY_WHISPER_BINARY` | `/opt/homebrew/bin/whisper-cli` | `/usr/local/bin/whisper-cli` | `whisper.cpp` executable |
 | `VOCAGATEWAY_WHISPER_MODEL` | `~/.local/share/whisper.cpp/models/ggml-base.en.bin` | same, and normally absent | Fallback `whisper.cpp` model used only when no model is selected in the WebUI |
+| `VOCAGATEWAY_WHISPER_SERVER_BINARY` | the `whisper-server` beside `whisper-cli`, else `PATH` | `/usr/local/bin/whisper-server` | Resident `whisper.cpp` worker; unset is normal, and a missing binary falls back to one `whisper-cli` run per request |
+| `VOCAGATEWAY_WHISPER_DECODER_PRESET` | `quality` | `quality` | `quality` keeps the narrowed beam search; `fast` decodes greedily — cheaper on a CPU-only host, and worth a WER comparison on your own audio before you keep it |
 | `VOCAGATEWAY_WHISPERKIT_BINARY` | `whisperkit-cli` | unavailable | Standalone WhisperKit executable and legacy VocaMac fallback |
 | `VOCAGATEWAY_VOCAMAC_APP` | `/Applications/VocaMac.app` | unavailable | Optional VocaMac app bundle |
 | `VOCAGATEWAY_VOCAMAC_MODEL` | unset | unset | Pin a VocaMac model instead of following the app's choice |
@@ -819,10 +824,32 @@ old URL.
 Engine probes are cached for five seconds. sherpa-onnx, MLX Audio,
 `faster-whisper`, and Moonshine load their selected model once and keep it
 resident. WhisperKit warmup starts its managed loopback service and keeps the
-Core ML model resident there. VocaMac 0.8.0+ headless transcription is one-shot,
+Core ML model resident there. `whisper.cpp` does the same when the build ships
+`whisper-server`: warmup prefetches the model file and then starts a private
+worker on an ephemeral `127.0.0.1` port, which holds the parsed model and the
+CUDA/Vulkan/Metal context between requests. Nothing extra is published and no
+host port is added. A build without that binary, or a worker that fails to
+start, falls back to one `whisper-cli` run per transcription — the behavior
+every earlier release had. VocaMac 0.8.0+ headless transcription is one-shot,
 so its first-load cost is included in each request; older WhisperKit-only
-VocaMac builds retain the persistent compatibility path. Handy and
-`whisper.cpp` retain the filesystem-prefetch warmup behavior.
+VocaMac builds retain the persistent compatibility path. Handy retains the
+filesystem-prefetch warmup behavior.
+
+Choosing **Load** on a downloaded model waits for that warmup before the Models
+view reports it active, so resident engines do not defer their model load to the
+first transcription. In **Settings → Speech engine**, **Offload model when idle**
+can release a resident model after 10, 15, 30, 60, or 120 minutes without a
+transcription. It is off by default. The selected model stays selected and loads
+again automatically on the next transcription. Saving a change to this setting
+alone keeps whatever is already loaded — only a change that actually selects a
+different engine, device, precision, thread count, or model rebuilds it. Active
+batch and streaming jobs hold a model lease, so neither the idle monitor nor a
+settings change unloads an engine while a transcription is still using it: the
+replaced engine is closed once its last request finishes. The
+setting also releases the `whisper.cpp` worker: the process is terminated and
+the next transcription starts it again, paying one model load. It does not apply
+to one-shot Handy or current VocaMac headless processes, because those engines
+keep no model resident between requests.
 
 ## Docker performance profiles
 

app/admin_queries.py

@@ -9,6 +9,7 @@
 from app.catalog import catalog_source_url, language_names, recommended_ids
 from app.context import BOOTSTRAP_TOKEN_ID, TOKEN_FILE_HINT, VERSION, GatewayContext
 from app.engine_state import active_model_path, available_engines, engine_id
+from app.runtime_config import DEFAULT_IDLE_OFFLOAD_MINUTES
 from app.serializers import metrics_status, model_covers
 from app.system import detect_system
 
@@ -200,6 +201,11 @@ def build_entry(self, model: Any) -> schemas.AdminModelEntry:
         inst = self.installed.get(model.id)
         resolution = self._resolve_state(download, inst)
         is_active = inst is not None and inst.path == self.active_path
+        is_offloaded = bool(
+            is_active
+            and self.ctx.engine_manager is not None
+            and self.ctx.engine_manager.model_is_offloaded
+        )
         return schemas.AdminModelEntry(
             id=model.id,
             engine=model.engine,
@@ -219,6 +225,7 @@ def build_entry(self, model: Any) -> schemas.AdminModelEntry:
             language_codes=list(model.language_codes),
             state=resolution[0],
             active=is_active,
+            offloaded=is_offloaded,
             recommended=model.id in self.recommended,
             progress=resolution[1],
             downloaded_bytes=download.downloaded_bytes if download else None,
@@ -243,6 +250,11 @@ def build_custom_entry(self, custom: Any) -> schemas.AdminModelEntry:
             source="Local file",
             state="installed",
             active=custom.path == self.active_path,
+            offloaded=bool(
+                custom.path == self.active_path
+                and self.ctx.engine_manager is not None
+                and self.ctx.engine_manager.model_is_offloaded
+            ),
             recommended=False,
         )
 
@@ -386,6 +398,8 @@ def config_response(ctx: GatewayContext) -> schemas.ConfigResponse:
             compute_device=rc.compute_device,
             compute_type=rc.compute_type,
             cpu_threads=rc.cpu_threads,
+            idle_offload_enabled=rc.idle_offload_enabled,
+            idle_offload_minutes=rc.idle_offload_minutes,
         )
     return schemas.ConfigResponse(
         engine="custom",
@@ -400,4 +414,6 @@ def config_response(ctx: GatewayContext) -> schemas.ConfigResponse:
         compute_device="auto",
         compute_type="auto",
         cpu_threads=0,
+        idle_offload_enabled=False,
+        idle_offload_minutes=DEFAULT_IDLE_OFFLOAD_MINUTES,
     )

app/config.py

@@ -43,6 +43,11 @@ def _env_path(name: str, default: Path) -> Path:
     return Path(configured_path).expanduser()
 
 
+def _optional_path(name: str) -> Path | None:
+    configured_path = _env(name)
+    return Path(configured_path).expanduser() if configured_path else None
+
+
 def _default_token_file() -> Path:
     base = Path(os.environ.get("XDG_CONFIG_HOME", "~/.config")).expanduser()
     return base / APP_DIR_NAME / "token"
@@ -66,6 +71,13 @@ class Settings:
     vocamac_app: Path = Path("/Applications/VocaMac.app")
     vocamac_model: str | None = None
     whisperkit_binary: str = "whisperkit-cli"
+    # Optional override for the `whisper-server` that keeps a whisper.cpp model
+    # resident. Unset means "the sibling of whisper_binary, else PATH".
+    whisper_server_binary: Path | None = None
+    # `quality` (the default) keeps whisper.cpp's narrowed beam search;
+    # `fast` decodes greedily, which is cheaper on a CPU-only host and may cost
+    # accuracy on accented or noisy audio. See app/models/whisper_cpp.py.
+    whisper_decoder_preset: str = "quality"
     models_dir: Path | None = None
     config_path: Path = Path("~/.config/vocagateway/config.json")
     token_file: Path = Path("~/.config/vocagateway/token")
@@ -131,6 +143,8 @@ def from_env(cls) -> Settings:
             ).expanduser(),
             vocamac_model=_env("VOCAGATEWAY_VOCAMAC_MODEL") or None,
             whisperkit_binary=_env("VOCAGATEWAY_WHISPERKIT_BINARY", "whisperkit-cli"),
+            whisper_server_binary=_optional_path("VOCAGATEWAY_WHISPER_SERVER_BINARY"),
+            whisper_decoder_preset=_env("VOCAGATEWAY_WHISPER_DECODER_PRESET", "quality").lower(),
             models_dir=Path(models_override).expanduser()
             if models_override
             else data_dir / "models",