feat(core+docs): enable startup warmup with token cap adjustments and stop token resolution

- Enabled the ASR startup warmup by default, setting `STARTUP_WARMUP=1` and `STARTUP_WARMUP_TOKENS=512`.
- Refined warmup token logic to use `MAX_NEW_TOKENS` as a fallback default for consistency.
- Introduced stop token ID resolution in the ASR inference pipeline for deterministic behavior.
- Updated Dockerfile, Taskfile, README, and documentation to reflect changes in warmup behavior and token handling.
- Improved health readiness checks to account for startup warmup impact.

Author: TheMasterOfDisasters

Dockerfile

@@ -66,8 +66,8 @@ ENV PYTHONDONTWRITEBYTECODE=1 \
     QWEN_ASR_CUDAGRAPH_MODE=PIECEWISE \
     QWEN_ASR_CUDAGRAPH_CAPTURE_SIZES=1,2 \
     QWEN_ASR_MAX_CUDAGRAPH_CAPTURE_SIZE=2 \
-    QWEN_ASR_STARTUP_WARMUP=0 \
-    QWEN_ASR_STARTUP_WARMUP_TOKENS=1 \
+    QWEN_ASR_STARTUP_WARMUP=1 \
+    QWEN_ASR_STARTUP_WARMUP_TOKENS=512 \
     QWEN_ASR_TRACE_REQUESTS=0 \
     PORT=8000 \
     HOST=0.0.0.0

README.md

@@ -203,6 +203,8 @@ Common environment variables:
 | `QWEN_ASR_MAX_NUM_BATCHED_TOKENS` | `2048` | vLLM batch-token cap |
 | `QWEN_ASR_MAX_INFERENCE_BATCH_SIZE` | `2` | ASR inference batch cap |
 | `QWEN_ASR_MAX_NEW_TOKENS` | `512` | Max generated tokens |
+| `QWEN_ASR_STARTUP_WARMUP` | `1` | Run a decode warmup before the service reports healthy |
+| `QWEN_ASR_STARTUP_WARMUP_TOKENS` | `512` | Token cap used by startup warmup |
 | `QWEN_ASR_PERFORMANCE_PROFILE` | `balanced` | Startup/runtime graph profile |
 | `VLLM_CACHE_ROOT` | `/app/.cache/vllm` | vLLM/Torch compile cache path |
 

Taskfile.yml

@@ -45,8 +45,8 @@ vars:
   CUDAGRAPH_MODE: 'PIECEWISE'
   CUDAGRAPH_CAPTURE_SIZES: '1,2'
   MAX_CUDAGRAPH_CAPTURE_SIZE: '2'
-  STARTUP_WARMUP: '0'
-  STARTUP_WARMUP_TOKENS: '1'
+  STARTUP_WARMUP: '1'
+  STARTUP_WARMUP_TOKENS: '512'
   TRACE_REQUESTS: '0'
   BACKEND_KWARGS: ''
   ALIGNER_KWARGS: ''

docs/dockerhub.md

@@ -181,9 +181,11 @@ Common knobs:
 - `QWEN_ASR_MAX_MODEL_LEN=2048`
 - `QWEN_ASR_MAX_NUM_BATCHED_TOKENS=2048`
 - `QWEN_ASR_MAX_NEW_TOKENS=512`
+- `QWEN_ASR_STARTUP_WARMUP=1`
+- `QWEN_ASR_STARTUP_WARMUP_TOKENS=512`
 - `QWEN_ASR_PERFORMANCE_PROFILE=balanced|throughput|custom`
 
-Decoding temperature is fixed at `0` for deterministic transcription.
+Decoding temperature is fixed at `0` for deterministic transcription. Startup warmup intentionally makes `/health` wait until the normal decode path is ready, so the first API transcription after readiness does not pay vLLM's lazy generation cost.
 
 ## Responsible Use and Privacy
 

qwen_asr/inference/qwen3_asr.py

@@ -36,6 +36,32 @@
 def _trace_requests_enabled() -> bool:
     return os.getenv("QWEN_ASR_TRACE_REQUESTS", "0").strip().lower() in {"1", "true", "yes", "y"}
 
+
+def _resolve_stop_token_ids(processor: Any) -> List[int]:
+    tokenizer = getattr(processor, "tokenizer", None)
+    if tokenizer is None:
+        return []
+
+    token_ids: List[int] = []
+    for token in ("<|im_end|>", "<|endoftext|>"):
+        try:
+            token_id = tokenizer.convert_tokens_to_ids(token)
+        except Exception:
+            continue
+        if isinstance(token_id, int) and token_id >= 0 and token_id not in token_ids:
+            token_ids.append(token_id)
+
+    eos_token_id = getattr(tokenizer, "eos_token_id", None)
+    if isinstance(eos_token_id, int) and eos_token_id >= 0 and eos_token_id not in token_ids:
+        token_ids.append(eos_token_id)
+
+    pad_token_id = getattr(tokenizer, "pad_token_id", None)
+    if isinstance(pad_token_id, int) and pad_token_id >= 0 and pad_token_id not in token_ids:
+        token_ids.append(pad_token_id)
+
+    return token_ids
+
+
 from .qwen3_forced_aligner import Qwen3ForcedAligner
 from .utils import (
     MAX_ASR_INPUT_SECONDS,
@@ -291,7 +317,12 @@ def LLM(
         # ASR/translation must remain deterministic: preserve the upstream
         # explicit zero-temperature sampling instead of falling back to model
         # generation config defaults.
-        sampling_params = SamplingParams(temperature=0.0, max_tokens=max_new_tokens)
+        stop_token_ids = _resolve_stop_token_ids(processor)
+        sampling_kwargs: Dict[str, Any] = {"temperature": 0.0, "max_tokens": max_new_tokens}
+        if stop_token_ids:
+            sampling_kwargs["stop_token_ids"] = stop_token_ids
+            log_startup(f"vLLM stop_token_ids: {stop_token_ids}")
+        sampling_params = SamplingParams(**sampling_kwargs)
 
         forced_aligner_model = None
         if forced_aligner is not None:
@@ -333,7 +364,14 @@ def warm_up(self, *, max_new_tokens: int = 1) -> None:
 
         sampling_cls = type(original_sampling_params)
         try:
-            self.sampling_params = sampling_cls(temperature=0.0, max_tokens=max(1, int(max_new_tokens)))
+            sampling_kwargs: Dict[str, Any] = {
+                "temperature": 0.0,
+                "max_tokens": max(1, int(max_new_tokens)),
+            }
+            stop_token_ids = getattr(original_sampling_params, "stop_token_ids", None)
+            if stop_token_ids:
+                sampling_kwargs["stop_token_ids"] = list(stop_token_ids)
+            self.sampling_params = sampling_cls(**sampling_kwargs)
             silence = np.zeros((SAMPLE_RATE // 2,), dtype=np.float32)
             self.transcribe(audio=(silence, SAMPLE_RATE), language="English", return_time_stamps=False)
         finally:

qwen_asr/server/app.py

@@ -821,7 +821,7 @@ def run_server(
 
     warmup_enabled = os.getenv("QWEN_ASR_STARTUP_WARMUP", "0").strip().lower() in {"1", "true", "yes", "y"}
     if warmup_enabled:
-        warmup_tokens = int(os.getenv("QWEN_ASR_STARTUP_WARMUP_TOKENS", "1"))
+        warmup_tokens = int(os.getenv("QWEN_ASR_STARTUP_WARMUP_TOKENS", os.getenv("QWEN_ASR_MAX_NEW_TOKENS", "512")))
         with StartupTimer(f"startup ASR warmup max_new_tokens={warmup_tokens}"):
             asr.warm_up(max_new_tokens=warmup_tokens)
 

qwen_asr/server/openai_api.py

@@ -610,7 +610,7 @@ def run_server(
 
     warmup_enabled = os.getenv("QWEN_ASR_STARTUP_WARMUP", "0").strip().lower() in {"1", "true", "yes", "y"}
     if warmup_enabled:
-        warmup_tokens = int(os.getenv("QWEN_ASR_STARTUP_WARMUP_TOKENS", "1"))
+        warmup_tokens = int(os.getenv("QWEN_ASR_STARTUP_WARMUP_TOKENS", os.getenv("QWEN_ASR_MAX_NEW_TOKENS", "512")))
         with StartupTimer(f"startup ASR warmup max_new_tokens={warmup_tokens}"):
             asr.warm_up(max_new_tokens=warmup_tokens)