Add per-stream language hint for Qwen3-ASR (and tighten scaffold cleanup) (#3472)

Author: kapitalismho

cxx-api-examples/qwen3-asr-cxx-api.cc

@@ -74,6 +74,7 @@ int32_t main(int32_t argc, char *argv[]) {
   const auto begin = std::chrono::steady_clock::now();
 
   OfflineStream stream = recognizer.CreateStream();
+  // stream.SetOption("language", "Korean");  // optional: force transcription language
   stream.AcceptWaveform(wave.sample_rate, wave.samples.data(),
                         wave.samples.size());
 

python-api-examples/offline-qwen3-asr-decode-files.py

@@ -79,6 +79,14 @@ def get_args():
         """,
     )
 
+    parser.add_argument(
+        "--language",
+        type=str,
+        default="",
+        required=False,
+        help="Force transcription language, e.g. Korean, Chinese, English",
+    )
+
     parser.add_argument(
         "--max-total-len",
         type=int,
@@ -179,11 +187,15 @@ def create_recognizer(args) -> sherpa_onnx.OfflineRecognizer:
     )
 
 
-def decode_file(recognizer: sherpa_onnx.OfflineRecognizer, filename: str):
+def decode_file(
+    recognizer: sherpa_onnx.OfflineRecognizer, filename: str, language: str = ""
+):
     audio, sample_rate = sf.read(filename, dtype="float32", always_2d=True)
     audio = audio[:, 0]
 
     stream = recognizer.create_stream()
+    if language:
+        stream.set_option("language", language)
     stream.accept_waveform(sample_rate, audio)
     recognizer.decode_stream(stream)
     return stream.result
@@ -200,7 +212,7 @@ def main():
         if not Path(f).is_file():
             print(f"Skip missing file: {f}", file=sys.stderr)
             continue
-        result = decode_file(recognizer, f)
+        result = decode_file(recognizer, f, language=args.language)
         print(f"{f}\n  text: {result.text}\n")
 
 

sherpa-onnx/csrc/offline-recognizer-qwen3-asr-impl.cc

@@ -356,10 +356,17 @@ void OfflineRecognizerQwen3ASRImpl::InitPromptTemplateIds() {
     SHERPA_ONNX_LOGE("Failed to tokenize <|audio_pad|> for qwen3-asr prompt");
     SHERPA_ONNX_EXIT(-1);
   }
+
+  asr_text_token_id_ = tokenizer_->GetTokenId("<asr_text>");
+  if (asr_text_token_id_ < 0) {
+    SHERPA_ONNX_LOGE("Failed to locate <asr_text> token id for qwen3-asr");
+    SHERPA_ONNX_EXIT(-1);
+  }
 }
 
 std::vector<int64_t> OfflineRecognizerQwen3ASRImpl::BuildSourceIds(
-    const std::string &hotwords, int32_t audio_token_len, int32_t *before_len,
+    const std::string &hotwords, const std::string &language,
+    int32_t audio_token_len, int32_t *before_len,
     int32_t *fake_audio_token_len) const {
   const std::string before_utf8 = std::string(kQwen3SystemPromptPrefix) +
                                   hotwords + kQwen3SystemPromptSuffix;
@@ -372,11 +379,27 @@ std::vector<int64_t> OfflineRecognizerQwen3ASRImpl::BuildSourceIds(
     *fake_audio_token_len = audio_token_len;
   }
 
+  std::vector<int64_t> prompt_ids_after_with_language;
+  const std::vector<int64_t> *ids_after = &prompt_ids_after_;
+  if (!language.empty()) {
+    auto language_ids = tokenizer_->Encode("language " + language);
+    prompt_ids_after_with_language.reserve(prompt_ids_after_.size() +
+                                           language_ids.size() + 1);
+    prompt_ids_after_with_language.insert(prompt_ids_after_with_language.end(),
+                                          prompt_ids_after_.begin(),
+                                          prompt_ids_after_.end());
+    prompt_ids_after_with_language.insert(prompt_ids_after_with_language.end(),
+                                          language_ids.begin(),
+                                          language_ids.end());
+    prompt_ids_after_with_language.push_back(asr_text_token_id_);
+    ids_after = &prompt_ids_after_with_language;
+  }
+
   std::vector<int64_t> source_ids;
   size_t estimated_size =
       prompt_ids_before.size() +
       static_cast<size_t>(audio_token_len) * audio_pad_ids_.size() +
-      prompt_ids_after_.size();
+      ids_after->size();
   source_ids.reserve(estimated_size);
   source_ids.insert(source_ids.end(), prompt_ids_before.begin(),
                     prompt_ids_before.end());
@@ -386,8 +409,7 @@ std::vector<int64_t> OfflineRecognizerQwen3ASRImpl::BuildSourceIds(
                       audio_pad_ids_.end());
   }
 
-  source_ids.insert(source_ids.end(), prompt_ids_after_.begin(),
-                    prompt_ids_after_.end());
+  source_ids.insert(source_ids.end(), ids_after->begin(), ids_after->end());
 
   return source_ids;
 }
@@ -689,10 +711,15 @@ OfflineRecognitionResult OfflineRecognizerQwen3ASRImpl::GenerateText(
   const std::string hotwords = Qwen3FormatHotwordsForPrompt(
       stream->HasOption("hotwords") ? stream->GetOption("hotwords") : "");
 
+  std::string language;
+  if (stream->HasOption("language")) {
+    language = stream->GetOption("language");
+  }
+
   int32_t before_len = 0;
   int32_t fake_audio_token_len = 0;
   std::vector<int64_t> source_ids = BuildSourceIds(
-      hotwords, audio_token_len, &before_len, &fake_audio_token_len);
+      hotwords, language, audio_token_len, &before_len, &fake_audio_token_len);
 
   int32_t context_len = static_cast<int32_t>(source_ids.size());
   if (context_len == 0) {
@@ -951,21 +978,37 @@ OfflineRecognitionResult OfflineRecognizerQwen3ASRImpl::GenerateText(
     ++cur_len;
   }
 
-  // drop the first 3 tokens which contain things like:
-  // language None<asr_text>
-  if (generated_ids.size() >= 3) {
-    generated_ids.erase(generated_ids.begin(), generated_ids.begin() + 3);
+  std::vector<int64_t> cleaned_ids = generated_ids;
+  if (!generated_ids.empty()) {
+    const size_t prefix_window = std::min<size_t>(16, generated_ids.size());
+    auto asr_text_it = std::find(generated_ids.begin(),
+                                 generated_ids.begin() + prefix_window,
+                                 asr_text_token_id_);
+
+    // Only strip a leading scaffold prefix recognized by token ID.
+    if (asr_text_it != generated_ids.begin() + prefix_window &&
+        asr_text_it != generated_ids.begin()) {
+      std::vector<int64_t> prefix_ids(generated_ids.begin(),
+                                      std::next(asr_text_it));
+      std::string prefix_text = tokenizer_->Decode(prefix_ids);
+      if (prefix_text.rfind("language ", 0) == 0 &&
+          prefix_text.size() >= 10 &&
+          prefix_text.compare(prefix_text.size() - 10, 10, "<asr_text>") ==
+              0) {
+        cleaned_ids.assign(std::next(asr_text_it), generated_ids.end());
+      }
+    }
   }
 
-  result.text = tokenizer_->Decode(generated_ids);
+  result.text = tokenizer_->Decode(cleaned_ids);
   RemoveUtf8ReplacementChars(&result.text);
 
-  if (!generated_ids.empty()) {
+  if (!cleaned_ids.empty()) {
     std::vector<std::string> all_tokens;
-    all_tokens.reserve(generated_ids.size());
+    all_tokens.reserve(cleaned_ids.size());
     std::string pending_bytes;
 
-    for (int64_t token_id : generated_ids) {
+    for (int64_t token_id : cleaned_ids) {
       std::string s =
           tokenizer_->GetTokenStringStreaming(token_id, &pending_bytes);
       all_tokens.push_back(std::move(s));
@@ -975,25 +1018,7 @@ OfflineRecognitionResult OfflineRecognizerQwen3ASRImpl::GenerateText(
       all_tokens.back().append("\xEF\xBF\xBD");
     }
 
-    std::string concat;
-    size_t skip = 0;
-    bool stripped = false;
-    for (size_t i = 0; i < all_tokens.size(); ++i) {
-      concat += all_tokens[i];
-      if (concat.find("<asr_text>") != std::string::npos) {
-        skip = i + 1;
-        stripped = true;
-        break;
-      }
-    }
-    if (stripped && skip <= all_tokens.size()) {
-      result.tokens.reserve(all_tokens.size() - skip);
-      for (size_t i = skip; i < all_tokens.size(); ++i) {
-        result.tokens.push_back(std::move(all_tokens[i]));
-      }
-    } else {
-      result.tokens = std::move(all_tokens);
-    }
+    result.tokens = std::move(all_tokens);
   }
 
   return result;

sherpa-onnx/csrc/offline-recognizer-qwen3-asr-impl.h

@@ -37,6 +37,7 @@ class OfflineRecognizerQwen3ASRImpl : public OfflineRecognizerImpl {
  private:
   void InitPromptTemplateIds();
   std::vector<int64_t> BuildSourceIds(const std::string &hotwords,
+                                      const std::string &language,
                                       int32_t audio_token_len,
                                       int32_t *before_len,
                                       int32_t *fake_audio_token_len) const;
@@ -62,6 +63,7 @@ class OfflineRecognizerQwen3ASRImpl : public OfflineRecognizerImpl {
   std::unique_ptr<QwenAsrTokenizer> tokenizer_;
   std::vector<int64_t> audio_pad_ids_;
   std::vector<int64_t> prompt_ids_after_;
+  int64_t asr_text_token_id_ = -1;
   mutable std::mt19937 rng_;
 };
 

sherpa-onnx/csrc/qwen-asr-tokenizer.cc

@@ -1243,6 +1243,15 @@ std::vector<int64_t> QwenAsrTokenizer::Encode(const std::string &text) {
   return ans;
 }
 
+int64_t QwenAsrTokenizer::GetTokenId(const std::string &token) const {
+  auto it = token2id_.find(token);
+  if (it == token2id_.end()) {
+    return -1;
+  }
+
+  return it->second;
+}
+
 std::string QwenAsrTokenizer::Decode(const std::vector<int64_t> &token_ids) {
   std::string ans;
   std::string buffer;

sherpa-onnx/csrc/qwen-asr-tokenizer.h

@@ -25,6 +25,7 @@ class QwenAsrTokenizer {
   std::string Decode(const std::vector<int64_t> &token_ids);
   std::string GetTokenStringStreaming(int64_t token_id,
                                       std::string *pending_bytes) const;
+  int64_t GetTokenId(const std::string &token) const;
 
   int64_t GetEosTokenId() const { return eos_token_id_; }
   int64_t GetPadTokenId() const { return pad_token_id_; }