Skip to content

Commit 1f5f963

Browse files
csukuangfjcsukuangfj
authored
Add C++ runtime and Python API for Google MedASR models (#2935)
This pull request significantly enhances sherpa-onnx by integrating comprehensive support for Google MedASR CTC models. The changes encompass the foundational C++ model implementation, a streamlined Python API for ease of use, and specialized audio feature processing and text post-processing logic to ensure accurate medical speech recognition. This integration also includes updates to the build system and continuous integration tests, alongside a practical Python example, making MedASR models readily available and verifiable within the framework.
1 parent ee3611d commit 1f5f963

21 files changed

Lines changed: 634 additions & 8 deletions

‎.github/scripts/test-python.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,17 @@ log() {
88
echo -e "$(date '+%Y-%m-%d %H:%M:%S') (${fname}:${BASH_LINENO[0]}:${FUNCNAME[1]}) $*"
99
}
1010

11+
log "test Google MedASR"
12+
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2
13+
tar xvf sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2
14+
rm sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2
15+
ls -lh sherpa-onnx-medasr-ctc-en-int8-2025-12-25
16+
17+
ls -lh sherpa-onnx-medasr-ctc-en-int8-2025-12-25/test_wavs
18+
19+
python3 ./python-api-examples/offline-medasr-ctc-decode-files.py
20+
rm -rf sherpa-onnx-medasr-ctc-en-int8-2025-12-25
21+
1122
log "test omnilingual ASR"
1223
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2
1324
tar xvf sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2

‎.github/workflows/export-medasr-ctc-to-onnx.yaml‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: export-medasr-ctc-to-onnx
33
on:
44
push:
55
branches:
6-
- export-medasr-onnx
6+
- cpp-medasr-2
77
workflow_dispatch:
88

99
concurrency:
@@ -42,7 +42,9 @@ jobs:
4242
run: |
4343
cd scripts/medasr
4444
45-
curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-medasr-ctc-en-int8-2025-12-25/resolve/main/test_wavs/0.wav
45+
for i in $(seq 0 5); do
46+
curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-medasr-ctc-en-int8-2025-12-25/resolve/main/test_wavs/$i.wav
47+
done
4648
4749
curl -SL -O https://huggingface.co/csukuangfj/sherpa-onnx-medasr-ctc-en-int8-2025-12-25/resolve/main/test_wavs/transcript.txt
4850
@@ -53,7 +55,9 @@ jobs:
5355
run: |
5456
cd scripts/medasr
5557
56-
python3 test_onnx.py --model ./model.onnx --tokens ./tokens.txt --wav ./0.wav
58+
for i in $(seq 0 5); do
59+
python3 test_onnx.py --model ./model.onnx --tokens ./tokens.txt --wav ./$i.wav
60+
done
5761
5862
cat transcript.txt
5963
@@ -62,7 +66,9 @@ jobs:
6266
run: |
6367
cd scripts/medasr
6468
65-
python3 test_onnx.py --model ./model.int8.onnx --tokens ./tokens.txt --wav ./0.wav
69+
for i in $(seq 0 5); do
70+
python3 test_onnx.py --model ./model.int8.onnx --tokens ./tokens.txt --wav ./$i.wav
71+
done
6672
6773
cat transcript.txt
6874

‎python-api-examples/offline-medasr-ctc-decode-files.py‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
#!/usr/bin/env python3
2+
3+
"""
4+
This file shows how to use a non-streaming Google MedASR CTC model from
5+
https://huggingface.co/google/medasr
6+
to decode files.
7+
8+
Please download model files from
9+
https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models
10+
11+
For instance,
12+
13+
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2
14+
tar xvf sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2
15+
rm sherpa-onnx-medasr-ctc-en-int8-2025-12-25.tar.bz2
16+
"""
17+
18+
import time
19+
from pathlib import Path
20+
21+
import librosa
22+
import numpy as np
23+
import sherpa_onnx
24+
25+
26+
def create_recognizer():
27+
model = "./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/model.int8.onnx"
28+
tokens = "./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/tokens.txt"
29+
test_wav_0 = "./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/test_wavs/0.wav"
30+
test_wav_1 = "./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/test_wavs/1.wav"
31+
test_wav_2 = "./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/test_wavs/2.wav"
32+
test_wav_3 = "./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/test_wavs/3.wav"
33+
test_wav_4 = "./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/test_wavs/4.wav"
34+
test_wav_5 = "./sherpa-onnx-medasr-ctc-en-int8-2025-12-25/test_wavs/5.wav"
35+
36+
for f in [
37+
model,
38+
tokens,
39+
test_wav_0,
40+
test_wav_1,
41+
test_wav_2,
42+
test_wav_3,
43+
test_wav_4,
44+
test_wav_5,
45+
]:
46+
if not Path(f).is_file():
47+
print(f"{f} does not exist")
48+
49+
raise ValueError(
50+
"""Please download model files from
51+
https://github.com/k2-fsa/sherpa-onnx/releases/tag/asr-models
52+
"""
53+
)
54+
return (
55+
sherpa_onnx.OfflineRecognizer.from_medasr_ctc(
56+
model=model,
57+
tokens=tokens,
58+
num_threads=2,
59+
),
60+
test_wav_0,
61+
test_wav_1,
62+
test_wav_2,
63+
test_wav_3,
64+
test_wav_4,
65+
test_wav_5,
66+
)
67+
68+
69+
def load_audio(filename):
70+
audio, sample_rate = librosa.load(filename, sr=16000)
71+
assert sample_rate == 16000, sample_rate
72+
73+
return np.ascontiguousarray(audio)
74+
75+
76+
def decode_single_file(recognizer, filename):
77+
samples = load_audio(filename)
78+
79+
start_time = time.time()
80+
81+
stream = recognizer.create_stream()
82+
stream.accept_waveform(sample_rate=16000, waveform=samples)
83+
recognizer.decode_stream(stream)
84+
85+
end_time = time.time()
86+
elapsed_seconds = end_time - start_time
87+
audio_duration = len(samples) / 16000
88+
real_time_factor = elapsed_seconds / audio_duration
89+
90+
print("---")
91+
print(filename)
92+
print(stream.result)
93+
print(f"Elapsed seconds: {elapsed_seconds:.3f}")
94+
print(f"Audio duration in seconds: {audio_duration:.3f}")
95+
print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {real_time_factor:.3f}")
96+
print()
97+
98+
99+
def decode_multiple_files(recognizer, filenames):
100+
streams = []
101+
102+
start_time = time.time()
103+
104+
audio_duration = 0
105+
106+
for filename in filenames:
107+
samples = load_audio(filename)
108+
audio_duration += len(samples) / 16000
109+
110+
stream = recognizer.create_stream()
111+
stream.accept_waveform(sample_rate=16000, waveform=samples)
112+
streams.append(stream)
113+
114+
recognizer.decode_streams(streams)
115+
116+
end_time = time.time()
117+
elapsed_seconds = end_time - start_time
118+
real_time_factor = elapsed_seconds / audio_duration
119+
120+
for name, stream in zip(filenames, streams):
121+
print("---")
122+
print(name)
123+
print(stream.result)
124+
print()
125+
126+
print(f"Elapsed seconds: {elapsed_seconds:.3f}")
127+
print(f"Audio duration in seconds: {audio_duration:.3f}")
128+
print(f"RTF: {elapsed_seconds:.3f}/{audio_duration:.3f} = {real_time_factor:.3f}")
129+
print()
130+
print()
131+
132+
133+
def main():
134+
recognizer, *filenames = create_recognizer()
135+
136+
decode_single_file(recognizer, filenames[0])
137+
decode_single_file(recognizer, filenames[1])
138+
decode_multiple_files(recognizer, filenames[2:])
139+
140+
141+
if __name__ == "__main__":
142+
main()

‎scripts/medasr/export_onnx.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ def main():
107107
"model_author": "google",
108108
"maintainer": "k2-fsa",
109109
"vocab_size": processor.tokenizer.vocab_size,
110+
"subsampling_factor": 4,
110111
"url": "https://github.com/Google-Health/medasr",
111112
"license": "https://developers.google.com/health-ai-developer-foundations/terms",
112113
}

‎sherpa-onnx/csrc/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,8 @@ set(sources
4040
offline-fire-red-asr-model.cc
4141
offline-lm-config.cc
4242
offline-lm.cc
43+
offline-medasr-ctc-model-config.cc
44+
offline-medasr-ctc-model.cc
4345
offline-model-config.cc
4446
offline-moonshine-greedy-search-decoder.cc
4547
offline-moonshine-model-config.cc

‎sherpa-onnx/csrc/offline-ctc-model.cc‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
#include "sherpa-onnx/csrc/file-utils.h"
2222
#include "sherpa-onnx/csrc/macros.h"
2323
#include "sherpa-onnx/csrc/offline-dolphin-model.h"
24+
#include "sherpa-onnx/csrc/offline-medasr-ctc-model.h"
2425
#include "sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model.h"
2526
#include "sherpa-onnx/csrc/offline-omnilingual-asr-ctc-model.h"
2627
#include "sherpa-onnx/csrc/offline-tdnn-ctc-model.h"
@@ -126,6 +127,8 @@ std::unique_ptr<OfflineCtcModel> OfflineCtcModel::Create(
126127
return std::make_unique<OfflineTeleSpeechCtcModel>(config);
127128
} else if (!config.omnilingual.model.empty()) {
128129
return std::make_unique<OfflineOmnilingualAsrCtcModel>(config);
130+
} else if (!config.medasr.model.empty()) {
131+
return std::make_unique<OfflineMedAsrCtcModel>(config);
129132
}
130133

131134
// TODO(fangjun): Refactor it. We don't need to use model_type here
@@ -192,6 +195,8 @@ std::unique_ptr<OfflineCtcModel> OfflineCtcModel::Create(
192195
return std::make_unique<OfflineTeleSpeechCtcModel>(mgr, config);
193196
} else if (!config.omnilingual.model.empty()) {
194197
return std::make_unique<OfflineOmnilingualAsrCtcModel>(mgr, config);
198+
} else if (!config.medasr.model.empty()) {
199+
return std::make_unique<OfflineMedAsrCtcModel>(mgr, config);
195200
}
196201

197202
// TODO(fangjun): Refactor it. We don't need to use model_type here

‎sherpa-onnx/csrc/offline-medasr-ctc-model-config.cc‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// sherpa-onnx/csrc/offline-medasr-ctc-model-config.cc
2+
//
3+
// Copyright (c) 2025 Xiaomi Corporation
4+
5+
#include "sherpa-onnx/csrc/offline-medasr-ctc-model-config.h"
6+
7+
#include <sstream>
8+
#include <string>
9+
10+
#include "sherpa-onnx/csrc/file-utils.h"
11+
#include "sherpa-onnx/csrc/macros.h"
12+
13+
namespace sherpa_onnx {
14+
15+
void OfflineMedAsrCtcModelConfig::Register(ParseOptions *po) {
16+
po->Register(
17+
"medasr", &model,
18+
"Path to model.onnx from MedASR. Please see "
19+
"https://github.com/k2-fsa/sherpa-onnx/pull/2934 for available models");
20+
}
21+
22+
bool OfflineMedAsrCtcModelConfig::Validate() const {
23+
if (!FileExists(model)) {
24+
SHERPA_ONNX_LOGE("MedASR model: '%s' does not exist", model.c_str());
25+
return false;
26+
}
27+
28+
return true;
29+
}
30+
31+
std::string OfflineMedAsrCtcModelConfig::ToString() const {
32+
std::ostringstream os;
33+
34+
os << "OfflineMedAsrCtcModelConfig(";
35+
os << "model=\"" << model << "\")";
36+
37+
return os.str();
38+
}
39+
40+
} // namespace sherpa_onnx

‎sherpa-onnx/csrc/offline-medasr-ctc-model-config.h‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
// sherpa-onnx/csrc/offline-medasr-ctc-model-config.h
2+
//
3+
// Copyright (c) 2025 Xiaomi Corporation
4+
5+
#ifndef SHERPA_ONNX_CSRC_OFFLINE_MEDASR_CTC_MODEL_CONFIG_H_
6+
#define SHERPA_ONNX_CSRC_OFFLINE_MEDASR_CTC_MODEL_CONFIG_H_
7+
8+
#include <string>
9+
10+
#include "sherpa-onnx/csrc/parse-options.h"
11+
12+
namespace sherpa_onnx {
13+
14+
struct OfflineMedAsrCtcModelConfig {
15+
std::string model;
16+
17+
OfflineMedAsrCtcModelConfig() = default;
18+
explicit OfflineMedAsrCtcModelConfig(const std::string &model)
19+
: model(model) {}
20+
21+
void Register(ParseOptions *po);
22+
bool Validate() const;
23+
24+
std::string ToString() const;
25+
};
26+
27+
} // namespace sherpa_onnx
28+
29+
#endif // SHERPA_ONNX_CSRC_OFFLINE_MEDASR_CTC_MODEL_CONFIG_H_

0 commit comments

Comments
 (0)