Skip to content

Commit b74c4df

Browse files
JulianPscheidJulianPscheid
authored
Add multilingual Nemotron-3.5 streaming ASR support (k2-fsa#3671)
1 parent 2aafc63 commit b74c4df

15 files changed

Lines changed: 1331 additions & 24 deletions

File tree

‎.github/workflows/export-nemotron-3.5-asr-streaming-0.6b.yaml‎

Lines changed: 185 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,185 @@
1+
name: export-nemotron-3-5-asr-streaming-06b
2+
3+
on:
4+
push:
5+
branches:
6+
- export-nemotron-3-5-asr-streaming
7+
workflow_dispatch:
8+
9+
concurrency:
10+
group: export-nemotron-3-5-asr-streaming-to-onnx-${{ github.ref }}
11+
cancel-in-progress: true
12+
13+
jobs:
14+
export-nemotron-3-5-asr-streaming-0-6b-to-onnx:
15+
if: github.repository_owner == 'k2-fsa' || github.repository_owner == 'csukuangfj'
16+
name: nemotron-3-5-asr-streaming-0-6b-to-onnx
17+
runs-on: ${{ matrix.os }}
18+
strategy:
19+
fail-fast: false
20+
matrix:
21+
os: [macos-latest]
22+
python-version: ["3.10"]
23+
24+
steps:
25+
- uses: actions/checkout@v4
26+
27+
- name: Setup Python ${{ matrix.python-version }}
28+
uses: actions/setup-python@v5
29+
with:
30+
python-version: ${{ matrix.python-version }}
31+
32+
- name: Install NeMo
33+
shell: bash
34+
run: |
35+
# nemotron-3.5-asr-streaming-0.6b requires EncDecRNNTBPEModelWithPrompt,
36+
# which is not in the stable nemo-toolkit release yet; the model card
37+
# instructs installing NeMo from main.
38+
BRANCH='main'
39+
pip install Cython packaging
40+
pip install "nemo_toolkit[asr] @ git+https://github.com/NVIDIA/NeMo.git@$BRANCH"
41+
pip install onnxruntime ipython sentencepiece
42+
pip install kaldi-native-fbank
43+
pip install soundfile librosa
44+
45+
- name: Run
46+
shell: bash
47+
run: |
48+
cd scripts/nemo/nemotron-3.5-asr-streaming-0.6b
49+
50+
python3 ./export_onnx.py
51+
52+
ls -lh
53+
echo "---"
54+
55+
ls -lh */
56+
57+
echo "---"
58+
59+
- name: Collect results
60+
shell: bash
61+
run: |
62+
src=scripts/nemo/nemotron-3.5-asr-streaming-0.6b
63+
64+
for chunk in 80 160 560 1120; do
65+
d=sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-${chunk}ms-2026-06-11
66+
mkdir -p $d
67+
68+
cp -av $src/$chunk/encoder.onnx $d/
69+
cp -av $src/$chunk/encoder.data $d/
70+
cp -av $src/$chunk/decoder.onnx $d/
71+
cp -av $src/$chunk/joiner.onnx $d/
72+
cp -av $src/tokens.txt $d/
73+
cat >$d/README.md <<EOF
74+
# Introduction
75+
This model is from https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
76+
77+
The chunk size is ${chunk}ms for the exported ONNX model.
78+
79+
Use per-stream language strings such as "en", "ja", or "auto".
80+
EOF
81+
82+
ls -lh $d
83+
84+
d=sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-${chunk}ms-int8-2026-06-11
85+
mkdir -p $d
86+
87+
cp -av $src/${chunk}/encoder.int8.onnx $d/
88+
cp -av $src/${chunk}/decoder.int8.onnx $d/
89+
cp -av $src/${chunk}/joiner.int8.onnx $d/
90+
cp -av $src/tokens.txt $d/
91+
cat >$d/README.md <<EOF
92+
# Introduction
93+
This model is from https://huggingface.co/nvidia/nemotron-3.5-asr-streaming-0.6b
94+
95+
The chunk size is ${chunk}ms for the exported ONNX model.
96+
97+
Use per-stream language strings such as "en", "ja", or "auto".
98+
EOF
99+
100+
ls -lh $d
101+
done
102+
103+
- name: Download test waves
104+
shell: bash
105+
run: |
106+
mkdir test_wavs
107+
pushd test_wavs
108+
curl -SL -O https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/en.wav
109+
curl -SL -O https://hf-mirror.com/csukuangfj/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17/resolve/main/test_wavs/ja.wav
110+
popd
111+
112+
models=(
113+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-80ms-int8-2026-06-11
114+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-160ms-int8-2026-06-11
115+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11
116+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11
117+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-80ms-2026-06-11
118+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-160ms-2026-06-11
119+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-2026-06-11
120+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-2026-06-11
121+
)
122+
for m in ${models[@]}; do
123+
cp -av test_wavs $m
124+
tar cjvf $m.tar.bz2 $m
125+
done
126+
127+
ls -lh *.tar.bz2
128+
129+
- name: Release
130+
uses: svenstaro/upload-release-action@v2
131+
with:
132+
file_glob: true
133+
file: sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-*-int8-2026-06-11.tar.bz2
134+
overwrite: true
135+
repo_name: k2-fsa/sherpa-onnx
136+
repo_token: ${{ secrets.UPLOAD_GH_SHERPA_ONNX_TOKEN }}
137+
tag: asr-models
138+
139+
- name: Publish to huggingface
140+
env:
141+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
142+
uses: nick-fields/retry@v3
143+
with:
144+
max_attempts: 20
145+
timeout_seconds: 200
146+
shell: bash
147+
command: |
148+
git config --global user.email "csukuangfj@gmail.com"
149+
git config --global user.name "Fangjun Kuang"
150+
151+
models=(
152+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-80ms-int8-2026-06-11
153+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-160ms-int8-2026-06-11
154+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11
155+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-int8-2026-06-11
156+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-80ms-2026-06-11
157+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-160ms-2026-06-11
158+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-2026-06-11
159+
sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-1120ms-2026-06-11
160+
)
161+
162+
for m in ${models[@]}; do
163+
if [ ! -d $m ]; then
164+
echo "skip $m"
165+
continue
166+
fi
167+
168+
rm -rf huggingface
169+
export GIT_LFS_SKIP_SMUDGE=1
170+
export GIT_CLONE_PROTECTION_ACTIVE=false
171+
git clone https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/$m huggingface
172+
cp -av $m/* huggingface
173+
cd huggingface
174+
git lfs track "*.onnx"
175+
git lfs track "*.data"
176+
git lfs track "*.wav"
177+
git status
178+
git add .
179+
git status
180+
git commit -m "first commit"
181+
git push https://csukuangfj2:$HF_TOKEN@huggingface.co/csukuangfj2/$m main
182+
cd ..
183+
done
184+
185+
rm -rf huggingface

‎c-api-examples/streaming-nemotron-c-api.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,10 @@ int32_t main() {
6767

6868
const SherpaOnnxOnlineStream *stream =
6969
SherpaOnnxCreateOnlineStream(recognizer);
70+
// Multilingual Nemotron models use the generic stream option "language".
71+
// For example: SherpaOnnxOnlineStreamSetOption(stream, "language", "ja");
72+
// Empty/unset means auto. English-only Nemotron ignores this option.
73+
SherpaOnnxOnlineStreamSetOption(stream, "language", "en");
7074

7175
const SherpaOnnxDisplay *display = SherpaOnnxCreateDisplay(50);
7276
int32_t segment_id = 0;

‎cxx-api-examples/streaming-nemotron-cxx-api.cc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,10 @@ int32_t main() {
6363
const auto begin = std::chrono::steady_clock::now();
6464

6565
OnlineStream stream = recognizer.CreateStream();
66+
// Multilingual Nemotron models use the generic stream option "language".
67+
// For example: stream.SetOption("language", "ja");
68+
// Empty/unset means auto. English-only Nemotron ignores this option.
69+
stream.SetOption("language", "en");
6670
stream.AcceptWaveform(wave.sample_rate, wave.samples.data(),
6771
wave.samples.size());
6872
stream.InputFinished();

‎flutter-examples/streaming_asr/lib/streaming_asr.dart‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ class _StreamingAsrScreenState extends State<StreamingAsrScreen> {
6464
sherpa_onnx.initBindings();
6565
_recognizer = await createOnlineRecognizer();
6666
_stream = _recognizer?.createStream();
67+
// Multilingual Nemotron: _stream?.setOption(key: 'language', value: 'ja');
6768

6869
_isInitialized = true;
6970
}

‎flutter/sherpa_onnx/lib/src/online_stream.dart‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,5 +67,25 @@ class OnlineStream {
6767
SherpaOnnxBindings.onlineStreamInputFinished?.call(ptr);
6868
}
6969

70+
/// Set a string option on the underlying stream.
71+
///
72+
/// For multilingual Nemotron models, use `key: 'language'` with values such
73+
/// as `ja` or `auto`.
74+
void setOption({required String key, required String value}) {
75+
if (SherpaOnnxBindings.onlineStreamSetOption == null) {
76+
throw Exception("Please initialize sherpa-onnx first");
77+
}
78+
79+
if (ptr == nullptr) {
80+
return;
81+
}
82+
83+
final pKey = key.toNativeUtf8();
84+
final pValue = value.toNativeUtf8();
85+
SherpaOnnxBindings.onlineStreamSetOption?.call(ptr, pKey, pValue);
86+
calloc.free(pKey);
87+
calloc.free(pValue);
88+
}
89+
7090
Pointer<SherpaOnnxOnlineStream> ptr;
7191
}

‎flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1795,6 +1795,20 @@ typedef OnlineStreamInputFinishedNative =
17951795
typedef OnlineStreamInputFinished =
17961796
void Function(Pointer<SherpaOnnxOnlineStream>);
17971797

1798+
typedef OnlineStreamSetOptionNative =
1799+
Void Function(
1800+
Pointer<SherpaOnnxOnlineStream>,
1801+
Pointer<Utf8>,
1802+
Pointer<Utf8>,
1803+
);
1804+
1805+
typedef OnlineStreamSetOption =
1806+
void Function(
1807+
Pointer<SherpaOnnxOnlineStream>,
1808+
Pointer<Utf8>,
1809+
Pointer<Utf8>,
1810+
);
1811+
17981812
typedef SherpaOnnxSpeakerEmbeddingExtractorIsReadyNative =
17991813
Int32 Function(
18001814
Pointer<SherpaOnnxSpeakerEmbeddingExtractor>,
@@ -2043,6 +2057,8 @@ class SherpaOnnxBindings {
20432057

20442058
static OnlineStreamInputFinished? onlineStreamInputFinished;
20452059

2060+
static OnlineStreamSetOption? onlineStreamSetOption;
2061+
20462062
static SherpaOnnxSpeakerEmbeddingExtractorIsReady?
20472063
speakerEmbeddingExtractorIsReady;
20482064

@@ -2749,6 +2765,12 @@ class SherpaOnnxBindings {
27492765
)
27502766
.asFunction();
27512767

2768+
onlineStreamSetOption ??= dynamicLibrary
2769+
.lookup<NativeFunction<OnlineStreamSetOptionNative>>(
2770+
'SherpaOnnxOnlineStreamSetOption',
2771+
)
2772+
.asFunction();
2773+
27522774
speakerEmbeddingExtractorIsReady ??= dynamicLibrary
27532775
.lookup<
27542776
NativeFunction<SherpaOnnxSpeakerEmbeddingExtractorIsReadyNative>

‎scripts/nemo/nemotron-3.5-asr-streaming-0.6b/README.md‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# Multilingual Nemotron-3.5 Streaming ASR
2+
3+
This directory exports the NVIDIA NeMo model
4+
`nvidia/nemotron-3.5-asr-streaming-0.6b` to sherpa-onnx streaming transducer
5+
packages.
6+
7+
The exporter writes the same package layout as
8+
`nvidia/nemotron-speech-streaming-en-0.6b`:
9+
10+
- `encoder.onnx`
11+
- `encoder.data`
12+
- `decoder.onnx`
13+
- `joiner.onnx`
14+
- int8 variants for all three ONNX graphs
15+
- `tokens.txt` converted from the model's SentencePiece tokenizer
16+
17+
The encoder metadata contains `prompt_dictionary` and `auto_prompt_id`. Users
18+
set the per-stream language as a string; the numerical prompt id is internal.
19+
An empty language string and `auto` use the model's auto-detect prompt.
20+
21+
## Export
22+
23+
As of June 2026, stable NeMo releases lack `EncDecRNNTBPEModelWithPrompt`,
24+
so install NeMo from git main.
25+
26+
```bash
27+
pip install Cython packaging
28+
pip install "nemo_toolkit[asr] @ git+https://github.com/NVIDIA/NeMo.git@main"
29+
pip install onnxruntime ipython sentencepiece
30+
pip install kaldi-native-fbank
31+
pip install soundfile librosa
32+
33+
python3 ./export_onnx.py
34+
```
35+
36+
The script exports 80ms, 160ms, 560ms, and 1120ms chunk sizes.
37+
38+
## Decode
39+
40+
Forced language:
41+
42+
```bash
43+
./build/bin/sherpa-onnx \
44+
--encoder=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/encoder.int8.onnx \
45+
--decoder=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/decoder.int8.onnx \
46+
--joiner=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/joiner.int8.onnx \
47+
--tokens=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/tokens.txt \
48+
--language=ja \
49+
./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/test_wavs/ja.wav
50+
```
51+
52+
Auto language:
53+
54+
```bash
55+
./build/bin/sherpa-onnx \
56+
--encoder=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/encoder.int8.onnx \
57+
--decoder=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/decoder.int8.onnx \
58+
--joiner=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/joiner.int8.onnx \
59+
--tokens=./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/tokens.txt \
60+
--language=auto \
61+
./sherpa-onnx-nemotron-3.5-asr-streaming-0.6b-560ms-int8-2026-06-11/test_wavs/ja.wav
62+
```
63+
64+
The same auto behavior is used when `--language` is omitted.

0 commit comments

Comments
 (0)