Skip to content

Commit cc02b69

Browse files
csukuangfjcsukuangfj
authored
Add WebAssembly example for PocketTTS (#3340)
1 parent 823bfd1 commit cc02b69

8 files changed

Lines changed: 94 additions & 18 deletions

File tree

‎.github/workflows/wasm-simd-hf-space-tts.yaml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ jobs:
2121
fail-fast: false
2222
matrix:
2323
os: [ubuntu-latest]
24-
total: ["5"]
25-
index: ["0", "1", "2", "3", "4"]
24+
total: ["7"]
25+
index: ["0", "1", "2", "3", "4", "5", "6"]
2626

2727
steps:
2828
- uses: actions/checkout@v4

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,3 +177,4 @@ sherpa-onnx-fire-red-asr2-ctc-zh_en-int8-2026-02-25
177177
non-streaming-fire-red-asr-ctc-decode-files
178178
sherpa-onnx-moonshine-*-quantized-2026-02-27
179179
sherpa-onnx-supertonic-tts-int8-2026-03-06
180+
token_scores.json

‎README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ We also have spaces built using WebAssembly. They are listed below:
145145
|Speech synthesis (Matcha, Chinese+English) |[Click me][wasm-hf-tts-matcha-zh-en]| [地址][wasm-ms-tts-matcha-zh-en]|
146146
|Speaker diarization |[Click me][wasm-hf-speaker-diarization]|[地址][wasm-ms-speaker-diarization]|
147147
|Voice cloning with ZipVoice (Chinese+English) |[Click me][wasm-hf-voice-cloning-zipvoice]|[地址][wasm-ms-voice-cloning-zipvoice]|
148+
|Voice cloning with Pocket TTS (English) |[Click me][wasm-hf-voice-cloning-pocket]|[地址][wasm-ms-voice-cloning-pocketzipvoice]|
148149

149150
</details>
150151

@@ -507,6 +508,8 @@ a multimodal chatbot based on go with sherpa-onnx's speech lib api.
507508
[wasm-ms-speaker-diarization]: https://www.modelscope.cn/studios/csukuangfj/web-assembly-speaker-diarization-sherpa-onnx
508509
[wasm-hf-voice-cloning-zipvoice]: https://huggingface.co/spaces/k2-fsa/web-assembly-zh-en-tts-zipvoice
509510
[wasm-ms-voice-cloning-zipvoice]: https://modelscope.cn/studios/csukuangfj/web-assembly-zh-en-tts-zipvoice
511+
[wasm-hf-voice-cloning-pocket]: https://huggingface.co/spaces/k2-fsa/web-assembly-en-tts-pocket
512+
[wasm-ms-voice-cloning-pocket]: https://modelscope.cn/studios/csukuangfj/web-assembly-en-tts-pocket
510513
[apk-speaker-diarization]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk.html
511514
[apk-speaker-diarization-cn]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/apk-cn.html
512515
[apk-streaming-asr]: https://k2-fsa.github.io/sherpa/onnx/android/apk.html

‎scripts/wasm/generate-tts.py‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,28 @@ def get_models():
161161
git diff
162162
""",
163163
),
164+
Model(
165+
model_name="sherpa-onnx-pocket-tts-int8-2026-01-26",
166+
hf="k2-fsa/web-assembly-en-tts-pocket",
167+
ms="csukuangfj/web-assembly-en-tts-pocket",
168+
cmd="""
169+
pushd $model_name
170+
171+
mv -v lm_flow.int8.onnx ../
172+
mv -v lm_main.int8.onnx ../
173+
mv -v encoder.onnx ../
174+
mv -v decoder.int8.onnx ../
175+
mv -v text_conditioner.onnx ../
176+
mv -v vocab.json ../
177+
mv -v token_scores.json ../
178+
popd
179+
180+
git checkout .
181+
sed -i.bak 's/let modelType = 0/let modelType = 5/g' ../sherpa-onnx-tts.js
182+
rm -rf $model_name
183+
git diff
184+
""",
185+
),
164186
]
165187
return models
166188

‎wasm/tts/CMakeLists.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ if(NOT $ENV{SHERPA_ONNX_IS_USING_BUILD_WASM_SH})
22
message(FATAL_ERROR "Please use ./build-wasm-simd-tts.sh to build for wasm TTS")
33
endif()
44

5-
if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/tokens.txt")
5+
if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/tokens.txt" AND
6+
NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/lm_flow.int8.onnx")
67
message(FATAL_ERROR "Please read ${CMAKE_CURRENT_SOURCE_DIR}/assets/README.md before you continue")
78
endif()
89

‎wasm/tts/app-tts.js‎

Lines changed: 22 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -123,8 +123,10 @@ speedInput.oninput = function() {
123123

124124
function updateUiForModelType() {
125125
const isZipVoice = ttsInstanceInfo.modelType === 4;
126-
speakerIdSection.classList.toggle('hidden', isZipVoice);
127-
referenceAudioSection.classList.toggle('hidden', !isZipVoice);
126+
const isPocketTts = ttsInstanceInfo.modelType === 5;
127+
const useGenerationConfig = isZipVoice || isPocketTts;
128+
speakerIdSection.classList.toggle('hidden', useGenerationConfig);
129+
referenceAudioSection.classList.toggle('hidden', !useGenerationConfig);
128130
referenceTextSection.classList.toggle('hidden', !isZipVoice);
129131
}
130132

@@ -193,9 +195,11 @@ function downloadBlob(blob, filename) {
193195

194196
generateBtn.onclick = async function() {
195197
const isZipVoice = ttsInstanceInfo.modelType === 4;
198+
const isPocketTts = ttsInstanceInfo.modelType === 5;
199+
const useGenerationConfig = isZipVoice || isPocketTts;
196200

197201
let speakerId = speakerIdInput.value;
198-
if (!isZipVoice) {
202+
if (!useGenerationConfig) {
199203
if (speakerId.trim().length == 0) {
200204
alert('Please input a speakerId');
201205
return;
@@ -224,7 +228,7 @@ generateBtn.onclick = async function() {
224228
console.log('speed', speedInput.value);
225229
console.log('text', text);
226230

227-
if (isZipVoice) {
231+
if (useGenerationConfig) {
228232
if (!referenceAudioInput.files || referenceAudioInput.files.length === 0) {
229233
alert('Please select a reference audio file');
230234
return;
@@ -236,24 +240,27 @@ generateBtn.onclick = async function() {
236240
return;
237241
}
238242

239-
const referenceText = referenceTextInput.value.trim();
240-
if (referenceText.length === 0) {
241-
alert('Please input the transcript of the reference audio');
242-
return;
243-
}
244-
245243
const referenceAudio = await readReferenceAudio(referenceFile);
246244
const genConfig = {
247245
speed: parseFloat(speedInput.value),
248246
referenceAudio: referenceAudio.samples,
249247
referenceSampleRate: referenceAudio.sampleRate,
250-
referenceText: referenceText,
251-
numSteps: 4,
252-
extra: {
253-
min_char_in_sentence: 10,
254-
},
248+
numSteps: isPocketTts ? 5 : 4,
255249
};
256250

251+
if (isZipVoice) {
252+
const referenceText = referenceTextInput.value.trim();
253+
if (referenceText.length === 0) {
254+
alert('Please input the transcript of the reference audio');
255+
return;
256+
}
257+
258+
genConfig.referenceText = referenceText;
259+
genConfig.extra = {
260+
min_char_in_sentence: 10,
261+
};
262+
}
263+
257264
generateBtn.disabled = true;
258265
setGenerationStatus('Generating audio...');
259266

‎wasm/tts/assets/README.md‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,25 @@ rm -rf sherpa-onnx-zipvoice-distill-int8-zh-en-emilia
3636
wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/vocoder-models/vocos_24khz.onnx
3737
```
3838

39+
PocketTTS example:
40+
41+
```bash
42+
cd sherpa-onnx/wasm/tts/assets
43+
44+
wget -q https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2
45+
tar xf sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2
46+
rm sherpa-onnx-pocket-tts-int8-2026-01-26.tar.bz2
47+
48+
mv sherpa-onnx-pocket-tts-int8-2026-01-26/lm_flow.int8.onnx ./
49+
mv sherpa-onnx-pocket-tts-int8-2026-01-26/lm_main.int8.onnx ./
50+
mv sherpa-onnx-pocket-tts-int8-2026-01-26/encoder.onnx ./
51+
mv sherpa-onnx-pocket-tts-int8-2026-01-26/decoder.int8.onnx ./
52+
mv sherpa-onnx-pocket-tts-int8-2026-01-26/text_conditioner.onnx ./
53+
mv sherpa-onnx-pocket-tts-int8-2026-01-26/vocab.json ./
54+
mv sherpa-onnx-pocket-tts-int8-2026-01-26/token_scores.json ./
55+
rm -rf sherpa-onnx-pocket-tts-int8-2026-01-26
56+
```
57+
3958
You should have the following files in `assets` before you can run
4059
`build-wasm-simd-tts.sh`
4160

‎wasm/tts/sherpa-onnx-tts.js‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -985,6 +985,17 @@ function createOfflineTts(Module, myConfig) {
985985
guidanceScale: 1.0,
986986
};
987987

988+
const offlineTtsPocketModelConfig = {
989+
lmFlow: '',
990+
lmMain: '',
991+
encoder: '',
992+
decoder: '',
993+
textConditioner: '',
994+
vocabJson: '',
995+
tokenScoresJson: '',
996+
voiceEmbeddingCacheCapacity: 50,
997+
};
998+
988999
let ruleFsts = '';
9891000

9901001
switch (modelType) {
@@ -1031,6 +1042,17 @@ function createOfflineTts(Module, myConfig) {
10311042
offlineTtsZipVoiceModelConfig.dataDir = './espeak-ng-data';
10321043
offlineTtsZipVoiceModelConfig.lexicon = './lexicon.txt';
10331044
break;
1045+
case 5:
1046+
// pocket tts
1047+
// https://k2-fsa.github.io/sherpa/onnx/tts/pocket.html
1048+
offlineTtsPocketModelConfig.lmFlow = './lm_flow.int8.onnx';
1049+
offlineTtsPocketModelConfig.lmMain = './lm_main.int8.onnx';
1050+
offlineTtsPocketModelConfig.encoder = './encoder.onnx';
1051+
offlineTtsPocketModelConfig.decoder = './decoder.int8.onnx';
1052+
offlineTtsPocketModelConfig.textConditioner = './text_conditioner.onnx';
1053+
offlineTtsPocketModelConfig.vocabJson = './vocab.json';
1054+
offlineTtsPocketModelConfig.tokenScoresJson = './token_scores.json';
1055+
break;
10341056
}
10351057

10361058
const offlineTtsModelConfig = {
@@ -1039,6 +1061,7 @@ function createOfflineTts(Module, myConfig) {
10391061
offlineTtsKokoroModelConfig: offlineTtsKokoroModelConfig,
10401062
offlineTtsKittenModelConfig: offlineTtsKittenModelConfig,
10411063
offlineTtsZipVoiceModelConfig: offlineTtsZipVoiceModelConfig,
1064+
offlineTtsPocketModelConfig: offlineTtsPocketModelConfig,
10421065
numThreads: 1,
10431066
debug: 1,
10441067
provider: 'cpu',

0 commit comments

Comments
 (0)