Skip to content

Commit d49e8f4

Browse files
authored
Add two Piper Chinese TTS models (k2-fsa#3546)
This pull request introduces support for Chinese Piper TTS models using G2PW pinyin instead of espeak-ng. Key changes include updating metadata generation to include a has_g2pw flag, adding scripts for lexicon generation, and modifying the C++ core to handle BOS/EOS tokens and rule-based FSTs.
1 parent 3193519 commit d49e8f4

12 files changed

Lines changed: 233 additions & 35 deletions

.github/workflows/export-piper.yaml

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -43,8 +43,13 @@ jobs:
4343
- name: Install Python dependencies
4444
shell: bash
4545
run: |
46-
python3 -m pip install --upgrade pip jinja2 iso639-lang onnx==1.17.0 onnxruntime==1.17.1 sherpa-onnx onnxmltools==1.13.0
47-
python3 -m pip install "numpy<2" soundfile
46+
python3 -m pip install --upgrade \
47+
pip jinja2 iso639-lang \
48+
onnx==1.17.0 onnxruntime==1.17.1 \
49+
onnxmltools==1.13.0 \
50+
"numpy<2" soundfile requests
51+
52+
python3 -m pip install --verbose sherpa_onnx_bin sherpa_onnx_core sherpa_onnx --no-index -f https://k2-fsa.github.io/sherpa/onnx/cpu.html
4853
4954
- name: Generate script
5055
env:
@@ -176,8 +181,13 @@ jobs:
176181
vits-piper-ar_JO-SA_miro_V2-high-int8
177182
vits-piper-ar_JO-SA_miro_V2-high-fp16
178183
vits-piper-ar_JO-SA_miro_V2-high
179-
vits-piper-ar_JO-SA_miro_V2-high
180184
vits-piper-sq_AL-edon-medium
185+
vits-piper-zh_CN-xiao_ya-medium
186+
vits-piper-zh_CN-xiao_ya-medium-int8
187+
vits-piper-zh_CN-xiao_ya-medium-fp16
188+
vits-piper-zh_CN-chaowen-medium
189+
vits-piper-zh_CN-chaowen-medium-int8
190+
vits-piper-zh_CN-chaowen-medium-fp16
181191
)
182192
for d in ${dirs[@]}; do
183193
src=scripts/piper/release/$d

scripts/apk/generate-tts-apk-script.py

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -277,10 +277,19 @@ def get_piper_models() -> List[TtsModel]:
277277
TtsModel(model_dir="vits-piper-vi_VN-vais1000-medium"),
278278
TtsModel(model_dir="vits-piper-vi_VN-vivos-x_low"),
279279
TtsModel(model_dir="vits-piper-zh_CN-huayan-medium"),
280+
TtsModel(model_dir="vits-piper-zh_CN-xiao_ya-medium"),
281+
TtsModel(model_dir="vits-piper-zh_CN-chaowen-medium"),
280282
]
281283

282284
for m in models:
283-
m.data_dir = m.model_dir + "/" + "espeak-ng-data"
285+
if "zh_CN" in m.model_dir and (
286+
"xiao_ya" in m.model_dir or "chaowen" in m.model_dir
287+
):
288+
m.lexicon = f"{m.model_dir}/lexicon.txt"
289+
m.rule_fsts = f"{m.model_dir}/phone.fst,{m.model_dir}/date.fst,{m.model_dir}/number.fst"
290+
else:
291+
m.data_dir = m.model_dir + "/" + "espeak-ng-data"
292+
284293
m.model_name = m.model_dir[len("vits-piper-") :] + ".onnx"
285294
m.lang = m.model_dir.split("-")[2][:2]
286295

scripts/piper/add_meta_data.py

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,11 @@
88
import onnx
99
from iso639 import Lang
1010

11+
# For the following model,
12+
# https://huggingface.co/rhasspy/piper-voices/blob/main/zh/zh_CN/xiao_ya/medium/zh_CN-xiao_ya-medium.onnx.json
13+
# it uses g2pw, not espeak-ng.
14+
# To handle that, we use has_g2pw = 1 in the meta_data
15+
1116

1217
def get_args():
1318
# For en_GB-semaine-medium
@@ -110,13 +115,22 @@ def main():
110115
else:
111116
voice = config["espeak"]["voice"]
112117

118+
has_g2pw = 0
119+
has_espeak = 1
120+
121+
if config["phoneme_type"] == "pinyin" and voice == "zh":
122+
has_espeak = 0
123+
has_g2pw = 1
124+
113125
print("add model metadata")
114126
meta_data = {
115127
"model_type": "vits",
116128
"comment": "piper", # must be piper for models from piper
117129
"language": lang_iso.name,
118130
"voice": voice, # e.g., en-us
119-
"has_espeak": 1,
131+
"version": 1,
132+
"has_espeak": has_espeak,
133+
"has_g2pw": has_g2pw,
120134
"n_speakers": config["num_speakers"],
121135
"sample_rate": sample_rate,
122136
}

scripts/piper/generate.py

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1647,6 +1647,37 @@ def get_vi_models():
16471647
return ans
16481648

16491649

1650+
# Chinese
1651+
def get_zh_models():
1652+
zh_CN = [
1653+
PiperModel(name="xiao_ya", kind="medium", sr=22050, ns=1),
1654+
PiperModel(name="chaowen", kind="medium", sr=22050, ns=1),
1655+
]
1656+
1657+
for m in zh_CN:
1658+
m.lang = "zh_CN"
1659+
1660+
ans = zh_CN
1661+
1662+
for m in ans:
1663+
m.text = "某某银行的副行长和一些行政领导表示,他们去过长江和长白山; 经济不断增长。2024年12月31号,拨打110或者18920240511。123456块钱。当夜幕降临,星光点点,伴随着微风拂面,我在静谧中感受着时光的流转,思念如涟漪荡漾,梦境如画卷展开,我与自然融为一体,沉静在这片宁静的美丽之中,感受着生命的奇迹与温柔."
1664+
1665+
if m.model_name == "":
1666+
m.model_name = f"{m.lang}-{m.name}-{m.kind}.onnx"
1667+
1668+
code = m.lang[:2]
1669+
if m.cmd == "":
1670+
m.cmd = f"""
1671+
wget -qq https://huggingface.co/rhasspy/piper-voices/resolve/main/{code}/{m.lang}/{m.name}/{m.kind}/{m.model_name}
1672+
wget -qq https://huggingface.co/rhasspy/piper-voices/resolve/main/{code}/{m.lang}/{m.name}/{m.kind}/{m.model_name}.json
1673+
wget -qq https://huggingface.co/rhasspy/piper-voices/resolve/main/{code}/{m.lang}/{m.name}/{m.kind}/MODEL_CARD
1674+
"""
1675+
if m.url == "":
1676+
m.url = f"https://huggingface.co/rhasspy/piper-voices/tree/main/{code}/{m.lang}/{m.name}/{m.kind}"
1677+
1678+
return ans
1679+
1680+
16501681
# Indonesian
16511682
def get_id_models():
16521683
id_ID = [
@@ -2114,6 +2145,7 @@ def get_all_models():
21142145
ans += get_tr_models()
21152146
ans += get_uk_models()
21162147
ans += get_vi_models()
2148+
ans += get_zh_models()
21172149

21182150
for i, m in enumerate(ans):
21192151
m.index = i
@@ -2182,8 +2214,21 @@ def main():
21822214
"model": f"{model_dir}/{m.model_name}",
21832215
"data_dir": f"{model_dir}/espeak-ng-data",
21842216
"tokens": f"{model_dir}/tokens.txt",
2217+
"lexicon": "",
21852218
"text": m.text,
2219+
"rule_fsts": "",
21862220
}
2221+
2222+
if m.lang == "zh_CN" and m.name in ("xiao_ya", "chaowen"):
2223+
d = {
2224+
"model": f"{model_dir}/{m.model_name}",
2225+
"data_dir": "",
2226+
"tokens": f"{model_dir}/tokens.txt",
2227+
"lexicon": f"{model_dir}/lexicon.txt",
2228+
"text": m.text,
2229+
"rule_fsts": f"{model_dir}/phone.fst,{model_dir}/date.fst,{model_dir}/number.fst",
2230+
}
2231+
21872232
for i in range(m.ns):
21882233
s = template.render(
21892234
**d,

scripts/piper/generate.sh.in

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,15 @@ log() {
1212
}
1313

1414
wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/espeak-ng-data.tar.bz2
15+
wget -qq https://github.com/k2-fsa/sherpa-onnx/releases/download/tts-models/lexicon-zh-g2pw.txt
16+
17+
wget -qq https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/date.fst
18+
wget -qq https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/number.fst
19+
wget -qq https://huggingface.co/csukuangfj/icefall-tts-aishell3-vits-low-2024-04-06/resolve/main/data/phone.fst
20+
1521
tar xf espeak-ng-data.tar.bz2
1622
rm espeak-ng-data.tar.bz2
23+
ls -lh lexicon-zh-g2pw.txt
1724

1825
mkdir -p release
1926

@@ -50,7 +57,16 @@ mv -v README $dst/ || true
5057
mv -v README.md $dst/ || true
5158
mv -v LICENSE.txt $dst/ || true
5259
mv -v *.json $dst/
53-
cp -a ./espeak-ng-data $dst/
60+
if [[ ! $model_name =~ zh_CN-(xiao_ya|chaowen)-medium ]]; then
61+
cp -a ./espeak-ng-data $dst/
62+
else
63+
cp lexicon-zh-g2pw.txt $dst/lexicon.txt
64+
cp date.fst $dst/
65+
cp number.fst $dst/
66+
cp phone.fst $dst/
67+
fi
68+
69+
ls -lh $dst/
5470

5571
cp -a $dst $dst_int8
5672
cp -a $dst $dst_fp16
@@ -79,6 +95,7 @@ if [ -d hf ]; then
7995
mkdir -p hf/piper/mp3/$lang/vits-piper-$lang-$name-$kind
8096
for i in $(seq $num_speakers); do
8197
i=$((i-1))
98+
cat ./generate_samples-$dst-$i.py
8299
python3 ./generate_samples-$dst-$i.py
83100
done
84101
ls -lh hf/piper/mp3/$lang/vits-piper-$lang-$name-$kind
Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2026 Xiaomi Corp. (authors: Fangjun Kuang)
3+
4+
# It may take 100 minutes to generate the lexicon.
5+
#
6+
# You can download a pre-generated one from
7+
# https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
8+
9+
from typing import List, Tuple
10+
11+
from pypinyin import phrases_dict, pinyin_dict
12+
13+
from piper.phonemize_chinese import ChinesePhonemizer
14+
15+
16+
def generate_chinese_lexicon():
17+
word_dict = pinyin_dict.pinyin_dict
18+
phrases = phrases_dict.phrases_dict
19+
20+
phonemizer = ChinesePhonemizer(model_dir="./abc")
21+
22+
lexicon = []
23+
for key in word_dict:
24+
if not (0x4E00 <= key <= 0x9FFF):
25+
continue
26+
w = chr(key)
27+
28+
phonemes = phonemizer.phonemize(w)[0]
29+
tokens = []
30+
for p in phonemes:
31+
tokens.append(p)
32+
if p in {"1", "2", "3", "4", "5"}:
33+
tokens.append("_")
34+
35+
lexicon.append((w, tokens))
36+
37+
for key in phrases:
38+
phonemes = phonemizer.phonemize(key)[0]
39+
tokens = []
40+
for p in phonemes:
41+
tokens.append(p)
42+
if p in {"1", "2", "3", "4", "5"}:
43+
tokens.append("_")
44+
45+
lexicon.append((key, tokens))
46+
47+
return lexicon
48+
49+
50+
def save(filename: str, lexicon: List[Tuple[str, List[str]]]):
51+
with open(filename, "w", encoding="utf-8") as f:
52+
for word, phones in lexicon:
53+
tokens = " ".join(phones)
54+
f.write(f"{word} {tokens}\n")
55+
56+
57+
def main():
58+
zh = generate_chinese_lexicon()
59+
60+
save("lexicon-zh-g2pw.txt", zh)
61+
62+
63+
if __name__ == "__main__":
64+
main()

scripts/piper/generate_samples.py.in

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,13 @@ config = sherpa_onnx.OfflineTtsConfig(
55
model=sherpa_onnx.OfflineTtsModelConfig(
66
vits=sherpa_onnx.OfflineTtsVitsModelConfig(
77
model="{{ model }}",
8-
lexicon="",
8+
lexicon="{{ lexicon }}",
99
data_dir="{{ data_dir }}",
1010
tokens="{{ tokens }}",
1111
),
1212
num_threads=1,
1313
),
14+
rule_fsts="{{ rule_fsts }}",
1415
)
1516

1617
if not config.validate():

0 commit comments

Comments
 (0)