Skip to content

Commit ceb1189

Browse files
dewana-slimdewan
dewana-sl
and
imdewan
authored
Add KittenTTS v0.8 support (#3591)
Co-authored-by: Dewan Shakil <mydsaproduction@gmail.com>
1 parent f95b772 commit ceb1189

19 files changed

Lines changed: 631 additions & 25 deletions

‎.github/workflows/export-kitten.yaml‎

Lines changed: 49 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,16 @@ jobs:
2020
fail-fast: false
2121
matrix:
2222
os: [ubuntu-latest]
23-
version: ["nano_v0_1", "nano_v0_2", "mini_v0_1"]
23+
version:
24+
[
25+
"nano_v0_1",
26+
"nano_v0_2",
27+
"mini_v0_1",
28+
"nano_v0_8_fp32",
29+
"nano_v0_8_int8",
30+
"micro_v0_8",
31+
"mini_v0_8",
32+
]
2433
python-version: ["3.10"]
2534

2635
steps:
@@ -41,8 +50,23 @@ jobs:
4150
HF_TOKEN: ${{ secrets.HF_TOKEN }}
4251
shell: bash
4352
run: |
44-
cd scripts/kitten-tts/${{ matrix.version }}
45-
./run.sh
53+
version=${{ matrix.version }}
54+
if [[ $version == "nano_v0_8_fp32" ]]; then
55+
cd scripts/kitten-tts/v0_8
56+
./run.sh KittenML/kitten-tts-nano-0.8-fp32
57+
elif [[ $version == "nano_v0_8_int8" ]]; then
58+
cd scripts/kitten-tts/v0_8
59+
./run.sh KittenML/kitten-tts-nano-0.8-int8
60+
elif [[ $version == "micro_v0_8" ]]; then
61+
cd scripts/kitten-tts/v0_8
62+
./run.sh KittenML/kitten-tts-micro-0.8
63+
elif [[ $version == "mini_v0_8" ]]; then
64+
cd scripts/kitten-tts/v0_8
65+
./run.sh KittenML/kitten-tts-mini-0.8
66+
else
67+
cd scripts/kitten-tts/$version
68+
./run.sh
69+
fi
4670
4771
- name: Collect results
4872
shell: bash
@@ -54,13 +78,30 @@ jobs:
5478
version=${{ matrix.version }}
5579
5680
src=scripts/kitten-tts/$version
81+
model_file=model.fp16.onnx
5782
5883
if [[ $version == "nano_v0_1" ]]; then
5984
d=kitten-nano-en-v0_1-fp16
6085
elif [[ $version == "nano_v0_2" ]]; then
6186
d=kitten-nano-en-v0_2-fp16
6287
elif [[ $version == "mini_v0_1" ]]; then
6388
d=kitten-mini-en-v0_1-fp16
89+
elif [[ $version == "nano_v0_8_fp32" ]]; then
90+
src=scripts/kitten-tts/v0_8
91+
d=kitten-nano-en-v0_8-fp32
92+
model_file=model.fp32.onnx
93+
elif [[ $version == "nano_v0_8_int8" ]]; then
94+
src=scripts/kitten-tts/v0_8
95+
d=kitten-nano-en-v0_8-int8
96+
model_file=model.int8.onnx
97+
elif [[ $version == "micro_v0_8" ]]; then
98+
src=scripts/kitten-tts/v0_8
99+
d=kitten-micro-en-v0_8
100+
model_file=model.onnx
101+
elif [[ $version == "mini_v0_8" ]]; then
102+
src=scripts/kitten-tts/v0_8
103+
d=kitten-mini-en-v0_8
104+
model_file=model.onnx
64105
else
65106
echo "version $version"
66107
exit 1
@@ -69,7 +110,7 @@ jobs:
69110
mkdir $d
70111
cp -a LICENSE $d/LICENSE
71112
cp -a espeak-ng-data $d/
72-
cp -v $src/model.fp16.onnx $d/model.fp16.onnx
113+
cp -v $src/$model_file $d/$model_file
73114
cp -v $src/voices.bin $d/
74115
cp -v $src/tokens.txt $d/
75116
cp -v $src/../README.md $d/README.md
@@ -114,6 +155,10 @@ jobs:
114155
kitten-nano-en-v0_1-fp16
115156
kitten-nano-en-v0_2-fp16
116157
kitten-mini-en-v0_1-fp16
158+
kitten-nano-en-v0_8-fp32
159+
kitten-nano-en-v0_8-int8
160+
kitten-micro-en-v0_8
161+
kitten-mini-en-v0_8
117162
)
118163
119164
export GIT_LFS_SKIP_SMUDGE=1

‎flutter-examples/tts/lib/model.dart‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ Future<sherpa_onnx.OfflineTts> createOfflineTts() async {
2424

2525
String modelDir = '';
2626
String modelName = '';
27-
String voices = ''; // for Kokoro only
27+
String voices = ''; // for Kokoro or Kitten
28+
bool isKitten = false;
2829
String ruleFsts = '';
2930
String ruleFars = '';
3031
String lexicon = '';
@@ -97,6 +98,14 @@ Future<sherpa_onnx.OfflineTts> createOfflineTts() async {
9798
// dataDir = 'kokoro-multi-lang-v1_0/espeak-ng-data';
9899
// lexicon = 'kokoro-multi-lang-v1_0/lexicon-us-en.txt,kokoro-multi-lang-v1_0/lexicon-zh.txt';
99100

101+
// Example 10
102+
// https://github.com/k2-fsa/sherpa-onnx/releases/tag/tts-models
103+
// modelDir = 'kitten-nano-en-v0_8-fp32';
104+
// modelName = 'model.fp32.onnx';
105+
// voices = 'voices.bin';
106+
// dataDir = 'kitten-nano-en-v0_8-fp32/espeak-ng-data';
107+
// isKitten = true;
108+
100109
// ============================================================
101110
// Please don't change the remaining part of this function
102111
// ============================================================
@@ -148,9 +157,20 @@ Future<sherpa_onnx.OfflineTts> createOfflineTts() async {
148157

149158
late final sherpa_onnx.OfflineTtsVitsModelConfig vits;
150159
late final sherpa_onnx.OfflineTtsKokoroModelConfig kokoro;
160+
late final sherpa_onnx.OfflineTtsKittenModelConfig kitten;
151161

152-
if (voices != '') {
162+
if (isKitten) {
163+
vits = sherpa_onnx.OfflineTtsVitsModelConfig();
164+
kokoro = sherpa_onnx.OfflineTtsKokoroModelConfig();
165+
kitten = sherpa_onnx.OfflineTtsKittenModelConfig(
166+
model: modelName,
167+
voices: voices,
168+
tokens: tokens,
169+
dataDir: dataDir,
170+
);
171+
} else if (voices != '') {
153172
vits = sherpa_onnx.OfflineTtsVitsModelConfig();
173+
kitten = sherpa_onnx.OfflineTtsKittenModelConfig();
154174
kokoro = sherpa_onnx.OfflineTtsKokoroModelConfig(
155175
model: modelName,
156176
voices: voices,
@@ -167,11 +187,13 @@ Future<sherpa_onnx.OfflineTts> createOfflineTts() async {
167187
);
168188

169189
kokoro = sherpa_onnx.OfflineTtsKokoroModelConfig();
190+
kitten = sherpa_onnx.OfflineTtsKittenModelConfig();
170191
}
171192

172193
final modelConfig = sherpa_onnx.OfflineTtsModelConfig(
173194
vits: vits,
174195
kokoro: kokoro,
196+
kitten: kitten,
175197
numThreads: 2,
176198
debug: true,
177199
provider: 'cpu',

‎scripts/apk/generate-tts-apk-script.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -562,6 +562,26 @@ def get_kitten_models() -> List[TtsModel]:
562562
model_name="model.fp16.onnx",
563563
lang="en",
564564
),
565+
TtsModel(
566+
model_dir="kitten-nano-en-v0_8-fp32",
567+
model_name="model.fp32.onnx",
568+
lang="en",
569+
),
570+
TtsModel(
571+
model_dir="kitten-nano-en-v0_8-int8",
572+
model_name="model.int8.onnx",
573+
lang="en",
574+
),
575+
TtsModel(
576+
model_dir="kitten-micro-en-v0_8",
577+
model_name="model.onnx",
578+
lang="en",
579+
),
580+
TtsModel(
581+
model_dir="kitten-mini-en-v0_8",
582+
model_name="model.onnx",
583+
lang="en",
584+
),
565585
]
566586
for m in english_models:
567587
m.data_dir = f"{m.model_dir}/espeak-ng-data"

‎scripts/flutter/build-android-tts.sh.in‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,11 @@ sed -i.bak s/"modelName = ''"/"modelName = \"$model_name\""/ ./model.dart
6262
sed -i.bak "s|lexicon = ''|lexicon = \"lexicon.txt\"|" ./model.dart
6363
{% endif %}
6464

65+
{% if tts_model.is_kitten %}
66+
sed -i.bak "s|voices = ''|voices = \"voices.bin\"|" ./model.dart
67+
sed -i.bak "s|isKitten = false|isKitten = true|" ./model.dart
68+
{% endif %}
69+
6570
git status
6671

6772
git diff .

‎scripts/flutter/build-linux-tts.sh.in‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ sed -i.bak s/"modelName = ''"/"modelName = \"$model_name\""/ ./model.dart
6969
sed -i.bak "s|lexicon = ''|lexicon = \"lexicon.txt\"|" ./model.dart
7070
{% endif %}
7171

72+
{% if tts_model.is_kitten %}
73+
sed -i.bak "s|voices = ''|voices = \"voices.bin\"|" ./model.dart
74+
sed -i.bak "s|isKitten = false|isKitten = true|" ./model.dart
75+
{% endif %}
76+
7277
git status
7378

7479
git diff .

‎scripts/flutter/build-macos-tts.sh.in‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ sed -i.bak s/"modelName = ''"/"modelName = \"$model_name\""/ ./model.dart
6969
sed -i.bak "s|lexicon = ''|lexicon = \"lexicon.txt\"|" ./model.dart
7070
{% endif %}
7171

72+
{% if tts_model.is_kitten %}
73+
sed -i.bak "s|voices = ''|voices = \"voices.bin\"|" ./model.dart
74+
sed -i.bak "s|isKitten = false|isKitten = true|" ./model.dart
75+
{% endif %}
76+
7277
git status
7378

7479
git diff .

‎scripts/flutter/build-windows-tts.sh.in‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,11 @@ sed -i.bak s/"modelName = ''"/"modelName = \"$model_name\""/ ./model.dart
7171
sed -i.bak "s|lexicon = ''|lexicon = \"lexicon.txt\"|" ./model.dart
7272
{% endif %}
7373

74+
{% if tts_model.is_kitten %}
75+
sed -i.bak "s|voices = ''|voices = \"voices.bin\"|" ./model.dart
76+
sed -i.bak "s|isKitten = false|isKitten = true|" ./model.dart
77+
{% endif %}
78+
7479
git status
7580

7681
git diff .

‎scripts/flutter/generate-tts.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ class TtsModel:
3434
data_dir: Optional[str] = None
3535
dict_dir: Optional[str] = None
3636
is_char: bool = False
37+
is_kitten: bool = False
3738

3839

3940
def get_coqui_models() -> List[TtsModel]:
@@ -364,6 +365,37 @@ def get_vits_models() -> List[TtsModel]:
364365
return all_models
365366

366367

368+
def get_kitten_models() -> List[TtsModel]:
369+
models = [
370+
TtsModel(
371+
model_dir="kitten-nano-en-v0_8-fp32",
372+
model_name="model.fp32.onnx",
373+
lang="en",
374+
),
375+
TtsModel(
376+
model_dir="kitten-nano-en-v0_8-int8",
377+
model_name="model.int8.onnx",
378+
lang="en",
379+
),
380+
TtsModel(
381+
model_dir="kitten-micro-en-v0_8",
382+
model_name="model.onnx",
383+
lang="en",
384+
),
385+
TtsModel(
386+
model_dir="kitten-mini-en-v0_8",
387+
model_name="model.onnx",
388+
lang="en",
389+
),
390+
]
391+
392+
for m in models:
393+
m.data_dir = f"{m.model_dir}/espeak-ng-data"
394+
m.is_kitten = True
395+
396+
return models
397+
398+
367399
def main():
368400
args = get_args()
369401
index = args.index
@@ -375,6 +407,7 @@ def main():
375407
all_model_list += get_piper_models()
376408
all_model_list += get_mimic3_models()
377409
all_model_list += get_coqui_models()
410+
all_model_list += get_kitten_models()
378411

379412
num_models = len(all_model_list)
380413

‎scripts/kitten-tts/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,24 @@
11
# Introduction
22

33
See also https://github.com/KittenML/KittenTTS
4+
5+
## KittenTTS v0.8
6+
7+
Use `v0_8/run.sh` to prepare a sherpa-onnx compatible model directory from
8+
the upstream Hugging Face assets:
9+
10+
```bash
11+
cd scripts/kitten-tts/v0_8
12+
python3 -m pip install numpy onnx
13+
./run.sh KittenML/kitten-tts-nano-0.8-fp32
14+
./run.sh KittenML/kitten-tts-nano-0.8-int8
15+
./run.sh KittenML/kitten-tts-micro-0.8
16+
./run.sh KittenML/kitten-tts-mini-0.8
17+
```
18+
19+
The generated `voices.bin` preserves all v0.8 reference rows. The ONNX metadata
20+
marks v0.8 models with `version=8`, `end_id=10`, `add_pad_after_end=1`, and
21+
`max_token_len=400` so the runtime selects the same style row as upstream.
22+
Use package directories like `kitten-mini-en-v0_8`,
23+
`kitten-micro-en-v0_8`, `kitten-nano-en-v0_8-fp32`, or
24+
`kitten-nano-en-v0_8-int8`.

0 commit comments

Comments
 (0)