Support extra languages in multi-lang kokoro tts (#2303)

Author: csukuangfj

.github/workflows/test-build-wheel.yaml

@@ -35,18 +35,18 @@ jobs:
       matrix:
         # See https://github.com/actions/runner-images
         include:
-          - os: ubuntu-22.04
-            python-version: "3.7"
-          - os: ubuntu-22.04
+          - os: ubuntu-latest
             python-version: "3.8"
-          - os: ubuntu-22.04
+          - os: ubuntu-latest
             python-version: "3.9"
-          - os: ubuntu-22.04
+          - os: ubuntu-latest
             python-version: "3.10"
-          - os: ubuntu-22.04
+          - os: ubuntu-latest
             python-version: "3.11"
-          - os: ubuntu-22.04
+          - os: ubuntu-latest
             python-version: "3.12"
+          - os: ubuntu-latest
+            python-version: "3.13"
 
           - os: macos-13
             python-version: "3.8"
@@ -103,7 +103,7 @@ jobs:
           export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
           cmake --version
 
-          export SHERPA_ONNX_MAKE_ARGS="VERBOSE=1 -j"
+          export SHERPA_ONNX_MAKE_ARGS="VERBOSE=1 -j2"
 
           python3 setup.py bdist_wheel
           ls -lh dist

README.md

@@ -1,18 +1,18 @@
 ### Supported functions
 
-|Speech recognition| Speech synthesis | Source separation |
+|Speech recognition| [Speech synthesis][tts-url] | [Source separation][ss-url] |
 |------------------|------------------|-------------------|
 |   ✔️              |         ✔️        |       ✔️           |
 
-|Speaker identification| Speaker diarization | Speaker verification |
+|Speaker identification| [Speaker diarization][sd-url] | Speaker verification |
 |----------------------|-------------------- |------------------------|
 |   ✔️                  |         ✔️           |            ✔️           |
 
-| Spoken Language identification | Audio tagging | Voice activity detection |
+| [Spoken Language identification][slid-url] | [Audio tagging][at-url] | [Voice activity detection][vad-url] |
 |--------------------------------|---------------|--------------------------|
 |                 ✔️              |          ✔️    |                ✔️         |
 
-| Keyword spotting | Add punctuation | Speech enhancement |
+| [Keyword spotting][kws-url] | [Add punctuation][punct-url] | [Speech enhancement][se-url] |
 |------------------|-----------------|--------------------|
 |     ✔️            |       ✔️         |      ✔️             |
 
@@ -501,3 +501,12 @@ It uses sherpa-onnx for speech-to-text and text-to-speech.
 [spleeter]: https://github.com/deezer/spleeter
 [UVR]: https://github.com/Anjok07/ultimatevocalremovergui
 [gtcrn]: https://github.com/Xiaobin-Rong/gtcrn
+[tts-url]: https://k2-fsa.github.io/sherpa/onnx/tts/all-in-one.html
+[ss-url]: https://k2-fsa.github.io/sherpa/onnx/source-separation/index.html
+[sd-url]: https://k2-fsa.github.io/sherpa/onnx/speaker-diarization/index.html
+[slid-url]: https://k2-fsa.github.io/sherpa/onnx/spoken-language-identification/index.html
+[at-url]: https://k2-fsa.github.io/sherpa/onnx/audio-tagging/index.html
+[vad-url]: https://k2-fsa.github.io/sherpa/onnx/vad/index.html
+[kws-url]: https://k2-fsa.github.io/sherpa/onnx/kws/index.html
+[punct-url]: https://k2-fsa.github.io/sherpa/onnx/punctuation/index.html
+[se-url]: https://k2-fsa.github.io/sherpa/onnx/speech-enhancment/index.html

flutter/sherpa_onnx/lib/src/sherpa_onnx_bindings.dart

@@ -201,6 +201,7 @@ final class SherpaOnnxOfflineTtsKokoroModelConfig extends Struct {
   external double lengthScale;
   external Pointer<Utf8> dictDir;
   external Pointer<Utf8> lexicon;
+  external Pointer<Utf8> lang;
 }
 
 final class SherpaOnnxOfflineTtsModelConfig extends Struct {

flutter/sherpa_onnx/lib/src/tts.dart

@@ -117,6 +117,7 @@ class OfflineTtsKokoroModelConfig {
     this.lengthScale = 1.0,
     this.dictDir = '',
     this.lexicon = '',
+    this.lang = '',
   });
 
   factory OfflineTtsKokoroModelConfig.fromJson(Map<String, dynamic> json) {
@@ -128,12 +129,13 @@ class OfflineTtsKokoroModelConfig {
       lengthScale: (json['lengthScale'] as num?)?.toDouble() ?? 1.0,
       dictDir: json['dictDir'] as String? ?? '',
       lexicon: json['lexicon'] as String? ?? '',
+      lang: json['lang'] as String? ?? '',
     );
   }
 
   @override
   String toString() {
-    return 'OfflineTtsKokoroModelConfig(model: $model, voices: $voices, tokens: $tokens, dataDir: $dataDir, lengthScale: $lengthScale, dictDir: $dictDir, lexicon: $lexicon)';
+    return 'OfflineTtsKokoroModelConfig(model: $model, voices: $voices, tokens: $tokens, dataDir: $dataDir, lengthScale: $lengthScale, dictDir: $dictDir, lexicon: $lexicon, lang: $lang)';
   }
 
   Map<String, dynamic> toJson() => {
@@ -144,6 +146,7 @@ class OfflineTtsKokoroModelConfig {
         'lengthScale': lengthScale,
         'dictDir': dictDir,
         'lexicon': lexicon,
+        'lang': lang,
       };
 
   final String model;
@@ -153,6 +156,7 @@ class OfflineTtsKokoroModelConfig {
   final double lengthScale;
   final String dictDir;
   final String lexicon;
+  final String lang;
 }
 
 class OfflineTtsModelConfig {
@@ -286,6 +290,7 @@ class OfflineTts {
     c.ref.model.kokoro.lengthScale = config.model.kokoro.lengthScale;
     c.ref.model.kokoro.dictDir = config.model.kokoro.dictDir.toNativeUtf8();
     c.ref.model.kokoro.lexicon = config.model.kokoro.lexicon.toNativeUtf8();
+    c.ref.model.kokoro.lang = config.model.kokoro.lang.toNativeUtf8();
 
     c.ref.model.numThreads = config.model.numThreads;
     c.ref.model.debug = config.model.debug ? 1 : 0;
@@ -302,6 +307,7 @@ class OfflineTts {
     calloc.free(c.ref.ruleFsts);
     calloc.free(c.ref.model.provider);
 
+    calloc.free(c.ref.model.kokoro.lang);
     calloc.free(c.ref.model.kokoro.lexicon);
     calloc.free(c.ref.model.kokoro.dictDir);
     calloc.free(c.ref.model.kokoro.dataDir);

harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/cpp/non-streaming-tts.cc

@@ -70,6 +70,7 @@ static SherpaOnnxOfflineTtsKokoroModelConfig GetOfflineTtsKokoroModelConfig(
   SHERPA_ONNX_ASSIGN_ATTR_FLOAT(length_scale, lengthScale);
   SHERPA_ONNX_ASSIGN_ATTR_STR(dict_dir, dictDir);
   SHERPA_ONNX_ASSIGN_ATTR_STR(lexicon, lexicon);
+  SHERPA_ONNX_ASSIGN_ATTR_STR(lang, lang);
 
   return c;
 }
@@ -177,6 +178,7 @@ static Napi::External<SherpaOnnxOfflineTts> CreateOfflineTtsWrapper(
   SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.data_dir);
   SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.dict_dir);
   SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.lexicon);
+  SHERPA_ONNX_DELETE_C_STR(c.model.kokoro.lang);
 
   SHERPA_ONNX_DELETE_C_STR(c.model.provider);
 

harmony-os/SherpaOnnxHar/sherpa_onnx/src/main/ets/components/NonStreamingTts.ets

@@ -36,6 +36,7 @@ export class OfflineTtsKokoroModelConfig {
   public lengthScale: number = 1.0;
   public dictDir: string = '';
   public lexicon: string = '';
+  public lang: string = '';
 }
 
 export class OfflineTtsModelConfig {

scripts/dotnet/OfflineTtsKokoroModelConfig.cs

@@ -18,6 +18,7 @@ public OfflineTtsKokoroModelConfig()
 
             DictDir = "";
             Lexicon = "";
+            Lang = "";
         }
         [MarshalAs(UnmanagedType.LPStr)]
         public string Model;
@@ -38,5 +39,8 @@ public OfflineTtsKokoroModelConfig()
 
         [MarshalAs(UnmanagedType.LPStr)]
         public string Lexicon;
+
+        [MarshalAs(UnmanagedType.LPStr)]
+        public string Lang;
     }
 }

scripts/go/sherpa_onnx.go

@@ -857,6 +857,7 @@ type OfflineTtsKokoroModelConfig struct {
 	DataDir     string  // Path to espeak-ng-data directory
 	DictDir     string  // Path to dict directory
 	Lexicon     string  // Path to lexicon files
+	Lang        string  // Example: es for Spanish, fr-fr for French. Can be empty
 	LengthScale float32 // Please use 1.0 in general. Smaller -> Faster speech speed. Larger -> Slower speech speed
 }
 
@@ -1006,6 +1007,9 @@ func NewOfflineTts(config *OfflineTtsConfig) *OfflineTts {
 	c.model.kokoro.lexicon = C.CString(config.Model.Kokoro.Lexicon)
 	defer C.free(unsafe.Pointer(c.model.kokoro.lexicon))
 
+	c.model.kokoro.lang = C.CString(config.Model.Kokoro.Lang)
+	defer C.free(unsafe.Pointer(c.model.kokoro.lang))
+
 	c.model.kokoro.length_scale = C.float(config.Model.Kokoro.LengthScale)
 
 	c.model.num_threads = C.int(config.Model.NumThreads)

sherpa-onnx/c-api/c-api.cc

@@ -1164,6 +1164,7 @@ static sherpa_onnx::OfflineTtsConfig GetOfflineTtsConfig(
       SHERPA_ONNX_OR(config->model.kokoro.dict_dir, "");
   tts_config.model.kokoro.lexicon =
       SHERPA_ONNX_OR(config->model.kokoro.lexicon, "");
+  tts_config.model.kokoro.lang = SHERPA_ONNX_OR(config->model.kokoro.lang, "");
 
   tts_config.model.num_threads = SHERPA_ONNX_OR(config->model.num_threads, 1);
   tts_config.model.debug = config->model.debug;

sherpa-onnx/c-api/c-api.h

@@ -958,6 +958,7 @@ SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsKokoroModelConfig {
   float length_scale;  // < 1, faster in speech speed; > 1, slower in speed
   const char *dict_dir;
   const char *lexicon;
+  const char *lang;
 } SherpaOnnxOfflineTtsKokoroModelConfig;
 
 SHERPA_ONNX_API typedef struct SherpaOnnxOfflineTtsModelConfig {