Add Koltin and Java API for Omnilingual ASR CTC models (k2-fsa#2783)

csukuangfj · web-flow · commit 0f647d32423d · 2025-11-13T19:03:01.000+08:00
diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml
@@ -108,6 +108,20 @@ jobs:
           cd ./java-api-examples
           ./run-version-test.sh
 
+      - name:  Run java test (Omnilingual ASR CTC)
+        shell: bash
+        run: |
+          cd ./java-api-examples
+          ./run-non-streaming-decode-file-omnilingual-asr-ctc.sh
+          rm -rf sherpa-onnx-omnilingual-*
+
+      - name:  Run java test (WeNet CTC)
+        shell: bash
+        run: |
+          cd ./java-api-examples
+          ./run-non-streaming-decode-file-wenet-ctc.sh
+          rm -rf sherpa-onnx-wenet*
+
       - name:  Run java test (Streaming T-one)
         shell: bash
         run: |
diff --git a/java-api-examples/NonStreamingDecodeFileOmnilingualAsrCtc.java b/java-api-examples/NonStreamingDecodeFileOmnilingualAsrCtc.java
@@ -0,0 +1,54 @@
+// Copyright 2025 Xiaomi Corporation
+
+// This file shows how to use an offline Omnilingual ASR CTC model,
+// i.e., non-streaming Omnilingual ASR CTC model,
+// to decode files.
+import com.k2fsa.sherpa.onnx.*;
+
+public class NonStreamingDecodeFileOmnilingualAsrCtc {
+  public static void main(String[] args) {
+    // please refer to
+    // https://k2-fsa.github.io/sherpa/onnx/sense-voice/index.html
+    // to download model files
+    String model =
+        "sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12/model.int8.onnx";
+
+    String tokens =
+        "sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12/tokens.txt";
+
+    String waveFilename =
+        "./sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12/test_wavs/en.wav";
+
+    WaveReader reader = new WaveReader(waveFilename);
+
+    OfflineOmnilingualAsrCtcModelConfig omnilingual =
+        OfflineOmnilingualAsrCtcModelConfig.builder().setModel(model).build();
+
+    OfflineModelConfig modelConfig =
+        OfflineModelConfig.builder()
+            .setOmnilingual(omnilingual)
+            .setTokens(tokens)
+            .setNumThreads(1)
+            .setDebug(true)
+            .build();
+
+    OfflineRecognizerConfig config =
+        OfflineRecognizerConfig.builder()
+            .setOfflineModelConfig(modelConfig)
+            .setDecodingMethod("greedy_search")
+            .build();
+
+    OfflineRecognizer recognizer = new OfflineRecognizer(config);
+    OfflineStream stream = recognizer.createStream();
+    stream.acceptWaveform(reader.getSamples(), reader.getSampleRate());
+
+    recognizer.decode(stream);
+
+    String text = recognizer.getResult(stream).getText();
+
+    System.out.printf("filename:%s\nresult:%s\n", waveFilename, text);
+
+    stream.release();
+    recognizer.release();
+  }
+}
diff --git a/java-api-examples/run-non-streaming-decode-file-omnilingual-asr-ctc.sh b/java-api-examples/run-non-streaming-decode-file-omnilingual-asr-ctc.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib  && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then
+  mkdir -p ../build
+  pushd ../build
+  cmake \
+    -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
+    -DSHERPA_ONNX_ENABLE_TESTS=OFF \
+    -DSHERPA_ONNX_ENABLE_CHECK=OFF \
+    -DBUILD_SHARED_LIBS=ON \
+    -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
+    -DSHERPA_ONNX_ENABLE_JNI=ON \
+    ..
+
+  make -j4
+  ls -lh lib
+  popd
+fi
+
+if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then
+  pushd ../sherpa-onnx/java-api
+  make
+  popd
+fi
+
+if [ ! -f sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12/tokens.txt ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2
+  tar xvf sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2
+  rm sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2
+fi
+
+
+java \
+  -Djava.library.path=$PWD/../build/lib \
+  -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
+  NonStreamingDecodeFileOmnilingualAsrCtc.java
diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh
@@ -488,6 +488,28 @@ function testOfflineNeMoCanary() {
   java -Djava.library.path=../build/lib -jar $out_filename
 }
 
+
+function testOfflineOmnilingualAsrCtc() {
+  if [ ! -f sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12/tokens.txt ]; then
+    curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2
+    tar xvf sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2
+    rm sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12.tar.bz2
+  fi
+
+  out_filename=test_offline_omnilingual_asr_ctc.jar
+  kotlinc-jvm -include-runtime -d $out_filename \
+    test_offline_omnilingual_asr_ctc.kt \
+    FeatureConfig.kt \
+    HomophoneReplacerConfig.kt \
+    OfflineRecognizer.kt \
+    OfflineStream.kt \
+    WaveReader.kt \
+    faked-asset-manager.kt
+
+  ls -lh $out_filename
+  java -Djava.library.path=../build/lib -jar $out_filename
+}
+
 function testOfflineWenetCtc() {
   if [ ! -f sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10/model.int8.onnx ]; then
     curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10.tar.bz2
@@ -511,6 +533,7 @@ function testOfflineWenetCtc() {
 
 testVersion
 
+testOfflineOmnilingualAsrCtc
 testOfflineWenetCtc
 testOfflineNeMoCanary
 testOfflineSenseVoiceWithHr
diff --git a/kotlin-api-examples/test_offline_omnilingual_asr_ctc.kt b/kotlin-api-examples/test_offline_omnilingual_asr_ctc.kt
@@ -0,0 +1,31 @@
+package com.k2fsa.sherpa.onnx
+
+fun main() {
+  val recognizer = createOfflineRecognizer()
+  val waveFilename = "./sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12/test_wavs/en.wav"
+
+  val objArray = WaveReader.readWaveFromFile(
+      filename = waveFilename,
+  )
+  val samples: FloatArray = objArray[0] as FloatArray
+  val sampleRate: Int = objArray[1] as Int
+
+  var stream = recognizer.createStream()
+  stream.acceptWaveform(samples, sampleRate=sampleRate)
+  recognizer.decode(stream)
+
+  var result = recognizer.getResult(stream)
+  println(result)
+
+  stream.release()
+  recognizer.release()
+}
+
+
+fun createOfflineRecognizer(): OfflineRecognizer {
+  val config = OfflineRecognizerConfig(
+      modelConfig = getOfflineModelConfig(type = 44)!!,
+  )
+
+  return OfflineRecognizer(config = config)
+}
diff --git a/scripts/apk/generate-vad-asr-apk-script.py b/scripts/apk/generate-vad-asr-apk-script.py
@@ -743,6 +743,22 @@ def get_models():
 
             ls -lh
 
+            popd
+            """,
+        ),
+        Model(
+            model_name="sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12",
+            idx=44,
+            lang="1600",
+            lang2="1600_languages",
+            short_name="omnilingual_asr_300M_ctc_int8",
+            cmd="""
+            pushd $model_name
+
+            rm -rfv test_wavs
+
+            ls -lh
+
             popd
             """,
         ),
diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile
@@ -37,6 +37,7 @@ java_files += OfflineMoonshineModelConfig.java
 java_files += OfflineNemoEncDecCtcModelConfig.java
 java_files += OfflineZipformerCtcModelConfig.java
 java_files += OfflineWenetCtcModelConfig.java
+java_files += OfflineOmnilingualAsrCtcModelConfig.java
 java_files += OfflineCanaryModelConfig.java
 java_files += OfflineSenseVoiceModelConfig.java
 java_files += OfflineDolphinModelConfig.java
diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineModelConfig.java
@@ -13,6 +13,7 @@ public class OfflineModelConfig {
     private final OfflineDolphinModelConfig dolphin;
     private final OfflineZipformerCtcModelConfig zipformerCtc;
     private final OfflineWenetCtcModelConfig wenetCtc;
+    private final OfflineOmnilingualAsrCtcModelConfig omnilingual;
     private final OfflineCanaryModelConfig canary;
     private final String teleSpeech;
     private final String tokens;
@@ -34,6 +35,7 @@ private OfflineModelConfig(Builder builder) {
         this.zipformerCtc = builder.zipformerCtc;
         this.canary = builder.canary;
         this.wenetCtc = builder.wenetCtc;
+        this.omnilingual = builder.omnilingual;
         this.senseVoice = builder.senseVoice;
         this.dolphin = builder.dolphin;
         this.teleSpeech = builder.teleSpeech;
@@ -86,6 +88,10 @@ public OfflineWenetCtcModelConfig getWenetCtc() {
         return wenetCtc;
     }
 
+    public OfflineOmnilingualAsrCtcModelConfig getOmnilingual() {
+        return omnilingual;
+    }
+
     public OfflineCanaryModelConfig getCanary() {
         return canary;
     }
@@ -133,6 +139,7 @@ public static class Builder {
         private OfflineDolphinModelConfig dolphin = OfflineDolphinModelConfig.builder().build();
         private OfflineZipformerCtcModelConfig zipformerCtc = OfflineZipformerCtcModelConfig.builder().build();
         private OfflineWenetCtcModelConfig wenetCtc = OfflineWenetCtcModelConfig.builder().build();
+        private OfflineOmnilingualAsrCtcModelConfig omnilingual = OfflineOmnilingualAsrCtcModelConfig.builder().build();
         private OfflineCanaryModelConfig canary = OfflineCanaryModelConfig.builder().build();
         private String teleSpeech = "";
         private String tokens = "";
@@ -177,6 +184,11 @@ public Builder setWenetCtc(OfflineWenetCtcModelConfig wenetCtc) {
             return this;
         }
 
+        public Builder setOmnilingual(OfflineOmnilingualAsrCtcModelConfig omnilingual) {
+            this.omnilingual = omnilingual;
+            return this;
+        }
+
         public Builder setCanary(OfflineCanaryModelConfig canary) {
             this.canary = canary;
             return this;
@@ -242,4 +254,4 @@ public Builder setBpeVocab(String bpeVocab) {
             return this;
         }
     }
-}
+}
diff --git a/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineOmnilingualAsrCtcModelConfig.java b/sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineOmnilingualAsrCtcModelConfig.java
@@ -0,0 +1,30 @@
+package com.k2fsa.sherpa.onnx;
+
+public class OfflineOmnilingualAsrCtcModelConfig {
+    private final String model;
+
+    private OfflineOmnilingualAsrCtcModelConfig(Builder builder) {
+        this.model = builder.model;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public String getModel() {
+        return model;
+    }
+
+    public static class Builder {
+        private String model = "";
+
+        public OfflineOmnilingualAsrCtcModelConfig build() {
+            return new OfflineOmnilingualAsrCtcModelConfig(this);
+        }
+
+        public Builder setModel(String model) {
+            this.model = model;
+            return this;
+        }
+    }
+}
diff --git a/sherpa-onnx/jni/offline-recognizer.cc b/sherpa-onnx/jni/offline-recognizer.cc
@@ -193,6 +193,18 @@ static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config,
   SHERPA_ONNX_JNI_READ_STRING(ans.model_config.wenet_ctc.model, model,
                               wenet_ctc_config_cls, wenet_ctc_config);
 
+  // omnilingual asr ctc
+  fid = env->GetFieldID(
+      model_config_cls, "omnilingual",
+      "Lcom/k2fsa/sherpa/onnx/OfflineOmnilingualAsrCtcModelConfig;");
+  jobject omnilingual_ctc_config = env->GetObjectField(model_config, fid);
+  jclass omnilingual_ctc_config_cls =
+      env->GetObjectClass(omnilingual_ctc_config);
+
+  SHERPA_ONNX_JNI_READ_STRING(ans.model_config.omnilingual.model, model,
+                              omnilingual_ctc_config_cls,
+                              omnilingual_ctc_config);
+
   // canary
   fid = env->GetFieldID(model_config_cls, "canary",
                         "Lcom/k2fsa/sherpa/onnx/OfflineCanaryModelConfig;");
diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt
@@ -40,6 +40,10 @@ data class OfflineWenetCtcModelConfig(
     var model: String = "",
 )
 
+data class OfflineOmnilingualAsrCtcModelConfig(
+    var model: String = "",
+)
+
 data class OfflineWhisperModelConfig(
     var encoder: String = "",
     var decoder: String = "",
@@ -85,6 +89,7 @@ data class OfflineModelConfig(
     var dolphin: OfflineDolphinModelConfig = OfflineDolphinModelConfig(),
     var zipformerCtc: OfflineZipformerCtcModelConfig = OfflineZipformerCtcModelConfig(),
     var wenetCtc: OfflineWenetCtcModelConfig = OfflineWenetCtcModelConfig(),
+    var omnilingual: OfflineOmnilingualAsrCtcModelConfig = OfflineOmnilingualAsrCtcModelConfig(),
     var canary: OfflineCanaryModelConfig = OfflineCanaryModelConfig(),
     var teleSpeech: String = "",
     var numThreads: Int = 1,
@@ -731,6 +736,16 @@ fun getOfflineModelConfig(type: Int): OfflineModelConfig? {
                 modelType = "paraformer",
             )
         }
+
+        44 -> {
+            val modelDir = "sherpa-onnx-omnilingual-asr-1600-languages-300M-ctc-int8-2025-11-12"
+            return OfflineModelConfig(
+                omnilingual = OfflineOmnilingualAsrCtcModelConfig(
+                    model = "$modelDir/model.int8.onnx",
+                ),
+                tokens = "$modelDir/tokens.txt",
+            )
+        }
     }
     return null
 }
