Skip to content

Commit 6cb5005

Browse files
csukuangfjcsukuangfj
authored
Add Android demo for Paraformer ASR with Qualcomm NPU. (#2932)
This pull request significantly advances the Android demo by integrating Paraformer ASR models with Qualcomm NPU acceleration. The core changes involve extending the model configuration to support QNN-specific parameters for Paraformer, introducing robust asset management utilities for handling multi-file QNN models, and updating the Android application's initialization logic to prepare these models for NPU inference. Additionally, the build scripts have been adjusted to reflect the new model additions and naming conventions, ensuring a comprehensive update for NPU-powered ASR on Android.
1 parent cdbf0a8 commit 6cb5005

9 files changed

Lines changed: 239 additions & 44 deletions

File tree

‎android/SherpaOnnxSimulateStreamingAsr/app/src/main/java/com/k2fsa/sherpa/onnx/simulate/streaming/asr/SimulateStreamingAsr.kt‎

Lines changed: 57 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,23 @@ fun assetExists(assetManager: AssetManager, path: String): Boolean {
2323
return files.contains(fileName)
2424
}
2525

26+
fun assetListExists(
27+
assetManager: AssetManager,
28+
paths: String
29+
): Boolean {
30+
if (paths.isBlank()) return false
31+
32+
val pathList = paths.split(",")
33+
.map { it.trim() }
34+
.filter { it.isNotEmpty() }
35+
36+
if (pathList.isEmpty()) return false
37+
38+
return pathList.all { path ->
39+
assetExists(assetManager, path)
40+
}
41+
}
42+
2643
fun copyAssetToInternalStorage(path: String, context: Context): String {
2744
val targetRoot = context.filesDir
2845
val outFile = File(targetRoot, path)
@@ -56,6 +73,23 @@ fun copyAssetToInternalStorage(path: String, context: Context): String {
5673
return outFile.absolutePath
5774
}
5875

76+
fun copyAssetListToInternalStorage(
77+
paths: String,
78+
context: Context
79+
): String {
80+
if (paths.isBlank()) return paths
81+
82+
val pathList = paths.split(",")
83+
.map { it.trim() }
84+
.filter { it.isNotEmpty() }
85+
86+
val copiedPaths = pathList.map { path ->
87+
copyAssetToInternalStorage(path, context)
88+
}
89+
90+
return copiedPaths.joinToString(",")
91+
}
92+
5993

6094
object SimulateStreamingAsr {
6195
private var _recognizer: OfflineRecognizer? = null
@@ -125,7 +159,10 @@ object SimulateStreamingAsr {
125159
OfflineRecognizer.prependAdspLibraryPath(context.applicationInfo.nativeLibraryDir)
126160

127161
// for qnn, we need to copy *.so files from assets folder to sd card
128-
if (config.modelConfig.senseVoice.qnnConfig.backendLib.isEmpty() && config.modelConfig.zipformerCtc.qnnConfig.backendLib.isEmpty()) {
162+
if (config.modelConfig.senseVoice.qnnConfig.backendLib.isEmpty()
163+
&& config.modelConfig.zipformerCtc.qnnConfig.backendLib.isEmpty()
164+
&& config.modelConfig.paraformer.qnnConfig.backendLib.isEmpty()
165+
) {
129166
Log.e(TAG, "You should provide libQnnHtp.so for qnn")
130167
throw IllegalArgumentException("You should provide libQnnHtp.so for qnn")
131168
}
@@ -166,6 +203,25 @@ object SimulateStreamingAsr {
166203
config.modelConfig.zipformerCtc.qnnConfig.contextBinary,
167204
context
168205
)
206+
} else if (config.modelConfig.paraformer.model.isNotEmpty()
207+
|| assetListExists(
208+
context.assets,
209+
config.modelConfig.paraformer.qnnConfig.contextBinary
210+
)
211+
) {
212+
if (config.modelConfig.paraformer.model.isNotEmpty()) {
213+
config.modelConfig.paraformer.model =
214+
copyAssetListToInternalStorage(
215+
config.modelConfig.paraformer.model,
216+
context
217+
)
218+
}
219+
220+
config.modelConfig.paraformer.qnnConfig.contextBinary =
221+
copyAssetListToInternalStorage(
222+
config.modelConfig.paraformer.qnnConfig.contextBinary,
223+
context
224+
)
169225
}
170226

171227
if (config.hr.lexicon.isNotEmpty()) {

‎android/SherpaOnnxSimulateStreamingAsr/app/src/main/java/com/k2fsa/sherpa/onnx/simulate/streaming/asr/screens/Home.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ fun HomeScreen() {
163163
offset += windowSize
164164
if (!isSpeechStarted && SimulateStreamingAsr.vad.isSpeechDetected()) {
165165
isSpeechStarted = true
166-
// offset 0.25s
166+
// offset 0.4s
167167
speechStartOffset = offset - 6400
168168
if(speechStartOffset < 0) {
169169
speechStartOffset = 0

‎scripts/apk/generate-asr-2pass-apk-script.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -114,15 +114,14 @@ def get_2nd_models():
114114
""",
115115
),
116116
Model(
117-
model_name="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17",
117+
model_name="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17",
118118
idx=15,
119119
lang="zh_en_ko_ja_yue",
120120
short_name="sense_voice_2024_07_17_int8",
121121
cmd="""
122122
pushd $model_name
123123
124124
rm -rfv test_wavs
125-
rm -fv model.onnx
126125
rm -fv *.py
127126
128127
ls -lh
@@ -410,7 +409,7 @@ def get_models():
410409
second_zh = [
411410
"sherpa-onnx-paraformer-zh-2023-09-14",
412411
"icefall-asr-zipformer-wenetspeech-20230615",
413-
"sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17",
412+
"sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17",
414413
"sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2025-09-09",
415414
"sherpa-onnx-dolphin-base-ctc-multi-lang-int8-2025-04-02",
416415
"sherpa-onnx-wenetspeech-yue-u2pp-conformer-ctc-zh-en-cantonese-int8-2025-09-10",
@@ -434,7 +433,7 @@ def get_models():
434433
),
435434
(
436435
"sherpa-onnx-streaming-zipformer-en-20M-2023-02-17",
437-
"sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17",
436+
"sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17",
438437
),
439438
(
440439
"sherpa-onnx-streaming-zipformer-en-20M-2023-02-17",

‎scripts/apk/generate-qnn-vad-asr-apk-script.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -399,6 +399,38 @@ def get_models():
399399
400400
ls -lh
401401
402+
popd
403+
""",
404+
),
405+
Model(
406+
model_name="sherpa-onnx-qnn-5-seconds-paraformer-zh-2023-03-28-int8",
407+
idx=9023,
408+
lang="zh",
409+
short_name="5-seconds-paraformer_zh_2023_03_28_int8",
410+
use_hr=True,
411+
cmd="""
412+
pushd $model_name
413+
414+
rm -rfv test_wavs
415+
416+
ls -lh
417+
418+
popd
419+
""",
420+
),
421+
Model(
422+
model_name="sherpa-onnx-qnn-5-seconds-paraformer-zh-2025-10-07-int8",
423+
idx=9024,
424+
lang="zh",
425+
short_name="5-seconds-paraformer_zh_2025_10_07_int8",
426+
use_hr=True,
427+
cmd="""
428+
pushd $model_name
429+
430+
rm -rfv test_wavs
431+
432+
ls -lh
433+
402434
popd
403435
""",
404436
),

‎scripts/apk/generate-vad-asr-apk-script.py‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ def get_models():
9595
""",
9696
),
9797
Model(
98-
model_name="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-2024-07-17",
98+
model_name="sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17",
9999
idx=15,
100100
lang="zh_en_ko_ja_yue",
101101
lang2="中英粤日韩",
@@ -105,7 +105,6 @@ def get_models():
105105
pushd $model_name
106106
107107
rm -rfv test_wavs
108-
rm -fv model.onnx
109108
rm -fv *.py
110109
111110
ls -lh

‎sherpa-onnx/csrc/qnn/offline-paraformer-model-qnn.cc‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,9 @@ class OfflineParaformerModelQnn::Impl {
9797

9898
template <typename Manager>
9999
Impl(Manager *mgr, const OfflineModelConfig &config) {
100-
SHERPA_ONNX_LOGE("TODO(fangjun): To be implemented");
100+
SHERPA_ONNX_LOGE(
101+
"Please copy all files from assets to SD card and set assetManager to "
102+
"null");
101103
SHERPA_ONNX_EXIT(-1);
102104
}
103105

‎sherpa-onnx/java-api/src/main/java/com/k2fsa/sherpa/onnx/OfflineParaformerModelConfig.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@
44

55
public class OfflineParaformerModelConfig {
66
private final String model;
7+
private final QnnConfig qnnConfig;
78

89
private OfflineParaformerModelConfig(Builder builder) {
910
this.model = builder.model;
11+
this.qnnConfig = builder.qnnConfig;
1012
}
1113

1214
public static Builder builder() {
@@ -17,9 +19,13 @@ public String getModel() {
1719
return model;
1820
}
1921

22+
public QnnConfig getQnnConfig() {
23+
return qnnConfig;
24+
}
2025

2126
public static class Builder {
2227
private String model = "";
28+
private QnnConfig qnnConfig = QnnConfig.builder().build();
2329

2430
public OfflineParaformerModelConfig build() {
2531
return new OfflineParaformerModelConfig(this);
@@ -29,5 +35,10 @@ public Builder setModel(String model) {
2935
this.model = model;
3036
return this;
3137
}
38+
39+
public Builder setQnnConfig(QnnConfig qnnConfig) {
40+
this.qnnConfig = qnnConfig;
41+
return this;
42+
}
3243
}
3344
}

‎sherpa-onnx/jni/offline-recognizer.cc‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,22 @@ static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config,
9999
SHERPA_ONNX_JNI_READ_STRING(ans.model_config.paraformer.model, model,
100100
paraformer_config_cls, paraformer_config);
101101

102+
fid = env->GetFieldID(paraformer_config_cls, "qnnConfig",
103+
"Lcom/k2fsa/sherpa/onnx/QnnConfig;");
104+
jobject qnn_config = env->GetObjectField(paraformer_config, fid);
105+
jclass qnn_config_cls = env->GetObjectClass(qnn_config);
106+
107+
SHERPA_ONNX_JNI_READ_STRING(
108+
ans.model_config.paraformer.qnn_config.backend_lib, backendLib,
109+
qnn_config_cls, qnn_config);
110+
111+
SHERPA_ONNX_JNI_READ_STRING(
112+
ans.model_config.paraformer.qnn_config.context_binary, contextBinary,
113+
qnn_config_cls, qnn_config);
114+
115+
SHERPA_ONNX_JNI_READ_STRING(ans.model_config.paraformer.qnn_config.system_lib,
116+
systemLib, qnn_config_cls, qnn_config);
117+
102118
fid = env->GetFieldID(model_config_cls, "whisper",
103119
"Lcom/k2fsa/sherpa/onnx/OfflineWhisperModelConfig;");
104120
jobject whisper_config = env->GetObjectField(model_config, fid);
@@ -168,8 +184,8 @@ static OfflineRecognizerConfig GetOfflineConfig(JNIEnv *env, jobject config,
168184

169185
fid = env->GetFieldID(sense_voice_config_cls, "qnnConfig",
170186
"Lcom/k2fsa/sherpa/onnx/QnnConfig;");
171-
jobject qnn_config = env->GetObjectField(sense_voice_config, fid);
172-
jclass qnn_config_cls = env->GetObjectClass(qnn_config);
187+
qnn_config = env->GetObjectField(sense_voice_config, fid);
188+
qnn_config_cls = env->GetObjectClass(qnn_config);
173189

174190
SHERPA_ONNX_JNI_READ_STRING(
175191
ans.model_config.sense_voice.qnn_config.backend_lib, backendLib,

0 commit comments

Comments
 (0)