Skip to content

Commit cdbf0a8

Browse files
csukuangfjcsukuangfj
authored
Add C++ runtime for Paraformer ASR models with Qualcomm NPU using QNN (#2931)
This pull request significantly expands the capabilities of the sherpa-onnx project by integrating C++ runtime support for Paraformer ASR models on Qualcomm NPUs. This allows for accelerated and more power-efficient speech recognition on devices equipped with Qualcomm hardware, enhancing the project's reach and performance on mobile and embedded platforms.
1 parent c08541c commit cdbf0a8

8 files changed

Lines changed: 821 additions & 8 deletions

‎sherpa-onnx/csrc/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -235,6 +235,7 @@ endif()
235235
if(SHERPA_ONNX_ENABLE_QNN)
236236
list(APPEND sources
237237
./qnn/offline-sense-voice-model-qnn.cc
238+
./qnn/offline-paraformer-model-qnn.cc
238239
./qnn/offline-zipformer-ctc-model-qnn.cc
239240
./qnn/qnn-backend.cc
240241
./qnn/qnn-model.cc

‎sherpa-onnx/csrc/offline-paraformer-model-config.cc‎

Lines changed: 92 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,11 @@ void OfflineParaformerModelConfig::Register(ParseOptions *po) {
2121
"/path/to/encoder.om,/path/to/predictor.om,/path/to/decoder.om"
2222
"If you use RK NPU, it is "
2323
"/path/to/encoder.rknn,/path/to/predictor.rknn,/path/to/decoder.rknn");
24+
25+
std::string prefix = "paraformer";
26+
ParseOptions p(prefix, po);
27+
28+
qnn_config.Register(&p);
2429
}
2530

2631
bool OfflineParaformerModelConfig::Validate() const {
@@ -46,6 +51,13 @@ bool OfflineParaformerModelConfig::Validate() const {
4651
return false;
4752
}
4853

54+
for (const auto &name : filenames) {
55+
if (!FileExists(name)) {
56+
SHERPA_ONNX_LOGE("Paraformer model '%s' does not exist", name.c_str());
57+
return false;
58+
}
59+
}
60+
4961
return true;
5062
}
5163

@@ -63,19 +75,96 @@ bool OfflineParaformerModelConfig::Validate() const {
6375
return false;
6476
}
6577

78+
for (const auto &name : filenames) {
79+
if (!FileExists(name)) {
80+
SHERPA_ONNX_LOGE("Paraformer model '%s' does not exist", name.c_str());
81+
return false;
82+
}
83+
}
84+
6685
return true;
6786
}
6887

69-
SHERPA_ONNX_LOGE("Please pass *.onnx, *.om, or *.rknn models. Given '%s'",
70-
model.c_str());
88+
if (EndsWith(model, ".so")) {
89+
std::vector<std::string> filenames;
90+
SplitStringToVector(model, ",", false, &filenames);
91+
if (filenames.size() != 3 || !EndsWith(filenames[0], "encoder.so") ||
92+
!EndsWith(filenames[1], "predictor.so") ||
93+
!EndsWith(filenames[2], "decoder.so")) {
94+
SHERPA_ONNX_LOGE(
95+
"For QNN, you should pass "
96+
"/path/libencoder.so,/path/libpredictor.so,/path/libdecoder.so. "
97+
"Given '%s'",
98+
model.c_str());
99+
return false;
100+
}
101+
102+
for (const auto &name : filenames) {
103+
if (!FileExists(name)) {
104+
SHERPA_ONNX_LOGE("Paraformer model '%s' does not exist", name.c_str());
105+
return false;
106+
}
107+
}
108+
109+
if (!qnn_config.Validate()) {
110+
return false;
111+
}
112+
113+
return true;
114+
}
115+
116+
if (model.empty() && !qnn_config.context_binary.empty()) {
117+
// we require that the context_binary exists
118+
if (!FileExists(qnn_config.context_binary)) {
119+
SHERPA_ONNX_LOGE(
120+
"Model is empty, but you provide a context binary that does not "
121+
"exist");
122+
return false;
123+
}
124+
125+
std::vector<std::string> filenames;
126+
SplitStringToVector(model, ",", false, &filenames);
127+
if (filenames.size() != 3) {
128+
SHERPA_ONNX_LOGE(
129+
"For Paraformer with QNN, you should pass "
130+
"/path/encoder.bin,/path/predictor.bin,/path/decoder.bin"
131+
"Given '%s'",
132+
model.c_str());
133+
return false;
134+
}
135+
136+
for (const auto &name : filenames) {
137+
if (!FileExists(name)) {
138+
SHERPA_ONNX_LOGE("Paraformer context binary '%s' does not exist",
139+
name.c_str());
140+
return false;
141+
}
142+
}
143+
144+
if (!qnn_config.Validate()) {
145+
return false;
146+
}
147+
148+
return true;
149+
}
150+
151+
SHERPA_ONNX_LOGE(
152+
"Please pass *.onnx, *.om, *.rknn, or *.so models. Given '%s'",
153+
model.c_str());
71154
return false;
72155
}
73156

74157
std::string OfflineParaformerModelConfig::ToString() const {
75158
std::ostringstream os;
76159

77160
os << "OfflineParaformerModelConfig(";
78-
os << "model=\"" << model << "\")";
161+
os << "model=\"" << model << "\"";
162+
163+
if (!qnn_config.backend_lib.empty()) {
164+
os << ", qnn_config=" << qnn_config.ToString();
165+
}
166+
167+
os << ")";
79168

80169
return os.str();
81170
}

‎sherpa-onnx/csrc/offline-paraformer-model-config.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
#include <string>
88

99
#include "sherpa-onnx/csrc/parse-options.h"
10+
#include "sherpa-onnx/csrc/qnn-config.h"
1011

1112
namespace sherpa_onnx {
1213

@@ -17,8 +18,14 @@ struct OfflineParaformerModelConfig {
1718
// for rknn,
1819
// model is
1920
// "/path/to/encoder.rknn,/path/to/predictor.rknn,/path/to/decoder.rknn"
21+
//
22+
// for qnn with shared libs, model is
23+
// model is
24+
// "/path/to/libencoder.so,/path/to/libpredictor.so,/path/to/libdecoder.so"
2025
std::string model;
2126

27+
QnnConfig qnn_config;
28+
2229
OfflineParaformerModelConfig() = default;
2330
explicit OfflineParaformerModelConfig(const std::string &model)
2431
: model(model) {}

‎sherpa-onnx/csrc/offline-recognizer-impl.cc‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
#endif
5959

6060
#if SHERPA_ONNX_ENABLE_QNN
61+
#include "sherpa-onnx/csrc/qnn/offline-paraformer-model-qnn.h"
6162
#include "sherpa-onnx/csrc/qnn/offline-recognizer-zipformer-ctc-qnn-impl.h"
6263
#include "sherpa-onnx/csrc/qnn/offline-sense-voice-model-qnn.h"
6364
#endif
@@ -180,10 +181,16 @@ std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create(
180181
!config.model_config.zipformer_ctc.qnn_config.context_binary
181182
.empty()) {
182183
return std::make_unique<OfflineRecognizerZipformerCtcQnnImpl>(config);
184+
} else if (!config.model_config.paraformer.model.empty() ||
185+
!config.model_config.paraformer.qnn_config.context_binary
186+
.empty()) {
187+
return std::make_unique<
188+
OfflineRecognizerParaformerTplImpl<OfflineParaformerModelQnn>>(
189+
config);
183190
} else {
184191
SHERPA_ONNX_LOGE(
185-
"Only SenseVoice models and Zipformer CTC models are currently "
186-
"supported by qnn for non-streaming ASR.");
192+
"Only SenseVoice, Paraformer, and Zipformer CTC models are currently "
193+
"supported by QNN for non-streaming ASR.");
187194
SHERPA_ONNX_EXIT(-1);
188195
return nullptr;
189196
}
@@ -504,10 +511,16 @@ std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create(
504511
.empty()) {
505512
return std::make_unique<OfflineRecognizerZipformerCtcQnnImpl>(mgr,
506513
config);
514+
} else if (!config.model_config.paraformer.model.empty() ||
515+
!config.model_config.paraformer.qnn_config.context_binary
516+
.empty()) {
517+
return std::make_unique<
518+
OfflineRecognizerParaformerTplImpl<OfflineParaformerModelQnn>>(
519+
mgr, config);
507520
} else {
508521
SHERPA_ONNX_LOGE(
509-
"Only SenseVoice models and Zipformer CTC models are currently "
510-
"supported by qnn for non-streaming ASR.");
522+
"Only SenseVoice, Paraformer, and Zipformer CTC models are currently "
523+
"supported by QNN for non-streaming ASR.");
511524
SHERPA_ONNX_EXIT(-1);
512525
return nullptr;
513526
}

‎sherpa-onnx/csrc/offline-stream.cc‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <utility>
1515
#include <vector>
1616

17+
#include "Eigen/Core"
1718
#include "kaldi-native-fbank/csrc/online-feature.h"
1819
#include "sherpa-onnx/csrc/macros.h"
1920
#include "sherpa-onnx/csrc/math.h"

0 commit comments

Comments
 (0)