Skip to content

Commit 20ac56e

Browse files
csukuangfjcsukuangfj
authored
Add C++ runtime for Whisper with Qualcomm NPU using QNN. (#3699)
1 parent 8c0586e commit 20ac56e

7 files changed

Lines changed: 900 additions & 14 deletions

‎.github/workflows/export-whisper-qnn.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ name: export-whisper-qnn
33
on:
44
push:
55
branches:
6-
- export-whisper-qnn
6+
- cpp-qnn-whisper-2
77
workflow_dispatch:
88

99
concurrency:
@@ -410,7 +410,7 @@ jobs:
410410
411411
echo "collect results"
412412
413-
d=sherpa-onnx-qnn-${{ matrix.soc }}-binary-${{ matrix.model_name }}
413+
d=sherpa-onnx-qnn-${{ matrix.soc }}-binary-whisper-${{ matrix.model_name }}
414414
mkdir -p $d
415415
mkdir -p $d/test_wavs
416416
cp -v non_quant/binary/*.bin $d/
@@ -427,9 +427,9 @@ jobs:
427427
428428
for p in x86_64-linux-clang aarch64-android; do
429429
if [[ $p == x86_64-linux-clang ]]; then
430-
d=sherpa-onnx-qnn-${{ matrix.model_name }}-linux-x64
430+
d=sherpa-onnx-qnn-whisper-${{ matrix.model_name }}-linux-x64
431431
elif [[ $p == aarch64-android ]]; then
432-
d=sherpa-onnx-qnn-${{ matrix.model_name }}-android-aarch64
432+
d=sherpa-onnx-qnn-whisper-${{ matrix.model_name }}-android-aarch64
433433
else
434434
echo "Unknown $p"
435435
exit -1

‎sherpa-onnx/csrc/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,7 @@ if(SHERPA_ONNX_ENABLE_QNN)
259259
./qnn/offline-parakeet-ctc-model-qnn.cc
260260
./qnn/offline-sense-voice-model-qnn.cc
261261
./qnn/offline-paraformer-model-qnn.cc
262+
./qnn/offline-whisper-model-qnn.cc
262263
./qnn/offline-zipformer-transducer-model-qnn.cc
263264
./qnn/offline-zipformer-ctc-model-qnn.cc
264265
./qnn/online-zipformer-transducer-model-qnn.cc

‎sherpa-onnx/csrc/offline-recognizer-impl.cc‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@
7171
#include "sherpa-onnx/csrc/qnn/offline-recognizer-transducer-qnn-impl.h"
7272
#include "sherpa-onnx/csrc/qnn/offline-recognizer-zipformer-ctc-qnn-impl.h"
7373
#include "sherpa-onnx/csrc/qnn/offline-sense-voice-model-qnn.h"
74+
#include "sherpa-onnx/csrc/qnn/offline-whisper-model-qnn.h"
7475
#endif
7576

7677
namespace sherpa_onnx {
@@ -206,11 +207,16 @@ std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create(
206207
!config.model_config.nemo_ctc.qnn_config.context_binary
207208
.empty()) {
208209
return std::make_unique<OfflineRecognizerParakeetCtcQnnImpl>(config);
210+
} else if (!config.model_config.whisper.encoder.empty() ||
211+
!config.model_config.whisper.qnn_config.context_binary
212+
.empty()) {
213+
return std::make_unique<
214+
OfflineRecognizerWhisperTplImpl<OfflineWhisperModelQnn>>(config);
209215
} else {
210216
SHERPA_ONNX_LOGE(
211217
"Only SenseVoice, Paraformer, offline transducer, Zipformer CTC, "
212-
"and NeMo CTC (Parakeet) models are currently supported by QNN "
213-
"for non-streaming ASR.");
218+
"NeMo CTC (Parakeet), and Whisper models are currently supported by "
219+
"QNN for non-streaming ASR.");
214220
SHERPA_ONNX_EXIT(-1);
215221
return nullptr;
216222
}
@@ -567,11 +573,16 @@ std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create(
567573
!config.model_config.nemo_ctc.qnn_config.context_binary
568574
.empty()) {
569575
return std::make_unique<OfflineRecognizerParakeetCtcQnnImpl>(mgr, config);
576+
} else if (!config.model_config.whisper.encoder.empty() ||
577+
!config.model_config.whisper.qnn_config.context_binary
578+
.empty()) {
579+
return std::make_unique<
580+
OfflineRecognizerWhisperTplImpl<OfflineWhisperModelQnn>>(mgr, config);
570581
} else {
571582
SHERPA_ONNX_LOGE(
572583
"Only SenseVoice, Paraformer, offline transducer, Zipformer CTC, "
573-
"and NeMo CTC (Parakeet) models are currently supported by QNN "
574-
"for non-streaming ASR.");
584+
"NeMo CTC (Parakeet), and Whisper models are currently supported by "
585+
"QNN for non-streaming ASR.");
575586
SHERPA_ONNX_EXIT(-1);
576587
return nullptr;
577588
}

‎sherpa-onnx/csrc/offline-whisper-model-config.cc‎

Lines changed: 93 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,40 @@
1010

1111
#include "sherpa-onnx/csrc/file-utils.h"
1212
#include "sherpa-onnx/csrc/macros.h"
13+
#include "sherpa-onnx/csrc/text-utils.h"
1314

1415
namespace sherpa_onnx {
1516

17+
static bool IsQnnModelLibFile(const std::string &filename) {
18+
return EndsWith(filename, ".so");
19+
}
20+
21+
static bool IsQnnWhisperArtifact(const OfflineWhisperModelConfig &config) {
22+
return IsQnnModelLibFile(config.encoder) ||
23+
IsQnnModelLibFile(config.decoder) ||
24+
!config.qnn_config.context_binary.empty();
25+
}
26+
27+
static bool ValidateQnnContextBinaries(const std::string &context_binary,
28+
std::vector<std::string> &filenames) {
29+
filenames.clear();
30+
31+
if (context_binary.empty()) {
32+
return true;
33+
}
34+
35+
SplitStringToVector(context_binary, ",", true, &filenames);
36+
if (filenames.size() != 2) {
37+
SHERPA_ONNX_LOGE(
38+
"For offline whisper with QNN, you should provide 2 context "
39+
"binaries separated by commas (encoder,decoder). Given '%s'",
40+
context_binary.c_str());
41+
return false;
42+
}
43+
44+
return true;
45+
}
46+
1647
void OfflineWhisperModelConfig::Register(ParseOptions *po) {
1748
po->Register("whisper-encoder", &encoder,
1849
"Path to onnx encoder of whisper, e.g., tiny-encoder.onnx, "
@@ -59,17 +90,71 @@ void OfflineWhisperModelConfig::Register(ParseOptions *po) {
5990
"--whisper-enable-token-timestamps for both segment-level and "
6091
"token-level "
6192
"timestamps. Default: false.");
93+
94+
std::string prefix = "whisper";
95+
ParseOptions p(prefix, po);
96+
qnn_config.Register(&p);
6297
}
6398

6499
bool OfflineWhisperModelConfig::Validate() const {
100+
bool uses_qnn = IsQnnWhisperArtifact(*this);
101+
102+
if (uses_qnn) {
103+
std::vector<std::string> context_binaries;
104+
if (!ValidateQnnContextBinaries(qnn_config.context_binary,
105+
context_binaries)) {
106+
return false;
107+
}
108+
109+
bool need_model_libs = context_binaries.empty();
110+
for (const auto &name : context_binaries) {
111+
if (!FileExists(name)) {
112+
need_model_libs = true;
113+
break;
114+
}
115+
}
116+
117+
if (need_model_libs) {
118+
if (!EndsWith(encoder, ".so") || !EndsWith(decoder, ".so")) {
119+
SHERPA_ONNX_LOGE(
120+
"For offline whisper with QNN, encoder/decoder should be "
121+
"*.so when context binaries are missing. Given encoder: '%s', "
122+
"decoder: '%s'",
123+
encoder.c_str(), decoder.c_str());
124+
return false;
125+
}
126+
127+
if (!FileExists(encoder)) {
128+
SHERPA_ONNX_LOGE("whisper encoder: '%s' does not exist",
129+
encoder.c_str());
130+
return false;
131+
}
132+
133+
if (!FileExists(decoder)) {
134+
SHERPA_ONNX_LOGE("whisper decoder: '%s' does not exist",
135+
decoder.c_str());
136+
return false;
137+
}
138+
}
139+
140+
for (const auto &name : context_binaries) {
141+
if (FileExists(name) && !EndsWith(name, ".bin")) {
142+
SHERPA_ONNX_LOGE("QNN context binary should end with .bin. Given '%s'",
143+
name.c_str());
144+
return false;
145+
}
146+
}
147+
148+
return qnn_config.Validate();
149+
}
150+
65151
if (encoder.empty()) {
66152
SHERPA_ONNX_LOGE("Please provide --whisper-encoder");
67153
return false;
68154
}
69155

70156
if (!FileExists(encoder)) {
71-
SHERPA_ONNX_LOGE("whisper encoder file '%s' does not exist",
72-
encoder.c_str());
157+
SHERPA_ONNX_LOGE("whisper encoder: '%s' does not exist", encoder.c_str());
73158
return false;
74159
}
75160

@@ -79,16 +164,14 @@ bool OfflineWhisperModelConfig::Validate() const {
79164
}
80165

81166
if (!FileExists(decoder)) {
82-
SHERPA_ONNX_LOGE("whisper decoder file '%s' does not exist",
83-
decoder.c_str());
167+
SHERPA_ONNX_LOGE("whisper decoder: '%s' does not exist", decoder.c_str());
84168
return false;
85169
}
86170

87171
if (task != "translate" && task != "transcribe") {
88172
SHERPA_ONNX_LOGE(
89173
"--whisper-task supports only translate and transcribe. Given: %s",
90174
task.c_str());
91-
92175
return false;
93176
}
94177

@@ -107,7 +190,11 @@ std::string OfflineWhisperModelConfig::ToString() const {
107190
os << "enable_token_timestamps="
108191
<< (enable_token_timestamps ? "True" : "False") << ", ";
109192
os << "enable_segment_timestamps="
110-
<< (enable_segment_timestamps ? "True" : "False") << ")";
193+
<< (enable_segment_timestamps ? "True" : "False") << ", ";
194+
if (!qnn_config.backend_lib.empty()) {
195+
os << "qnn_config=" << qnn_config.ToString() << ", ";
196+
}
197+
os << ")";
111198

112199
return os.str();
113200
}

‎sherpa-onnx/csrc/offline-whisper-model-config.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,13 +8,16 @@
88
#include <vector>
99

1010
#include "sherpa-onnx/csrc/parse-options.h"
11+
#include "sherpa-onnx/csrc/qnn-config.h"
1112

1213
namespace sherpa_onnx {
1314

1415
struct OfflineWhisperModelConfig {
1516
std::string encoder;
1617
std::string decoder;
1718

19+
QnnConfig qnn_config;
20+
1821
// Available languages can be found at
1922
// https://github.com/openai/whisper/blob/main/whisper/tokenizer.py#L10
2023
//

0 commit comments

Comments
 (0)