Add C++ runtime for Parakeet CTC with QNN. (k2-fsa#3688)

Author: csukuangfj

sherpa-onnx/csrc/CMakeLists.txt

@@ -256,6 +256,7 @@ endif()
 
 if(SHERPA_ONNX_ENABLE_QNN)
   list(APPEND sources
+    ./qnn/offline-parakeet-ctc-model-qnn.cc
     ./qnn/offline-sense-voice-model-qnn.cc
     ./qnn/offline-paraformer-model-qnn.cc
     ./qnn/offline-zipformer-transducer-model-qnn.cc

sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model-config.cc

@@ -8,18 +8,48 @@
 
 #include "sherpa-onnx/csrc/file-utils.h"
 #include "sherpa-onnx/csrc/macros.h"
+#include "sherpa-onnx/csrc/text-utils.h"
 
 namespace sherpa_onnx {
 
 void OfflineNemoEncDecCtcModelConfig::Register(ParseOptions *po) {
-  po->Register("nemo-ctc-model", &model,
-               "Path to model.onnx of Nemo EncDecCtcModel.");
+  po->Register(
+      "nemo-ctc-model", &model,
+      "Path to NeMo CTC model. For qnn, use compiled model library "
+      "(e.g. libmodel.so). For onnxruntime, use model.onnx.");
+
+  std::string prefix = "nemo-ctc";
+  ParseOptions p(prefix, po);
+
+  qnn_config.Register(&p);
 }
 
 bool OfflineNemoEncDecCtcModelConfig::Validate() const {
-  if (!FileExists(model)) {
-    SHERPA_ONNX_LOGE("NeMo model: '%s' does not exist", model.c_str());
-    return false;
+  if (qnn_config.context_binary.empty()) {
+    if (model.empty()) {
+      SHERPA_ONNX_LOGE("Please provide a NeMo CTC model");
+      return false;
+    }
+
+    if (!FileExists(model)) {
+      SHERPA_ONNX_LOGE("NeMo model: '%s' does not exist", model.c_str());
+      return false;
+    }
+  }
+
+  if (model.empty() && !qnn_config.context_binary.empty()) {
+    // we require that the context_binary exists
+    if (!FileExists(qnn_config.context_binary)) {
+      SHERPA_ONNX_LOGE(
+          "Model is empty, but you provide a context binary that does not "
+          "exist");
+      return false;
+    }
+  }
+
+  if (EndsWith(model, ".so") || EndsWith(model, ".bin") ||
+      (model.empty() && !qnn_config.context_binary.empty())) {
+    return qnn_config.Validate();
   }
 
   return true;
@@ -29,7 +59,13 @@ std::string OfflineNemoEncDecCtcModelConfig::ToString() const {
   std::ostringstream os;
 
   os << "OfflineNemoEncDecCtcModelConfig(";
-  os << "model=\"" << model << "\")";
+  os << "model=\"" << model << "\"";
+
+  if (!qnn_config.backend_lib.empty()) {
+    os << ", qnn_config=" << qnn_config.ToString() << ", ";
+  }
+
+  os << ")";
 
   return os.str();
 }

sherpa-onnx/csrc/offline-nemo-enc-dec-ctc-model-config.h

@@ -7,11 +7,13 @@
 #include <string>
 
 #include "sherpa-onnx/csrc/parse-options.h"
+#include "sherpa-onnx/csrc/qnn-config.h"
 
 namespace sherpa_onnx {
 
 struct OfflineNemoEncDecCtcModelConfig {
   std::string model;
+  QnnConfig qnn_config;
 
   OfflineNemoEncDecCtcModelConfig() = default;
   explicit OfflineNemoEncDecCtcModelConfig(const std::string &model)

sherpa-onnx/csrc/offline-recognizer-impl.cc

@@ -67,6 +67,7 @@
 
 #if SHERPA_ONNX_ENABLE_QNN
 #include "sherpa-onnx/csrc/qnn/offline-paraformer-model-qnn.h"
+#include "sherpa-onnx/csrc/qnn/offline-recognizer-parakeet-ctc-qnn-impl.h"
 #include "sherpa-onnx/csrc/qnn/offline-recognizer-transducer-qnn-impl.h"
 #include "sherpa-onnx/csrc/qnn/offline-recognizer-zipformer-ctc-qnn-impl.h"
 #include "sherpa-onnx/csrc/qnn/offline-sense-voice-model-qnn.h"
@@ -201,10 +202,15 @@ std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create(
       return std::make_unique<
           OfflineRecognizerParaformerTplImpl<OfflineParaformerModelQnn>>(
           config);
+    } else if (!config.model_config.nemo_ctc.model.empty() ||
+               !config.model_config.nemo_ctc.qnn_config.context_binary
+                    .empty()) {
+      return std::make_unique<OfflineRecognizerParakeetCtcQnnImpl>(config);
     } else {
       SHERPA_ONNX_LOGE(
-          "Only SenseVoice, Paraformer, offline transducer, and Zipformer CTC "
-          "models are currently supported by QNN for non-streaming ASR.");
+          "Only SenseVoice, Paraformer, offline transducer, Zipformer CTC, "
+          "and NeMo CTC (Parakeet) models are currently supported by QNN "
+          "for non-streaming ASR.");
       SHERPA_ONNX_EXIT(-1);
       return nullptr;
     }
@@ -557,10 +563,15 @@ std::unique_ptr<OfflineRecognizerImpl> OfflineRecognizerImpl::Create(
       return std::make_unique<
           OfflineRecognizerParaformerTplImpl<OfflineParaformerModelQnn>>(
           mgr, config);
+    } else if (!config.model_config.nemo_ctc.model.empty() ||
+               !config.model_config.nemo_ctc.qnn_config.context_binary
+                    .empty()) {
+      return std::make_unique<OfflineRecognizerParakeetCtcQnnImpl>(mgr, config);
     } else {
       SHERPA_ONNX_LOGE(
-          "Only SenseVoice, Paraformer, offline transducer, and Zipformer CTC "
-          "models are currently supported by QNN for non-streaming ASR.");
+          "Only SenseVoice, Paraformer, offline transducer, Zipformer CTC, "
+          "and NeMo CTC (Parakeet) models are currently supported by QNN "
+          "for non-streaming ASR.");
       SHERPA_ONNX_EXIT(-1);
       return nullptr;
     }