Enhance Paraformer support and add streaming example

- Updated `.gitignore` to include new Paraformer models.
- Added `serde` and `serde_json` dependencies in `Cargo.toml` and `Cargo.lock`.
- Introduced `OnlineRecognizerJsonResult` and `OnlineRecognizerResult` structs for handling recognition results.
- Implemented `ParaformerOnlineRecognizer` with configuration options and transcription capabilities.
- Added a new example for streaming transcription using Paraformer.

Author: BenLocal

.gitignore

@@ -15,6 +15,9 @@ sherpa-onnx-whisper*
 sherpa-onnx-pyannote-*
 sherpa-onnx-zipformer-*
 sherpa-onnx-moonshine-*
+sherpa-onnx-paraformer-*
+sherpa-onnx-streaming-paraformer-*
+sherpa-onnx-sense-voice-*
 *.txt
 !checksum.txt
 sherpa-onnx-v*

Cargo.lock

@@ -23,6 +23,8 @@ eyre = "0.6.12"
 hound = { version = "3.5.1" }
 sherpa-rs-sys = { path = "../sherpa-rs-sys", version = "0.6.8", default-features = false }
 tracing = "0.1.40"
+serde = { version = "1.0.216", features = ["derive"] }
+serde_json = "1.0.134"
 
 [dev-dependencies]
 clap = { version = "4.5.8", features = ["derive"] }
@@ -139,3 +141,7 @@ path = "../../examples/dolphin.rs"
 [[example]]
 name = "parakeet"
 path = "../../examples/parakeet.rs"
+
+[[example]]
+name = "paraformer_streaming"
+path = "../../examples/paraformer_streaming.rs"

crates/sherpa-rs/Cargo.toml

@@ -22,6 +22,7 @@ pub mod tts;
 
 use std::ffi::CStr;
 
+use serde::{Deserialize, Serialize};
 #[cfg(feature = "sys")]
 pub use sherpa_rs_sys;
 
@@ -135,3 +136,76 @@ impl Default for OnnxConfig {
         }
     }
 }
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct OnlineRecognizerJsonResult {
+    pub text: String,
+    pub tokens: Vec<String>,
+    pub timestamps: Vec<f32>,
+    pub ys_probs: Vec<f32>,
+    pub lm_probs: Vec<f32>,
+    pub context_scores: Vec<f32>,
+    pub segment: i32,
+    pub words: Vec<Word>,
+    pub start_time: f32,
+    pub is_final: bool,
+    pub is_eof: bool,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize, Default)]
+pub struct Word {
+    pub word: String,
+    pub start: f32,
+    pub end: f32,
+}
+
+#[derive(Debug, Clone)]
+pub struct OnlineRecognizerResult {
+    pub text: String,
+    pub timestamps: Vec<f32>,
+    pub tokens: Vec<String>,
+    /// Whether the result is final
+    pub is_final: bool,
+    /// segment id
+    pub segment: i32,
+    /// start_time of the segment in seconds
+    pub start_time: f32,
+}
+
+impl OnlineRecognizerResult {
+    fn new(result: &sherpa_rs_sys::SherpaOnnxOnlineRecognizerResult) -> Self {
+        let text = unsafe { cstr_to_string(result.text) };
+        let count = result.count.try_into().unwrap();
+        let timestamps = if result.timestamps.is_null() {
+            Vec::new()
+        } else {
+            unsafe { std::slice::from_raw_parts(result.timestamps, count).to_vec() }
+        };
+        let mut tokens = Vec::with_capacity(count);
+        let mut next_token = result.tokens;
+        let json_str = unsafe { cstr_to_string(result.json) };
+        let json: OnlineRecognizerJsonResult = serde_json::from_str(&json_str).unwrap_or_default();
+
+        for _ in 0..count {
+            let token = unsafe { CStr::from_ptr(next_token) };
+            tokens.push(token.to_string_lossy().into_owned());
+            next_token = next_token
+                .wrapping_byte_offset(token.to_bytes_with_nul().len().try_into().unwrap());
+        }
+
+        Self {
+            text,
+            timestamps,
+            tokens,
+            is_final: json.is_final,
+            segment: json.segment,
+            start_time: json.start_time,
+        }
+    }
+}
+
+impl From<&sherpa_rs_sys::SherpaOnnxOnlineRecognizerResult> for OnlineRecognizerResult {
+    fn from(value: &sherpa_rs_sys::SherpaOnnxOnlineRecognizerResult) -> Self {
+        Self::new(value)
+    }
+}

crates/sherpa-rs/src/lib.rs

@@ -8,6 +8,7 @@ pub struct ParaformerRecognizer {
 }
 
 pub type ParaformerRecognizerResult = super::OfflineRecognizerResult;
+pub type ParaformerOnlineRecognizerResult = super::OnlineRecognizerResult;
 
 #[derive(Debug, Clone)]
 pub struct ParaformerConfig {
@@ -137,3 +138,154 @@ impl Drop for ParaformerRecognizer {
         }
     }
 }
+
+#[derive(Debug, Clone)]
+pub struct ParaformerOnlineConfig {
+    pub encoder_model_path: String,
+    pub decoder_model_path: String,
+    pub tokens: String,
+    pub provider: Option<String>,
+    pub num_threads: Option<i32>,
+    pub debug: bool,
+    pub enable_endpoint: Option<bool>,
+    pub rule1_min_trailing_silence: Option<f32>,
+    pub rule2_min_trailing_silence: Option<f32>,
+    pub rule3_min_utterance_length: Option<f32>,
+}
+
+impl Default for ParaformerOnlineConfig {
+    fn default() -> Self {
+        Self {
+            encoder_model_path: String::new(),
+            decoder_model_path: String::new(),
+            tokens: String::new(),
+            provider: None,
+            num_threads: None,
+            debug: false,
+            enable_endpoint: None,
+            rule1_min_trailing_silence: None,
+            rule2_min_trailing_silence: None,
+            rule3_min_utterance_length: None,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub struct ParaformerOnlineRecognizer {
+    recognizer: *const sherpa_rs_sys::SherpaOnnxOnlineRecognizer,
+    stream: *const sherpa_rs_sys::SherpaOnnxOnlineStream,
+    segment_id: i32,
+}
+
+impl ParaformerOnlineRecognizer {
+    pub fn new(config: ParaformerOnlineConfig) -> Result<Self> {
+        let debug = config.debug.into();
+        let provider = config.provider.unwrap_or(get_default_provider());
+        let provider_ptr = cstring_from_str(&provider);
+        let tokens_ptr = cstring_from_str(&config.tokens);
+
+        let encoder_model_path = if config.encoder_model_path.is_empty() {
+            bail!("Encoder model path is required for online Paraformer")
+        } else {
+            cstring_from_str(&config.encoder_model_path)
+        };
+        let decoder_model_path = if config.decoder_model_path.is_empty() {
+            bail!("Decoder model path is required for online Paraformer")
+        } else {
+            cstring_from_str(&config.decoder_model_path)
+        };
+        let paraformer_config = sherpa_rs_sys::SherpaOnnxOnlineParaformerModelConfig {
+            encoder: encoder_model_path.as_ptr(),
+            decoder: decoder_model_path.as_ptr(),
+        };
+        let empty_str = cstring_from_str("");
+        let mut model_config = sherpa_rs_sys::SherpaOnnxOnlineModelConfig::default();
+        model_config.debug = debug;
+        model_config.num_threads = config.num_threads.unwrap_or(1);
+        model_config.provider = provider_ptr.as_ptr();
+        model_config.tokens = tokens_ptr.as_ptr();
+        model_config.paraformer = paraformer_config;
+
+        // Recognizer config
+        let mut recognizer_config = sherpa_rs_sys::SherpaOnnxOnlineRecognizerConfig::default();
+        recognizer_config.feat_config = sherpa_rs_sys::SherpaOnnxFeatureConfig {
+            sample_rate: 16000,
+            feature_dim: 80,
+        };
+        recognizer_config.model_config = model_config;
+        recognizer_config.rule_fsts = empty_str.as_ptr();
+        recognizer_config.rule_fars = empty_str.as_ptr();
+
+        recognizer_config.enable_endpoint = config.enable_endpoint.unwrap_or(false).into();
+        recognizer_config.rule1_min_trailing_silence =
+            config.rule1_min_trailing_silence.unwrap_or(2.4);
+        recognizer_config.rule2_min_trailing_silence =
+            config.rule2_min_trailing_silence.unwrap_or(1.2);
+        recognizer_config.rule3_min_utterance_length =
+            config.rule3_min_utterance_length.unwrap_or(300.0);
+
+        let recognizer =
+            unsafe { sherpa_rs_sys::SherpaOnnxCreateOnlineRecognizer(&recognizer_config) };
+        if recognizer.is_null() {
+            bail!("Failed to create online Paraformer recognizer");
+        }
+        let stream = unsafe { sherpa_rs_sys::SherpaOnnxCreateOnlineStream(recognizer) };
+        if stream.is_null() {
+            unsafe {
+                sherpa_rs_sys::SherpaOnnxDestroyOnlineRecognizer(recognizer);
+            }
+            bail!("Failed to create online Paraformer stream");
+        }
+        Ok(Self {
+            recognizer,
+            stream,
+            segment_id: 0,
+        })
+    }
+
+    pub fn transcribe(
+        &mut self,
+        sample_rate: u32,
+        samples: &[f32],
+    ) -> ParaformerOnlineRecognizerResult {
+        unsafe {
+            sherpa_rs_sys::SherpaOnnxOnlineStreamAcceptWaveform(
+                self.stream,
+                sample_rate as i32,
+                samples.as_ptr(),
+                samples.len() as i32,
+            );
+
+            while sherpa_rs_sys::SherpaOnnxIsOnlineStreamReady(self.recognizer, self.stream) == 1 {
+                sherpa_rs_sys::SherpaOnnxDecodeOnlineStream(self.recognizer, self.stream);
+            }
+
+            let result_ptr =
+                sherpa_rs_sys::SherpaOnnxGetOnlineStreamResult(self.recognizer, self.stream);
+            let raw_result = result_ptr.read();
+            let mut result = ParaformerOnlineRecognizerResult::from(&raw_result);
+            sherpa_rs_sys::SherpaOnnxDestroyOnlineRecognizerResult(result_ptr);
+
+            if sherpa_rs_sys::SherpaOnnxOnlineStreamIsEndpoint(self.recognizer, self.stream) == 1 {
+                self.segment_id += 1;
+                sherpa_rs_sys::SherpaOnnxOnlineStreamReset(self.recognizer, self.stream);
+                result.is_final = true;
+            }
+
+            result.segment = self.segment_id;
+            result
+        }
+    }
+}
+
+unsafe impl Send for ParaformerOnlineRecognizer {}
+unsafe impl Sync for ParaformerOnlineRecognizer {}
+
+impl Drop for ParaformerOnlineRecognizer {
+    fn drop(&mut self) {
+        unsafe {
+            sherpa_rs_sys::SherpaOnnxDestroyOnlineStream(self.stream);
+            sherpa_rs_sys::SherpaOnnxDestroyOnlineRecognizer(self.recognizer);
+        }
+    }
+}

crates/sherpa-rs/src/paraformer.rs

@@ -0,0 +1,49 @@
+/*
+Transcribe wav file using streaming Paraformer and punctuate the result
+
+wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2
+tar xvf sherpa-onnx-streaming-paraformer-bilingual-zh-en.tar.bz2
+
+wget https://github.com/k2-fsa/sherpa-onnx/releases/download/punctuation-models/sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
+tar xvf sherpa-onnx-punct-ct-transformer-zh-en-vocab272727-2024-04-12.tar.bz2
+
+cargo run --example paraformer_streaming motivation.wav
+*/
+
+use sherpa_rs::{
+    paraformer::{ParaformerOnlineConfig, ParaformerOnlineRecognizer},
+    read_audio_file,
+};
+
+fn main() {
+    let path = std::env::args().nth(1).expect("Missing file path argument");
+    let provider = std::env::args().nth(2).unwrap_or("cpu".into());
+    let (samples, sample_rate) = read_audio_file(&path).unwrap();
+    assert_eq!(sample_rate, 16000, "The sample rate must be 16000.");
+
+    let config = ParaformerOnlineConfig {
+        tokens: "sherpa-onnx-streaming-paraformer-bilingual-zh-en/tokens.txt".into(),
+        provider: Some(provider),
+        debug: true,
+        encoder_model_path: "sherpa-onnx-streaming-paraformer-bilingual-zh-en/encoder.int8.onnx"
+            .into(),
+        decoder_model_path: "sherpa-onnx-streaming-paraformer-bilingual-zh-en/decoder.int8.onnx"
+            .into(),
+        enable_endpoint: Some(true),
+        ..Default::default()
+    };
+
+    let mut recognizer = ParaformerOnlineRecognizer::new(config).unwrap();
+
+    for chunk in samples.chunks(1600) {
+        let result = recognizer.transcribe(sample_rate, &chunk);
+        if result.text.is_empty() {
+            continue;
+        }
+        if result.is_final {
+            println!("🎉 Final: {}", result.text);
+        } else {
+            println!("💬 Partial: {}", result.text);
+        }
+    }
+}