fix: handle short ONNX audio and update sherpa-onnx

Author: Mr-Sunglasses

AGENTS.md

@@ -190,7 +190,7 @@ Version-bump changelog tables go in the **PR description**, not a tracked file.
 |------------|---------|-----|
 | [WhisperKit](https://github.com/argmaxinc/WhisperKit) | Whisper CoreML | `from: "0.9.4"` |
 | [FluidAudio](https://github.com/FluidInference/FluidAudio) | Parakeet CoreML / ANE | `.upToNextMinor(from: "0.15.5")` (pre-1.0) |
-| [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) | Specialized ONNX, CPU | revision pin (SPM not in a tagged release; xcframework v1.13.4) |
+| [sherpa-onnx](https://github.com/k2-fsa/sherpa-onnx) | Specialized ONNX, CPU | exact `1.13.7` (matching xcframework) |
 
 Keep dependencies minimal. Do not bump FluidAudio across a minor without checking `AsrManager.loadModels` / TDT decoder APIs.
 

Package.swift

@@ -27,11 +27,10 @@ let package = Package(
         .package(url: "https://github.com/FluidInference/FluidAudio.git", .upToNextMinor(from: "0.15.5")),
         // sherpa-onnx — specialized ONNX models (Moonshine, SenseVoice,
         // GigaAM, Canary) via ONNX Runtime, CPU-only.
-        // Pinned to a revision: the SPM manifest is not in a tagged release
-        // yet; the pinned manifest references the v1.13.4 binary xcframework.
+        // Pin the release and its matching binary xcframework for reproducible builds.
         .package(
             url: "https://github.com/k2-fsa/sherpa-onnx",
-            revision: "00ad9a19a63751a6c4b12050a00eacfeb204814e"
+            exact: "1.13.7"
         ),
     ],
     targets: [
@@ -62,7 +61,8 @@ let package = Package(
         .testTarget(
             name: "VocaMacTests",
             dependencies: ["VocaMac"],
-            path: "Tests/VocaMacTests"
+            path: "Tests/VocaMacTests",
+            exclude: ["Fixtures"]
         )
     ]
 )

Sources/VocaMac/Services/SherpaAudioPreparation.swift

@@ -0,0 +1,25 @@
+// SherpaAudioPreparation.swift
+// VocaMac
+
+/// Conditions individual 16 kHz segments before they reach native ONNX code.
+/// Very short waveforms can have too few frames for feature extraction or
+/// encoder subsampling. Padding preserves the speech and gives the decoder
+/// trailing context without changing the recording's reported duration.
+enum SherpaAudioPreparation {
+    static let minimumSampleCount = 16_000
+
+    /// Reject malformed input before segmentation or any native inference.
+    static func validate(_ samples: [Float]) throws {
+        guard !samples.isEmpty else { throw SherpaError.emptyAudio }
+        guard samples.allSatisfy({ $0.isFinite }) else {
+            throw SherpaError.transcriptionFailed(reason: "Audio contains non-finite samples.")
+        }
+    }
+
+    static func prepare(_ samples: [Float]) -> [Float] {
+        // Do not ask generative decoders to invent words for digital silence.
+        guard samples.contains(where: { $0 != 0 }) else { return [] }
+        guard samples.count < minimumSampleCount else { return samples }
+        return samples + [Float](repeating: 0, count: minimumSampleCount - samples.count)
+    }
+}

Sources/VocaMac/Services/SherpaService.swift

@@ -105,18 +105,36 @@ final class SherpaService: @unchecked Sendable {
 
     // MARK: - Properties
 
-    /// The active C recognizer (created when a model is loaded)
-    private var recognizer: OpaquePointer?
+    /// A request retains its native model even if Settings unloads or replaces it.
+    /// The per-model lock serializes native decodes without blocking state reads.
+    private final class LoadedRecognizer: @unchecked Sendable {
+        let pointer: OpaquePointer
+        let size: ModelSize
+        let decodeLock = NSLock()
+
+        init(pointer: OpaquePointer, size: ModelSize) {
+            self.pointer = pointer
+            self.size = size
+        }
 
-    /// Which model is currently loaded
-    private var loadedSize: ModelSize?
+        deinit {
+            SherpaOnnxDestroyOfflineRecognizer(pointer)
+        }
+    }
 
-    /// Serializes recognizer lifecycle against decoding
+    private var loadedRecognizer: LoadedRecognizer?
+    private var loadGeneration = UUID()
     private let recognizerLock = NSLock()
 
-    var isModelLoaded: Bool { recognizer != nil }
+    var isModelLoaded: Bool { snapshot() != nil }
+    var loadedModelName: String? { snapshot()?.size.rawValue }
 
-    var loadedModelName: String? { loadedSize?.rawValue }
+    /// Retain the model atomically so its lifetime includes all request segments.
+    private func snapshot() -> LoadedRecognizer? {
+        recognizerLock.lock()
+        defer { recognizerLock.unlock() }
+        return loadedRecognizer
+    }
 
     deinit {
         unloadModel()
@@ -147,7 +165,7 @@ final class SherpaService: @unchecked Sendable {
         language: String?,
         onPhaseChange: ((String) -> Void)? = nil
     ) async throws {
-        unloadModel()
+        let generation = clearModel()
 
         guard let size = modelName.flatMap(ModelSize.init(rawValue:)),
               let spec = SherpaModelCatalog.spec(for: size) else {
@@ -191,38 +209,49 @@ final class SherpaService: @unchecked Sendable {
             throw SherpaError.initializationFailed(reason: "sherpa-onnx rejected the model files at \(directory.path)")
         }
 
-        adopt(recognizer: created, size: size)
+        guard !Task.isCancelled else {
+            SherpaOnnxDestroyOfflineRecognizer(created)
+            throw CancellationError()
+        }
+        guard adopt(recognizer: created, size: size, generation: generation) else {
+            throw CancellationError()
+        }
 
         let elapsed = CFAbsoluteTimeGetCurrent() - startTime
         VocaLogger.info(.sherpaService, "ONNX model loaded in \(String(format: "%.2f", elapsed))s")
     }
 
-    /// Take ownership of a freshly created recognizer.
-    ///
-    /// Destroys whatever was installed before rather than overwriting it:
-    /// the pointer is native memory, so dropping the reference would leak the
-    /// model. Loads are serialized upstream, but this keeps the object safe
-    /// on its own terms.
-    private func adopt(recognizer created: OpaquePointer, size: ModelSize) {
+    /// Atomically install the model; existing requests retain their previous model.
+    private func adopt(recognizer created: OpaquePointer, size: ModelSize, generation: UUID) -> Bool {
+        let model = LoadedRecognizer(pointer: created, size: size)
         recognizerLock.lock()
-        defer { recognizerLock.unlock() }
-        if let existing = recognizer {
-            SherpaOnnxDestroyOfflineRecognizer(existing)
+        guard loadGeneration == generation else {
+            recognizerLock.unlock()
+            return false
         }
-        recognizer = created
-        loadedSize = size
+        let previous = loadedRecognizer
+        loadedRecognizer = model
+        recognizerLock.unlock()
+        // Release native memory outside the state lock.
+        withExtendedLifetime(previous) {}
+        return true
     }
 
-    /// Unload the current model and free memory
+    /// Remove the active model. In-flight requests release it when decoding finishes.
     func unloadModel() {
+        _ = clearModel()
+    }
+
+    /// Invalidate pending loads as well as removing the active model.
+    private func clearModel() -> UUID {
         recognizerLock.lock()
-        if let recognizer {
-            SherpaOnnxDestroyOfflineRecognizer(recognizer)
-            VocaLogger.info(.sherpaService, "ONNX model unloaded")
-        }
-        recognizer = nil
-        loadedSize = nil
+        let generation = UUID()
+        loadGeneration = generation
+        let previous = loadedRecognizer
+        loadedRecognizer = nil
         recognizerLock.unlock()
+        withExtendedLifetime(previous) {}
+        return generation
     }
 
     // MARK: - Transcription
@@ -236,13 +265,10 @@ final class SherpaService: @unchecked Sendable {
         audioData: [Float],
         language: String? = nil
     ) async throws -> VocaTranscription {
-        guard isModelLoaded, let size = loadedSize else {
-            throw SherpaError.modelNotLoaded
-        }
+        guard let request = snapshot() else { throw SherpaError.modelNotLoaded }
+        let size = request.size
 
-        guard !audioData.isEmpty else {
-            throw SherpaError.emptyAudio
-        }
+        try SherpaAudioPreparation.validate(audioData)
 
         let audioLengthSeconds = Double(audioData.count) / 16000.0
         VocaLogger.info(.sherpaService, "ONNX transcribing \(String(format: "%.1f", audioLengthSeconds))s of audio...")
@@ -264,36 +290,32 @@ final class SherpaService: @unchecked Sendable {
             segments = [audioData]
         }
 
-        let decoded: (text: String, lang: String)? = await withCheckedContinuation { continuation in
-            DispatchQueue.global(qos: .userInitiated).async { [weak self] in
-                guard let self else {
-                    continuation.resume(returning: nil)
-                    return
-                }
-                var pieces: [String] = []
-                var detected = ""
-                for segment in segments {
-                    guard let result = self.decodeLocked(samples: segment) else {
-                        continuation.resume(returning: nil)
-                        return
+        let decoded: (text: String, lang: String) = try await withCheckedThrowingContinuation { continuation in
+            DispatchQueue.global(qos: .userInitiated).async {
+                do {
+                    // Keep one recognizer for the entire recording, including all segments.
+                    request.decodeLock.lock()
+                    defer { request.decodeLock.unlock() }
+                    var pieces: [String] = []
+                    var detected = ""
+                    for segment in segments {
+                        let samples = SherpaAudioPreparation.prepare(segment)
+                        guard !samples.isEmpty else { continue }
+                        let result = try Self.decode(samples: samples, recognizer: request.pointer)
+                        let piece = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
+                        if !piece.isEmpty { pieces.append(piece) }
+                        if detected.isEmpty { detected = result.lang }
                     }
-                    let piece = result.text.trimmingCharacters(in: .whitespacesAndNewlines)
-                    if !piece.isEmpty { pieces.append(piece) }
-                    if detected.isEmpty { detected = result.lang }
+                    let joinLanguage = detected.isEmpty ? (language ?? "") : detected
+                    continuation.resume(returning: (
+                        Self.joinTranscriptPieces(pieces, language: joinLanguage), detected
+                    ))
+                } catch {
+                    continuation.resume(throwing: error)
                 }
-                // Prefer the model's detected language; fall back to the caller's
-                // preference so CJK SenseVoice output is not space-joined.
-                let joinLanguage = detected.isEmpty ? (language ?? "") : detected
-                continuation.resume(
-                    returning: (Self.joinTranscriptPieces(pieces, language: joinLanguage), detected)
-                )
             }
         }
 
-        guard let decoded else {
-            throw SherpaError.transcriptionFailed(reason: "The model was unloaded during transcription.")
-        }
-
         let elapsed = CFAbsoluteTimeGetCurrent() - startTime
         let text = decoded.text.trimmingCharacters(in: .whitespacesAndNewlines)
 
@@ -326,16 +348,12 @@ final class SherpaService: @unchecked Sendable {
         return pieces.joined(separator: cjk ? "" : " ")
     }
 
-    /// Run one decode against the active recognizer. Returns nil if no model
-    /// is loaded. Called off the main thread; holds the lock so the
-    /// recognizer cannot be destroyed mid-decode.
-    private func decodeLocked(samples: [Float]) -> (text: String, lang: String)? {
-        recognizerLock.lock()
-        defer { recognizerLock.unlock() }
-
-        guard let recognizer,
-              let stream = SherpaOnnxCreateOfflineStream(recognizer) else {
-            return nil
+    /// Decode while the caller holds the recognizer lock for the whole request.
+    private static func decode(
+        samples: [Float], recognizer: OpaquePointer
+    ) throws -> (text: String, lang: String) {
+        guard let stream = SherpaOnnxCreateOfflineStream(recognizer) else {
+            throw SherpaError.transcriptionFailed(reason: "Could not create an ONNX audio stream.")
         }
         defer { SherpaOnnxDestroyOfflineStream(stream) }
 
@@ -345,7 +363,7 @@ final class SherpaService: @unchecked Sendable {
         SherpaOnnxDecodeOfflineStream(recognizer, stream)
 
         guard let result = SherpaOnnxGetOfflineStreamResult(stream) else {
-            return ("", "")
+            throw SherpaError.transcriptionFailed(reason: "The ONNX decoder returned no result.")
         }
         defer { SherpaOnnxDestroyOfflineRecognizerResult(result) }
 

Sources/VocaMac/Vendor/SherpaOnnxConfigBuilders.swift

@@ -2,8 +2,8 @@
 // VocaMac (vendored)
 //
 // Trimmed copy of swift-api-examples/SherpaOnnx.swift from the sherpa-onnx
-// project (https://github.com/k2-fsa/sherpa-onnx), revision
-// 00ad9a19a63751a6c4b12050a00eacfeb204814e, Copyright (c) 2023 Xiaomi
+// project (https://github.com/k2-fsa/sherpa-onnx), release
+// v1.13.7, Copyright (c) 2023 Xiaomi
 // Corporation, Apache License 2.0.
 //
 // The upstream file lives in the package's example target and declares
@@ -22,6 +22,7 @@ func toCPointer(_ s: String) -> UnsafePointer<Int8>! {
   let cs = (s as NSString).utf8String
   return UnsafePointer<Int8>(cs)
 }
+
 func sherpaOnnxFeatureConfig(
   sampleRate: Int = 16000,
   featureDim: Int = 80
@@ -50,6 +51,7 @@ func sherpaOnnxHomophoneReplacerConfig(
     lexicon: toCPointer(lexicon),
     rule_fsts: toCPointer(ruleFsts))
 }
+
 func sherpaOnnxOfflineTransducerModelConfig(
   encoder: String = "",
   decoder: String = "",

Tests/VocaMacTests/Fixtures/README.md

@@ -0,0 +1,11 @@
+Short English speech fixtures generated locally with macOS `say`, Samantha,
+220 words per minute, mono signed 16-bit PCM at 16 kHz. Leading/trailing
+samples below amplitude 150 were trimmed, preserving 10 ms on either side.
+
+- `short-yes.wav`: "Yes", 0.369875 seconds. Before padding, Moonshine Tiny
+  v2 with sherpa-onnx 1.13.4 returned "Yes, yes, yes".
+- `short-stop.wav`: "Stop", 0.3684375 seconds. Before padding, the same
+  configuration returned "Star".
+
+These are synthetic voices, not user recordings. Integration tests skip
+models which are not installed; audio preparation tests always run in CI.

Tests/VocaMacTests/Fixtures/short-stop.wav

@@ -0,0 +1,46 @@
+import XCTest
+@testable import VocaMac
+
+final class SherpaAudioPreparationTests: XCTestCase {
+    func testValidationRejectsEmptyAndNonFiniteInput() {
+        let invalid: [[Float]] = [[], [.nan], [.infinity], [-.infinity], [0.1, .nan]]
+        for samples in invalid {
+            XCTAssertThrowsError(try SherpaAudioPreparation.validate(samples))
+        }
+        XCTAssertNoThrow(try SherpaAudioPreparation.validate([0, 0.000001, -1, 1]))
+    }
+
+    func testShortSpeechPreservesEverySampleAndAddsTrailingSilence() {
+        let speech: [Float] = [0.1, -0.3, 0.2]
+        let prepared = SherpaAudioPreparation.prepare(speech)
+        XCTAssertEqual(prepared.count, 16_000)
+        XCTAssertEqual(Array(prepared.prefix(speech.count)), speech)
+        XCTAssertTrue(prepared.dropFirst(speech.count).allSatisfy { $0 == 0 })
+    }
+
+    func testEmptyAndDigitalSilenceDoNotReachDecoder() {
+        XCTAssertEqual(SherpaAudioPreparation.prepare([]), [])
+        XCTAssertEqual(SherpaAudioPreparation.prepare([Float](repeating: 0, count: 32_000)), [])
+    }
+
+    func testQuietSpeechIsNotDiscarded() {
+        XCTAssertEqual(SherpaAudioPreparation.prepare([0.000001]).first, 0.000001)
+    }
+
+    func testNormalLengthAudioIsUnchanged() {
+        let samples = [Float](repeating: 0.1, count: 16_001)
+        XCTAssertEqual(SherpaAudioPreparation.prepare(samples), samples)
+    }
+
+    func testShortFinalSegmentAlsoReceivesPadding() {
+        var recording = [Float](repeating: 0.1, count: 128_001)
+        recording.replaceSubrange(108_800..<128_000, with: repeatElement(Float.zero, count: 19_200))
+        let segments = AudioSegmenter.segment(recording, maxSeconds: 8)
+        XCTAssertTrue(segments.contains { $0.count < 16_000 })
+        for segment in segments {
+            let prepared = SherpaAudioPreparation.prepare(segment)
+            XCTAssertGreaterThanOrEqual(prepared.count, 16_000)
+            XCTAssertEqual(Array(prepared.prefix(segment.count)), segment)
+        }
+    }
+}