From 022d7c6e8cdcb54758b7639dd523ded68ec3378f Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Sat, 5 Sep 2026 11:04:33 +0530 Subject: [PATCH 1/2] fix: keep ONNX models from dropping whole recordings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The specialized ONNX models silently returned nothing for perfectly good audio, at any length and seemingly at random. The user-visible effect is a recording that transcribes to nothing at all. Cause: sherpa-onnx's NeMo-derived decoders (Canary here, same decoder family elsewhere) run greedy attention decoding and stop as soon as they emit end-of-transcript. When speech begins in sample zero — which push-to-talk recording makes common, since capture starts on the first frame the tap delivers — the model reads the clip as an utterance already in progress and emits end-of-transcript as its *first* token. sherpa-onnx then pops that token and hands back an empty string rather than an error, so nothing downstream can tell the recording was dropped. Give every segment 200ms of silence on each side before it reaches native code. Reproduced against Canary 180M with clips that returned "" — a lead-in as short as 50ms recovers all of them; 200ms leaves margin, and the matching tail keeps a final word from being cut off mid-decode. Verified across 14 clips: every previously empty one now transcribes and the ones that already worked are unchanged. The recording's reported duration is measured before padding, so it is unaffected. Also log a warning when a decode still comes back empty. The old INFO line read "Result: ..." either way, which is why this went unnoticed in the logs for so long. --- .../Services/SherpaAudioPreparation.swift | 29 ++++++++++++++- Sources/VocaMac/Services/SherpaService.swift | 13 ++++++- .../SherpaAudioPreparationTests.swift | 37 +++++++++++++++---- Tests/VocaMacTests/SherpaServiceTests.swift | 3 +- 4 files changed, 70 insertions(+), 12 deletions(-) diff --git a/Sources/VocaMac/Services/SherpaAudioPreparation.swift b/Sources/VocaMac/Services/SherpaAudioPreparation.swift index c1eab4d..2109928 100644 --- a/Sources/VocaMac/Services/SherpaAudioPreparation.swift +++ b/Sources/VocaMac/Services/SherpaAudioPreparation.swift @@ -8,6 +8,18 @@ enum SherpaAudioPreparation { static let minimumSampleCount = 16_000 + /// Silence added on both sides of every segment — 200ms at 16kHz. + /// + /// A push-to-talk recording starts on the first frame the tap delivers, so + /// speech often begins in sample zero. The NeMo-derived models decode that + /// as an utterance already in progress and their attention decoder emits + /// end-of-transcript as its very first token, which comes back as an empty + /// result for perfectly good audio — the recording is simply dropped. + /// Measured against Canary 180M, a lead-in as short as 50ms is enough to + /// recover every clip that failed this way; 200ms leaves margin, and the + /// matching tail keeps a final word from being cut off mid-decode. + static let edgeSilenceSampleCount = 3_200 + /// Reject malformed input before segmentation or any native inference. static func validate(_ samples: [Float]) throws { guard !samples.isEmpty else { throw SherpaError.emptyAudio } @@ -19,7 +31,20 @@ enum SherpaAudioPreparation { static func prepare(_ samples: [Float]) -> [Float] { // Do not ask generative decoders to invent words for digital silence. guard samples.contains(where: { $0 != 0 }) else { return [] } - guard samples.count < minimumSampleCount else { return samples } - return samples + [Float](repeating: 0, count: minimumSampleCount - samples.count) + + let edge = repeatElement(Float.zero, count: edgeSilenceSampleCount) + let paddedCount = samples.count + 2 * edgeSilenceSampleCount + var prepared: [Float] = [] + prepared.reserveCapacity(max(minimumSampleCount, paddedCount)) + prepared.append(contentsOf: edge) + prepared.append(contentsOf: samples) + prepared.append(contentsOf: edge) + + if prepared.count < minimumSampleCount { + prepared.append( + contentsOf: repeatElement(Float.zero, count: minimumSampleCount - prepared.count) + ) + } + return prepared } } diff --git a/Sources/VocaMac/Services/SherpaService.swift b/Sources/VocaMac/Services/SherpaService.swift index 1b79add..a4e8814 100644 --- a/Sources/VocaMac/Services/SherpaService.swift +++ b/Sources/VocaMac/Services/SherpaService.swift @@ -310,7 +310,18 @@ final class SherpaService: @unchecked Sendable { let text = decoded.text.trimmingCharacters(in: .whitespacesAndNewlines) VocaLogger.info(.sherpaService, "ONNX transcription completed in \(String(format: "%.2f", elapsed))s") - VocaLogger.info(.sherpaService, "Result: \(text.prefix(100))...") + if text.isEmpty { + // The decoders return an empty string rather than an error when + // they stop on their first token, so nothing else marks a dropped + // recording. Say so plainly — an INFO line reading "Result: ..." + // is indistinguishable from a successful decode in a log. + VocaLogger.warning( + .sherpaService, + "ONNX decode returned no text for \(String(format: "%.1f", audioLengthSeconds))s of audio" + ) + } else { + VocaLogger.info(.sherpaService, "Result: \(text.prefix(100))...") + } // SenseVoice reports the detected language; other models are // monolingual or fixed at load time. diff --git a/Tests/VocaMacTests/SherpaAudioPreparationTests.swift b/Tests/VocaMacTests/SherpaAudioPreparationTests.swift index 9d3c4ed..23bd966 100644 --- a/Tests/VocaMacTests/SherpaAudioPreparationTests.swift +++ b/Tests/VocaMacTests/SherpaAudioPreparationTests.swift @@ -2,6 +2,8 @@ import XCTest @testable import VocaMac final class SherpaAudioPreparationTests: XCTestCase { + private let edge = SherpaAudioPreparation.edgeSilenceSampleCount + func testValidationRejectsEmptyAndNonFiniteInput() { let invalid: [[Float]] = [[], [.nan], [.infinity], [-.infinity], [0.1, .nan]] for samples in invalid { @@ -10,12 +12,29 @@ final class SherpaAudioPreparationTests: XCTestCase { XCTAssertNoThrow(try SherpaAudioPreparation.validate([0, 0.000001, -1, 1])) } - func testShortSpeechPreservesEverySampleAndAddsTrailingSilence() { + func testShortSpeechIsPaddedToTheMinimumSampleCount() { let speech: [Float] = [0.1, -0.3, 0.2] let prepared = SherpaAudioPreparation.prepare(speech) XCTAssertEqual(prepared.count, 16_000) - XCTAssertEqual(Array(prepared.prefix(speech.count)), speech) - XCTAssertTrue(prepared.dropFirst(speech.count).allSatisfy { $0 == 0 }) + XCTAssertEqual(Array(prepared[edge..<(edge + speech.count)]), speech) + XCTAssertTrue(prepared.dropFirst(edge + speech.count).allSatisfy { $0 == 0 }) + } + + /// Speech that starts in sample zero makes the NeMo decoders emit + /// end-of-transcript immediately and return nothing at all, so every + /// segment gets a silent lead-in. + func testSpeechNeverStartsInTheFirstSample() { + let cases: [[Float]] = [ + [0.4], + [Float](repeating: 0.1, count: 16_001), + [Float](repeating: -0.2, count: 320_000), + ] + for samples in cases { + let prepared = SherpaAudioPreparation.prepare(samples) + XCTAssertEqual(Array(prepared.prefix(edge)), [Float](repeating: 0, count: edge)) + XCTAssertEqual(Array(prepared.suffix(edge)), [Float](repeating: 0, count: edge)) + XCTAssertEqual(Array(prepared[edge..<(edge + samples.count)]), samples) + } } func testEmptyAndDigitalSilenceDoNotReachDecoder() { @@ -24,12 +43,14 @@ final class SherpaAudioPreparationTests: XCTestCase { } func testQuietSpeechIsNotDiscarded() { - XCTAssertEqual(SherpaAudioPreparation.prepare([0.000001]).first, 0.000001) + XCTAssertEqual(SherpaAudioPreparation.prepare([0.000001])[edge], 0.000001) } - func testNormalLengthAudioIsUnchanged() { - let samples = [Float](repeating: 0.1, count: 16_001) - XCTAssertEqual(SherpaAudioPreparation.prepare(samples), samples) + func testLongAudioKeepsEverySampleAndOnlyGainsEdgeSilence() { + let samples = (0..<16_001).map { Float($0 % 7) * 0.1 + 0.01 } + let prepared = SherpaAudioPreparation.prepare(samples) + XCTAssertEqual(prepared.count, samples.count + 2 * edge) + XCTAssertEqual(Array(prepared[edge..<(edge + samples.count)]), samples) } func testShortFinalSegmentAlsoReceivesPadding() { @@ -40,7 +61,7 @@ final class SherpaAudioPreparationTests: XCTestCase { for segment in segments { let prepared = SherpaAudioPreparation.prepare(segment) XCTAssertGreaterThanOrEqual(prepared.count, 16_000) - XCTAssertEqual(Array(prepared.prefix(segment.count)), segment) + XCTAssertEqual(Array(prepared[edge..<(edge + segment.count)]), segment) } } } diff --git a/Tests/VocaMacTests/SherpaServiceTests.swift b/Tests/VocaMacTests/SherpaServiceTests.swift index 2187711..dbcf055 100644 --- a/Tests/VocaMacTests/SherpaServiceTests.swift +++ b/Tests/VocaMacTests/SherpaServiceTests.swift @@ -180,7 +180,8 @@ final class SherpaServiceTests: XCTestCase { counts.append(samples.count) return counts.count == 1 ? ("안녕하세요", "ko") : ("반갑습니다", "ko") } - XCTAssertEqual(counts, [32_000, 16_000]) + let edge = SherpaAudioPreparation.edgeSilenceSampleCount + XCTAssertEqual(counts, [32_000 + 2 * edge, 16_000]) XCTAssertEqual(result.text, "안녕하세요 반갑습니다") } From 321c1b04b34ed89f6ac0d32b780ccd99d29579ee Mon Sep 17 00:00:00 2001 From: Kanishk Pachauri Date: Sat, 5 Sep 2026 11:17:14 +0530 Subject: [PATCH 2/2] fix: keep segment padding inside the models' one-pass limits Two follow-ups from review. The lead-in and tail count against the same length limit the segmenter is there to respect: capping a segment at maxSegmentSeconds and then adding 400ms sent up to 8.4s to a model documented to degrade past 8s, which is the failure this change set out to fix. Take the padding out of the segment budget so what reaches the decoder stays inside the limit, and pin that with a test over every model in the catalog. Digital silence is skipped before native inference, so reporting it as a decode that returned nothing blurs the very signal the new warning exists to make obvious. Log it as the no-op it is. Both the GUI and the CLI already reject silence upstream, so this is reachable only through direct service calls today. --- .../Services/SherpaAudioPreparation.swift | 6 ++++ Sources/VocaMac/Services/SherpaService.swift | 28 +++++++++++++------ .../SherpaAudioPreparationTests.swift | 26 +++++++++++++++++ 3 files changed, 52 insertions(+), 8 deletions(-) diff --git a/Sources/VocaMac/Services/SherpaAudioPreparation.swift b/Sources/VocaMac/Services/SherpaAudioPreparation.swift index 2109928..75a5b1d 100644 --- a/Sources/VocaMac/Services/SherpaAudioPreparation.swift +++ b/Sources/VocaMac/Services/SherpaAudioPreparation.swift @@ -20,6 +20,12 @@ enum SherpaAudioPreparation { /// matching tail keeps a final word from being cut off mid-decode. static let edgeSilenceSampleCount = 3_200 + /// Total silence `prepare` adds to a segment, in seconds. + /// + /// Callers that cap segment length must subtract this: the models' limits + /// apply to what actually reaches the decoder, not to the speech alone. + static let addedSilenceSeconds = Double(2 * edgeSilenceSampleCount) / 16_000 + /// Reject malformed input before segmentation or any native inference. static func validate(_ samples: [Float]) throws { guard !samples.isEmpty else { throw SherpaError.emptyAudio } diff --git a/Sources/VocaMac/Services/SherpaService.swift b/Sources/VocaMac/Services/SherpaService.swift index a4e8814..2958947 100644 --- a/Sources/VocaMac/Services/SherpaService.swift +++ b/Sources/VocaMac/Services/SherpaService.swift @@ -279,14 +279,17 @@ final class SherpaService: @unchecked Sendable { // These models decode an utterance in one pass and degrade past a // certain length — Moonshine returns nothing at all — so anything - // longer is split at pauses and decoded segment by segment. - let maxSeconds = SherpaModelCatalog.spec(for: size)?.maxSegmentSeconds + // longer is split at pauses and decoded segment by segment. Every + // segment then gains a silent lead-in and tail, which counts against + // the same limit, so the speech budget is what is left after it. + let segmentLimit = SherpaModelCatalog.spec(for: size)?.maxSegmentSeconds + let maxSeconds = segmentLimit.map { $0 - SherpaAudioPreparation.addedSilenceSeconds } let segments: [[Float]] if let maxSeconds, audioLengthSeconds > maxSeconds { segments = AudioSegmenter.segment(audioData, maxSeconds: maxSeconds) VocaLogger.info( .sherpaService, - "Audio exceeds \(String(format: "%.0f", maxSeconds))s for \(size.rawValue) — split into \(segments.count) segments" + "Audio exceeds \(String(format: "%.1f", maxSeconds))s for \(size.rawValue) — split into \(segments.count) segments" ) } else { segments = [audioData] @@ -314,11 +317,20 @@ final class SherpaService: @unchecked Sendable { // The decoders return an empty string rather than an error when // they stop on their first token, so nothing else marks a dropped // recording. Say so plainly — an INFO line reading "Result: ..." - // is indistinguishable from a successful decode in a log. - VocaLogger.warning( - .sherpaService, - "ONNX decode returned no text for \(String(format: "%.1f", audioLengthSeconds))s of audio" - ) + // is indistinguishable from a successful decode in a log. Digital + // silence never reaches the decoder at all, so it is not a + // failure and must not look like one. + if audioData.allSatisfy({ $0 == 0 }) { + VocaLogger.info( + .sherpaService, + "Nothing to decode — the recording is digital silence" + ) + } else { + VocaLogger.warning( + .sherpaService, + "ONNX decode returned no text for \(String(format: "%.1f", audioLengthSeconds))s of audio" + ) + } } else { VocaLogger.info(.sherpaService, "Result: \(text.prefix(100))...") } diff --git a/Tests/VocaMacTests/SherpaAudioPreparationTests.swift b/Tests/VocaMacTests/SherpaAudioPreparationTests.swift index 23bd966..4875fb6 100644 --- a/Tests/VocaMacTests/SherpaAudioPreparationTests.swift +++ b/Tests/VocaMacTests/SherpaAudioPreparationTests.swift @@ -37,6 +37,32 @@ final class SherpaAudioPreparationTests: XCTestCase { } } + /// The segment budget in SherpaService subtracts this, so it has to stay + /// equal to what `prepare` actually adds. + func testAddedSilenceSecondsMatchesWhatPrepareAdds() { + let samples = [Float](repeating: 0.1, count: 320_000) + let prepared = SherpaAudioPreparation.prepare(samples) + let added = Double(prepared.count - samples.count) / 16_000 + XCTAssertEqual(added, SherpaAudioPreparation.addedSilenceSeconds, accuracy: 0.0001) + } + + /// A segment cut to the model's budget must still fit that model's + /// one-pass limit once it is padded — Moonshine returns nothing at all + /// past its limit, and SenseVoice drops characters. + func testMaximumSegmentStillFitsEveryModelsLimitAfterPadding() { + for spec in SherpaModelCatalog.specs { + let budget = spec.maxSegmentSeconds - SherpaAudioPreparation.addedSilenceSeconds + XCTAssertGreaterThan(budget, 0, "\(spec.size.rawValue) has no room for the padding") + let segment = [Float](repeating: 0.1, count: Int(budget * 16_000)) + let prepared = SherpaAudioPreparation.prepare(segment) + XCTAssertLessThanOrEqual( + Double(prepared.count) / 16_000, + spec.maxSegmentSeconds, + "\(spec.size.rawValue) overruns its limit once padded" + ) + } + } + func testEmptyAndDigitalSilenceDoNotReachDecoder() { XCTAssertEqual(SherpaAudioPreparation.prepare([]), []) XCTAssertEqual(SherpaAudioPreparation.prepare([Float](repeating: 0, count: 32_000)), [])