diff --git a/Sources/VocaMac/Services/SherpaAudioPreparation.swift b/Sources/VocaMac/Services/SherpaAudioPreparation.swift index c1eab4d..75a5b1d 100644 --- a/Sources/VocaMac/Services/SherpaAudioPreparation.swift +++ b/Sources/VocaMac/Services/SherpaAudioPreparation.swift @@ -8,6 +8,24 @@ enum SherpaAudioPreparation { static let minimumSampleCount = 16_000 + /// Silence added on both sides of every segment — 200ms at 16kHz. + /// + /// A push-to-talk recording starts on the first frame the tap delivers, so + /// speech often begins in sample zero. The NeMo-derived models decode that + /// as an utterance already in progress and their attention decoder emits + /// end-of-transcript as its very first token, which comes back as an empty + /// result for perfectly good audio — the recording is simply dropped. + /// Measured against Canary 180M, a lead-in as short as 50ms is enough to + /// recover every clip that failed this way; 200ms leaves margin, and the + /// matching tail keeps a final word from being cut off mid-decode. + static let edgeSilenceSampleCount = 3_200 + + /// Total silence `prepare` adds to a segment, in seconds. + /// + /// Callers that cap segment length must subtract this: the models' limits + /// apply to what actually reaches the decoder, not to the speech alone. + static let addedSilenceSeconds = Double(2 * edgeSilenceSampleCount) / 16_000 + /// Reject malformed input before segmentation or any native inference. static func validate(_ samples: [Float]) throws { guard !samples.isEmpty else { throw SherpaError.emptyAudio } @@ -19,7 +37,20 @@ enum SherpaAudioPreparation { static func prepare(_ samples: [Float]) -> [Float] { // Do not ask generative decoders to invent words for digital silence. guard samples.contains(where: { $0 != 0 }) else { return [] } - guard samples.count < minimumSampleCount else { return samples } - return samples + [Float](repeating: 0, count: minimumSampleCount - samples.count) + + let edge = repeatElement(Float.zero, count: edgeSilenceSampleCount) + let paddedCount = samples.count + 2 * edgeSilenceSampleCount + var prepared: [Float] = [] + prepared.reserveCapacity(max(minimumSampleCount, paddedCount)) + prepared.append(contentsOf: edge) + prepared.append(contentsOf: samples) + prepared.append(contentsOf: edge) + + if prepared.count < minimumSampleCount { + prepared.append( + contentsOf: repeatElement(Float.zero, count: minimumSampleCount - prepared.count) + ) + } + return prepared } } diff --git a/Sources/VocaMac/Services/SherpaService.swift b/Sources/VocaMac/Services/SherpaService.swift index 1b79add..2958947 100644 --- a/Sources/VocaMac/Services/SherpaService.swift +++ b/Sources/VocaMac/Services/SherpaService.swift @@ -279,14 +279,17 @@ final class SherpaService: @unchecked Sendable { // These models decode an utterance in one pass and degrade past a // certain length — Moonshine returns nothing at all — so anything - // longer is split at pauses and decoded segment by segment. - let maxSeconds = SherpaModelCatalog.spec(for: size)?.maxSegmentSeconds + // longer is split at pauses and decoded segment by segment. Every + // segment then gains a silent lead-in and tail, which counts against + // the same limit, so the speech budget is what is left after it. + let segmentLimit = SherpaModelCatalog.spec(for: size)?.maxSegmentSeconds + let maxSeconds = segmentLimit.map { $0 - SherpaAudioPreparation.addedSilenceSeconds } let segments: [[Float]] if let maxSeconds, audioLengthSeconds > maxSeconds { segments = AudioSegmenter.segment(audioData, maxSeconds: maxSeconds) VocaLogger.info( .sherpaService, - "Audio exceeds \(String(format: "%.0f", maxSeconds))s for \(size.rawValue) — split into \(segments.count) segments" + "Audio exceeds \(String(format: "%.1f", maxSeconds))s for \(size.rawValue) — split into \(segments.count) segments" ) } else { segments = [audioData] @@ -310,7 +313,27 @@ final class SherpaService: @unchecked Sendable { let text = decoded.text.trimmingCharacters(in: .whitespacesAndNewlines) VocaLogger.info(.sherpaService, "ONNX transcription completed in \(String(format: "%.2f", elapsed))s") - VocaLogger.info(.sherpaService, "Result: \(text.prefix(100))...") + if text.isEmpty { + // The decoders return an empty string rather than an error when + // they stop on their first token, so nothing else marks a dropped + // recording. Say so plainly — an INFO line reading "Result: ..." + // is indistinguishable from a successful decode in a log. Digital + // silence never reaches the decoder at all, so it is not a + // failure and must not look like one. + if audioData.allSatisfy({ $0 == 0 }) { + VocaLogger.info( + .sherpaService, + "Nothing to decode — the recording is digital silence" + ) + } else { + VocaLogger.warning( + .sherpaService, + "ONNX decode returned no text for \(String(format: "%.1f", audioLengthSeconds))s of audio" + ) + } + } else { + VocaLogger.info(.sherpaService, "Result: \(text.prefix(100))...") + } // SenseVoice reports the detected language; other models are // monolingual or fixed at load time. diff --git a/Tests/VocaMacTests/SherpaAudioPreparationTests.swift b/Tests/VocaMacTests/SherpaAudioPreparationTests.swift index 9d3c4ed..4875fb6 100644 --- a/Tests/VocaMacTests/SherpaAudioPreparationTests.swift +++ b/Tests/VocaMacTests/SherpaAudioPreparationTests.swift @@ -2,6 +2,8 @@ import XCTest @testable import VocaMac final class SherpaAudioPreparationTests: XCTestCase { + private let edge = SherpaAudioPreparation.edgeSilenceSampleCount + func testValidationRejectsEmptyAndNonFiniteInput() { let invalid: [[Float]] = [[], [.nan], [.infinity], [-.infinity], [0.1, .nan]] for samples in invalid { @@ -10,12 +12,55 @@ final class SherpaAudioPreparationTests: XCTestCase { XCTAssertNoThrow(try SherpaAudioPreparation.validate([0, 0.000001, -1, 1])) } - func testShortSpeechPreservesEverySampleAndAddsTrailingSilence() { + func testShortSpeechIsPaddedToTheMinimumSampleCount() { let speech: [Float] = [0.1, -0.3, 0.2] let prepared = SherpaAudioPreparation.prepare(speech) XCTAssertEqual(prepared.count, 16_000) - XCTAssertEqual(Array(prepared.prefix(speech.count)), speech) - XCTAssertTrue(prepared.dropFirst(speech.count).allSatisfy { $0 == 0 }) + XCTAssertEqual(Array(prepared[edge..<(edge + speech.count)]), speech) + XCTAssertTrue(prepared.dropFirst(edge + speech.count).allSatisfy { $0 == 0 }) + } + + /// Speech that starts in sample zero makes the NeMo decoders emit + /// end-of-transcript immediately and return nothing at all, so every + /// segment gets a silent lead-in. + func testSpeechNeverStartsInTheFirstSample() { + let cases: [[Float]] = [ + [0.4], + [Float](repeating: 0.1, count: 16_001), + [Float](repeating: -0.2, count: 320_000), + ] + for samples in cases { + let prepared = SherpaAudioPreparation.prepare(samples) + XCTAssertEqual(Array(prepared.prefix(edge)), [Float](repeating: 0, count: edge)) + XCTAssertEqual(Array(prepared.suffix(edge)), [Float](repeating: 0, count: edge)) + XCTAssertEqual(Array(prepared[edge..<(edge + samples.count)]), samples) + } + } + + /// The segment budget in SherpaService subtracts this, so it has to stay + /// equal to what `prepare` actually adds. + func testAddedSilenceSecondsMatchesWhatPrepareAdds() { + let samples = [Float](repeating: 0.1, count: 320_000) + let prepared = SherpaAudioPreparation.prepare(samples) + let added = Double(prepared.count - samples.count) / 16_000 + XCTAssertEqual(added, SherpaAudioPreparation.addedSilenceSeconds, accuracy: 0.0001) + } + + /// A segment cut to the model's budget must still fit that model's + /// one-pass limit once it is padded — Moonshine returns nothing at all + /// past its limit, and SenseVoice drops characters. + func testMaximumSegmentStillFitsEveryModelsLimitAfterPadding() { + for spec in SherpaModelCatalog.specs { + let budget = spec.maxSegmentSeconds - SherpaAudioPreparation.addedSilenceSeconds + XCTAssertGreaterThan(budget, 0, "\(spec.size.rawValue) has no room for the padding") + let segment = [Float](repeating: 0.1, count: Int(budget * 16_000)) + let prepared = SherpaAudioPreparation.prepare(segment) + XCTAssertLessThanOrEqual( + Double(prepared.count) / 16_000, + spec.maxSegmentSeconds, + "\(spec.size.rawValue) overruns its limit once padded" + ) + } } func testEmptyAndDigitalSilenceDoNotReachDecoder() { @@ -24,12 +69,14 @@ final class SherpaAudioPreparationTests: XCTestCase { } func testQuietSpeechIsNotDiscarded() { - XCTAssertEqual(SherpaAudioPreparation.prepare([0.000001]).first, 0.000001) + XCTAssertEqual(SherpaAudioPreparation.prepare([0.000001])[edge], 0.000001) } - func testNormalLengthAudioIsUnchanged() { - let samples = [Float](repeating: 0.1, count: 16_001) - XCTAssertEqual(SherpaAudioPreparation.prepare(samples), samples) + func testLongAudioKeepsEverySampleAndOnlyGainsEdgeSilence() { + let samples = (0..<16_001).map { Float($0 % 7) * 0.1 + 0.01 } + let prepared = SherpaAudioPreparation.prepare(samples) + XCTAssertEqual(prepared.count, samples.count + 2 * edge) + XCTAssertEqual(Array(prepared[edge..<(edge + samples.count)]), samples) } func testShortFinalSegmentAlsoReceivesPadding() { @@ -40,7 +87,7 @@ final class SherpaAudioPreparationTests: XCTestCase { for segment in segments { let prepared = SherpaAudioPreparation.prepare(segment) XCTAssertGreaterThanOrEqual(prepared.count, 16_000) - XCTAssertEqual(Array(prepared.prefix(segment.count)), segment) + XCTAssertEqual(Array(prepared[edge..<(edge + segment.count)]), segment) } } } diff --git a/Tests/VocaMacTests/SherpaServiceTests.swift b/Tests/VocaMacTests/SherpaServiceTests.swift index 2187711..dbcf055 100644 --- a/Tests/VocaMacTests/SherpaServiceTests.swift +++ b/Tests/VocaMacTests/SherpaServiceTests.swift @@ -180,7 +180,8 @@ final class SherpaServiceTests: XCTestCase { counts.append(samples.count) return counts.count == 1 ? ("안녕하세요", "ko") : ("반갑습니다", "ko") } - XCTAssertEqual(counts, [32_000, 16_000]) + let edge = SherpaAudioPreparation.edgeSilenceSampleCount + XCTAssertEqual(counts, [32_000 + 2 * edge, 16_000]) XCTAssertEqual(result.text, "안녕하세요 반갑습니다") }