From e81f1faea5a24bac37443282cd71f56bda4eb072 Mon Sep 17 00:00:00 2001 From: Jatin K Malik Date: Fri, 10 Jul 2026 17:29:30 -0700 Subject: [PATCH 1/2] fix: retry empty prompted transcriptions --- Sources/VocaMac/Services/WhisperService.swift | 28 +++++++++++++++---- Tests/VocaMacTests/WhisperServiceTests.swift | 6 ++++ 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/Sources/VocaMac/Services/WhisperService.swift b/Sources/VocaMac/Services/WhisperService.swift index 94aaca7..8b094ce 100644 --- a/Sources/VocaMac/Services/WhisperService.swift +++ b/Sources/VocaMac/Services/WhisperService.swift @@ -166,7 +166,7 @@ final class WhisperService: @unchecked Sendable { let promptTokens = Self.promptTokens(for: vocabulary, tokenizer: kit.tokenizer) // Configure decoding options — optimized for low latency dictation - let options = DecodingOptions( + var options = DecodingOptions( task: translate ? .translate : .transcribe, language: language, temperature: 0.0, @@ -179,19 +179,31 @@ final class WhisperService: @unchecked Sendable { ) do { - let results = try await kit.transcribe( + var results = try await kit.transcribe( audioArray: audioData, decodeOptions: options ) - let elapsed = CFAbsoluteTimeGetCurrent() - startTime - // Concatenate all segment texts - let rawText = results.map { $0.text }.joined(separator: " ") + var rawText = results.map { $0.text }.joined(separator: " ") // Filter out WhisperKit hallucination tokens that should not be // exposed to the user (e.g. "[BLANK_AUDIO]", "(blank audio)", etc.) - let fullText = Self.filterHallucinationTokens(rawText) + var fullText = Self.filterHallucinationTokens(rawText) + + // WhisperKit can exit during prompt prefill and return no text for + // some models. Preserve vocabulary bias normally, but recover the + // dictation by retrying once without custom prompt tokens. + if Self.shouldRetryWithoutVocabulary(text: fullText, promptTokens: promptTokens) { + VocaLogger.warning(.whisperService, "Prompted transcription was empty; retrying without custom vocabulary") + options.promptTokens = nil + options.usePrefillPrompt = language != nil + results = try await kit.transcribe(audioArray: audioData, decodeOptions: options) + rawText = results.map { $0.text }.joined(separator: " ") + fullText = Self.filterHallucinationTokens(rawText) + } + + let elapsed = CFAbsoluteTimeGetCurrent() - startTime // Get detected language from first result let detectedLanguage = results.first?.language ?? language ?? "en" @@ -274,6 +286,10 @@ final class WhisperService: @unchecked Sendable { .filter { !$0.isEmpty } } + static func shouldRetryWithoutVocabulary(text: String, promptTokens: [Int]?) -> Bool { + text.isEmpty && promptTokens != nil + } + /// Encode custom vocabulary into WhisperKit conditioning tokens. /// Returns nil when there are no terms or the tokenizer isn't ready yet. /// Framed as a "Glossary:" prompt, which nudges Whisper to treat the terms diff --git a/Tests/VocaMacTests/WhisperServiceTests.swift b/Tests/VocaMacTests/WhisperServiceTests.swift index f6517c5..c1c8a73 100644 --- a/Tests/VocaMacTests/WhisperServiceTests.swift +++ b/Tests/VocaMacTests/WhisperServiceTests.swift @@ -88,4 +88,10 @@ final class WhisperServiceVocabularyTests: XCTestCase { let input = "Namrata,,\n\n, ,VocaMac" XCTAssertEqual(WhisperService.vocabularyTerms(from: input), ["Namrata", "VocaMac"]) } + + func testRetriesOnlyEmptyPromptedTranscriptions() { + XCTAssertTrue(WhisperService.shouldRetryWithoutVocabulary(text: "", promptTokens: [1])) + XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(text: "transcribed", promptTokens: [1])) + XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(text: "", promptTokens: nil)) + } } From fa1bed6bc91e2814e59cf953000d1c4b68183c9d Mon Sep 17 00:00:00 2001 From: Jatin K Malik Date: Fri, 10 Jul 2026 17:38:06 -0700 Subject: [PATCH 2/2] fix: narrow vocabulary retry condition --- Sources/VocaMac/Services/WhisperService.swift | 11 +++++++---- Tests/VocaMacTests/WhisperServiceTests.swift | 7 ++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Sources/VocaMac/Services/WhisperService.swift b/Sources/VocaMac/Services/WhisperService.swift index 8b094ce..60c3a61 100644 --- a/Sources/VocaMac/Services/WhisperService.swift +++ b/Sources/VocaMac/Services/WhisperService.swift @@ -194,8 +194,11 @@ final class WhisperService: @unchecked Sendable { // WhisperKit can exit during prompt prefill and return no text for // some models. Preserve vocabulary bias normally, but recover the // dictation by retrying once without custom prompt tokens. - if Self.shouldRetryWithoutVocabulary(text: fullText, promptTokens: promptTokens) { - VocaLogger.warning(.whisperService, "Prompted transcription was empty; retrying without custom vocabulary") + if Self.shouldRetryWithoutVocabulary(rawText: rawText, promptTokens: promptTokens) { + VocaLogger.warning( + .whisperService, + "Prompted transcription was empty for \(loadedModelName ?? "unknown model"); retrying without custom vocabulary" + ) options.promptTokens = nil options.usePrefillPrompt = language != nil results = try await kit.transcribe(audioArray: audioData, decodeOptions: options) @@ -286,8 +289,8 @@ final class WhisperService: @unchecked Sendable { .filter { !$0.isEmpty } } - static func shouldRetryWithoutVocabulary(text: String, promptTokens: [Int]?) -> Bool { - text.isEmpty && promptTokens != nil + static func shouldRetryWithoutVocabulary(rawText: String, promptTokens: [Int]?) -> Bool { + promptTokens != nil && rawText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty } /// Encode custom vocabulary into WhisperKit conditioning tokens. diff --git a/Tests/VocaMacTests/WhisperServiceTests.swift b/Tests/VocaMacTests/WhisperServiceTests.swift index c1c8a73..d261430 100644 --- a/Tests/VocaMacTests/WhisperServiceTests.swift +++ b/Tests/VocaMacTests/WhisperServiceTests.swift @@ -90,8 +90,9 @@ final class WhisperServiceVocabularyTests: XCTestCase { } func testRetriesOnlyEmptyPromptedTranscriptions() { - XCTAssertTrue(WhisperService.shouldRetryWithoutVocabulary(text: "", promptTokens: [1])) - XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(text: "transcribed", promptTokens: [1])) - XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(text: "", promptTokens: nil)) + XCTAssertTrue(WhisperService.shouldRetryWithoutVocabulary(rawText: " \n", promptTokens: [1])) + XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(rawText: "transcribed", promptTokens: [1])) + XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(rawText: "[BLANK_AUDIO]", promptTokens: [1])) + XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(rawText: "", promptTokens: nil)) } }