Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 25 additions & 6 deletions Sources/VocaMac/Services/WhisperService.swift
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,7 @@ final class WhisperService: @unchecked Sendable {
let promptTokens = Self.promptTokens(for: vocabulary, tokenizer: kit.tokenizer)

// Configure decoding options — optimized for low latency dictation
let options = DecodingOptions(
var options = DecodingOptions(
task: translate ? .translate : .transcribe,
language: language,
temperature: 0.0,
Expand All @@ -179,19 +179,34 @@ final class WhisperService: @unchecked Sendable {
)

do {
let results = try await kit.transcribe(
var results = try await kit.transcribe(
audioArray: audioData,
decodeOptions: options
)

let elapsed = CFAbsoluteTimeGetCurrent() - startTime

// Concatenate all segment texts
let rawText = results.map { $0.text }.joined(separator: " ")
var rawText = results.map { $0.text }.joined(separator: " ")

// Filter out WhisperKit hallucination tokens that should not be
// exposed to the user (e.g. "[BLANK_AUDIO]", "(blank audio)", etc.)
let fullText = Self.filterHallucinationTokens(rawText)
var fullText = Self.filterHallucinationTokens(rawText)

// WhisperKit can exit during prompt prefill and return no text for
// some models. Preserve vocabulary bias normally, but recover the
// dictation by retrying once without custom prompt tokens.
if Self.shouldRetryWithoutVocabulary(rawText: rawText, promptTokens: promptTokens) {
VocaLogger.warning(
.whisperService,
"Prompted transcription was empty for \(loadedModelName ?? "unknown model"); retrying without custom vocabulary"
)
options.promptTokens = nil
options.usePrefillPrompt = language != nil
results = try await kit.transcribe(audioArray: audioData, decodeOptions: options)
rawText = results.map { $0.text }.joined(separator: " ")
fullText = Self.filterHallucinationTokens(rawText)
}

let elapsed = CFAbsoluteTimeGetCurrent() - startTime

// Get detected language from first result
let detectedLanguage = results.first?.language ?? language ?? "en"
Expand Down Expand Up @@ -274,6 +289,10 @@ final class WhisperService: @unchecked Sendable {
.filter { !$0.isEmpty }
}

static func shouldRetryWithoutVocabulary(rawText: String, promptTokens: [Int]?) -> Bool {
promptTokens != nil && rawText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty
}

/// Encode custom vocabulary into WhisperKit conditioning tokens.
/// Returns nil when there are no terms or the tokenizer isn't ready yet.
/// Framed as a "Glossary:" prompt, which nudges Whisper to treat the terms
Expand Down
7 changes: 7 additions & 0 deletions Tests/VocaMacTests/WhisperServiceTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -88,4 +88,11 @@ final class WhisperServiceVocabularyTests: XCTestCase {
let input = "Namrata,,\n\n, ,VocaMac"
XCTAssertEqual(WhisperService.vocabularyTerms(from: input), ["Namrata", "VocaMac"])
}

func testRetriesOnlyEmptyPromptedTranscriptions() {
XCTAssertTrue(WhisperService.shouldRetryWithoutVocabulary(rawText: " \n", promptTokens: [1]))
XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(rawText: "transcribed", promptTokens: [1]))
XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(rawText: "[BLANK_AUDIO]", promptTokens: [1]))
XCTAssertFalse(WhisperService.shouldRetryWithoutVocabulary(rawText: "", promptTokens: nil))
}
}
Loading