Skip to content

Commit 8e318f5

Browse files
Merge pull request #65 from hoangkien1703/codex/fix-word-pronunciation
Fix silent word pronunciation with speech engine and voice fallback
2 parents b77596f + b29ca98 commit 8e318f5

10 files changed

Lines changed: 596 additions & 35 deletions

File tree

‎PRIVACY.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ DualSub Replay is designed without an application account, analytics SDK, advert
1010
- The app stores the last Browse URL, target language, subtitle text size, and landscape split ratio in local Android preferences.
1111
- Saved words, meanings, subtitle context, video IDs, timestamp ranges, and review schedules are stored in a local SQLite database. They are not sent to an application server. Android's normal app backup may include this database and preferences.
1212
- Optional offline clip downloads send video requests to YouTube and its media hosts using the bundled yt-dlp downloader. Downloads do not copy cookies from the browsing WebView. Clip files are stored privately and excluded from Android backups. Restored cards may require their clips to be downloaded again.
13-
- Pronunciation sends the selected word and language to the device's configured Android text-to-speech engine. That engine may use an online voice according to its own settings and privacy policy.
13+
- Pronunciation sends the selected word and language to the device's preferred Android text-to-speech engine and, if needed, other installed speech engines. It prefers installed offline voices but may try a supported online voice when local speech fails or is unavailable. Online voices process the word according to the speech engine provider's settings and privacy policy.
1414
- If you use the experimental Google/YouTube sign-in flow, authentication occurs inside YouTube's embedded web experience and WebView cookies persist locally. Google may reject embedded-WebView sign-in.
1515

1616
## What the project does not do

‎README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ Online video playback uses the native player in YouTube's mobile webpage. Saved
102102

103103
## Saved words and practice
104104

105-
Tap a word in either subtitle line to hear it, inspect its meaning, and choose **Save word**. Pronunciation uses the tapped word's language and the device's installed speech engine; Android voice availability varies. Automatic pronunciation can be disabled in Settings, and the Pronounce button remains available.
105+
Tap a word in either subtitle line to hear it, inspect its meaning, and choose **Save word**. Pronunciation uses the tapped word's language, trying your preferred Android speech engine and then other installed engines. It prefers installed offline voices and can fall back to a supported network voice. If no voice works, use **Speech settings** to install or select one, then tap **Pronounce** again. Automatic pronunciation can be disabled in Settings, and the Pronounce button remains available.
106106

107107
Each card stores an editable meaning and its original subtitle context. The Online example option replays the saved sentence in the existing YouTube page and stops at its end. A translated word's example contains the original spoken sentence, which may not literally contain the translated word.
108108

‎app/src/androidTest/java/com/kienhoang/dualsubreplay/ui/WordLearningDialogTest.kt‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,4 +57,30 @@ class WordLearningDialogTest {
5757
compose.waitForIdle()
5858
compose.runOnIdle { assertEquals(Pair("từ", false), saved) }
5959
}
60+
61+
@Test
62+
fun missingVoiceOffersSettingsAndAllowsPronunciationRetry() {
63+
var settingsOpened = 0
64+
var spoken = 0
65+
compose.setContent {
66+
DualSubTheme {
67+
WordLearningDialog(
68+
selection = selection,
69+
autoPronounce = false,
70+
onTranslateWord = { "từ" },
71+
onSave = { _, _ -> },
72+
onSpeak = { spoken++ },
73+
speechMessage = "No installed speech engine has a voice for this language.",
74+
onDismiss = {},
75+
onSpeechSettings = { settingsOpened++ },
76+
)
77+
}
78+
}
79+
compose.onNodeWithTag("speech_settings").performScrollTo().performClick()
80+
compose.onNodeWithTag("pronounce_word").performScrollTo().performClick()
81+
compose.runOnIdle {
82+
assertEquals(1, settingsOpened)
83+
assertEquals(1, spoken)
84+
}
85+
}
6086
}
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
package com.kienhoang.dualsubreplay.ui
2+
3+
import android.content.Context
4+
import android.content.Intent
5+
import android.media.AudioAttributes
6+
import android.os.Bundle
7+
import android.os.Handler
8+
import android.os.Looper
9+
import android.provider.Settings
10+
import android.speech.tts.TextToSpeech
11+
import android.speech.tts.UtteranceProgressListener
12+
import kotlinx.coroutines.CompletableDeferred
13+
import java.util.Locale
14+
15+
internal fun installedPronunciationEngines(context: Context): List<String?> {
16+
val preferred = Settings.Secure.getString(context.contentResolver, Settings.Secure.TTS_DEFAULT_SYNTH)
17+
val installed =
18+
context.packageManager
19+
.queryIntentServices(Intent(TextToSpeech.Engine.INTENT_ACTION_TTS_SERVICE), 0)
20+
.map { it.serviceInfo.packageName }
21+
return (listOf(preferred) + installed).distinct()
22+
}
23+
24+
internal class AndroidPronunciationEngine(
25+
context: Context,
26+
engineName: String?,
27+
) : PronunciationEngine {
28+
private val handler = Handler(Looper.getMainLooper())
29+
private val initialized = CompletableDeferred<Boolean>()
30+
private var closed = false
31+
private var nextUtterance = 0L
32+
private var utteranceId: String? = null
33+
private var playback: CompletableDeferred<Boolean>? = null
34+
private val tts =
35+
TextToSpeech(context.applicationContext, { status ->
36+
// onInit may run before the constructor returns. Post before touching tts or UI state.
37+
handler.post { if (!closed) initialized.complete(status == TextToSpeech.SUCCESS) }
38+
}, engineName)
39+
40+
override suspend fun initialize(): Boolean {
41+
if (!initialized.await()) return false
42+
tts.setAudioAttributes(
43+
AudioAttributes
44+
.Builder()
45+
.setUsage(AudioAttributes.USAGE_MEDIA)
46+
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
47+
.build(),
48+
)
49+
tts.setOnUtteranceProgressListener(
50+
object : UtteranceProgressListener() {
51+
override fun onStart(id: String?) = Unit
52+
53+
override fun onDone(id: String?) = complete(id, true)
54+
55+
@Deprecated("Required by UtteranceProgressListener")
56+
override fun onError(id: String?) = complete(id, false)
57+
58+
override fun onError(
59+
id: String?,
60+
errorCode: Int,
61+
) = complete(id, false)
62+
63+
override fun onStop(
64+
id: String?,
65+
interrupted: Boolean,
66+
) = complete(id, false)
67+
},
68+
)
69+
return true
70+
}
71+
72+
private fun complete(
73+
id: String?,
74+
success: Boolean,
75+
) {
76+
handler.post {
77+
if (!closed && id == utteranceId) playback?.complete(success)
78+
}
79+
}
80+
81+
override fun voices(): List<PronunciationVoice> =
82+
tts.voices.orEmpty().map {
83+
PronunciationVoice(
84+
it.name,
85+
it.locale,
86+
it.isNetworkConnectionRequired,
87+
TextToSpeech.Engine.KEY_FEATURE_NOT_INSTALLED !in it.features.orEmpty(),
88+
)
89+
}
90+
91+
override fun selectVoice(voice: PronunciationVoice): Boolean {
92+
val native = tts.voices.orEmpty().firstOrNull { it.name == voice.name } ?: return false
93+
return tts.setVoice(native) == TextToSpeech.SUCCESS
94+
}
95+
96+
override fun selectLanguage(locale: Locale): Boolean = tts.setLanguage(locale) >= TextToSpeech.LANG_AVAILABLE
97+
98+
override suspend fun speak(word: String): Boolean {
99+
val result = CompletableDeferred<Boolean>()
100+
playback = result
101+
val id = "word-${++nextUtterance}"
102+
utteranceId = id
103+
val parameters = Bundle().apply { putFloat(TextToSpeech.Engine.KEY_PARAM_VOLUME, 1f) }
104+
return try {
105+
if (tts.speak(word, TextToSpeech.QUEUE_FLUSH, parameters, id) == TextToSpeech.ERROR) false else result.await()
106+
} finally {
107+
utteranceId = null
108+
playback = null
109+
tts.stop()
110+
}
111+
}
112+
113+
override fun close() {
114+
closed = true
115+
utteranceId = null
116+
playback?.cancel()
117+
initialized.cancel()
118+
handler.removeCallbacksAndMessages(null)
119+
try {
120+
tts.stop()
121+
} finally {
122+
tts.shutdown()
123+
}
124+
}
125+
}

‎app/src/main/java/com/kienhoang/dualsubreplay/ui/DualSubApp.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,7 @@ fun DualSubApp(
192192
onSave = { meaning, online -> viewModel.saveWord(selection, meaning, online); Unit },
193193
onSpeak = { webController.pause(); pronouncer.speak(selection.token.text, selection.wordLanguage) },
194194
speechMessage = pronouncer.message,
195+
onSpeechSettings = if (pronouncer.showSpeechSettings) pronouncer::openSpeechSettings else null,
195196
onDismiss = { viewModel.selectLearningWord(null) },
196197
)
197198
}
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
package com.kienhoang.dualsubreplay.ui
2+
3+
import kotlinx.coroutines.CancellationException
4+
import kotlinx.coroutines.withTimeoutOrNull
5+
import java.util.Locale
6+
7+
internal data class PronunciationVoice(
8+
val name: String,
9+
val locale: Locale,
10+
val network: Boolean = false,
11+
val installed: Boolean = true,
12+
)
13+
14+
internal interface PronunciationEngine {
15+
suspend fun initialize(): Boolean
16+
17+
fun voices(): List<PronunciationVoice>
18+
19+
fun selectVoice(voice: PronunciationVoice): Boolean
20+
21+
fun selectLanguage(locale: Locale): Boolean
22+
23+
suspend fun speak(word: String): Boolean
24+
25+
fun close()
26+
}
27+
28+
internal enum class PronunciationResult {
29+
SPOKEN,
30+
NO_VOICE,
31+
UNAVAILABLE,
32+
PLAYBACK_FAILED,
33+
INVALID_LANGUAGE,
34+
}
35+
36+
internal fun pronunciationLocale(language: String): Locale? =
37+
Locale.forLanguageTag(language.trim().replace('_', '-')).takeIf {
38+
it.language.isNotBlank() && it.language !in setOf("und", "auto")
39+
}
40+
41+
internal fun pronunciationVoices(
42+
voices: List<PronunciationVoice>,
43+
locale: Locale,
44+
): List<PronunciationVoice> {
45+
val matching =
46+
voices
47+
.filter { it.installed && it.locale.language == locale.language }
48+
.filter { locale.script.isBlank() || it.locale.script.isBlank() || it.locale.script == locale.script }
49+
.sortedWith(
50+
compareBy<PronunciationVoice> { it.network }
51+
.thenBy { it.locale != locale }
52+
.thenBy { it.locale.country != locale.country }
53+
.thenBy { it.name },
54+
)
55+
// Keep a network fallback reachable even when an engine lists many offline voices.
56+
return (matching.take(1) + matching.filter { it.network }.take(1) + matching.drop(1))
57+
.distinctBy { it.name }
58+
.take(3)
59+
}
60+
61+
/** Try the user's default engine first, then other installed engines, without changing system settings. */
62+
internal suspend fun pronounceWord(
63+
word: String,
64+
language: String,
65+
engineNames: List<String?>,
66+
createEngine: (String?) -> PronunciationEngine,
67+
): PronunciationResult {
68+
val locale = pronunciationLocale(language) ?: return PronunciationResult.INVALID_LANGUAGE
69+
var initialized = false
70+
var playbackFailed = false
71+
for (name in engineNames.distinct()) {
72+
var engine: PronunciationEngine? = null
73+
try {
74+
val active = createEngine(name).also { engine = it }
75+
if (withTimeoutOrNull(4_000) { active.initialize() } != true) continue
76+
initialized = true
77+
val voices = pronunciationVoices(active.voices(), locale)
78+
if (voices.isEmpty()) {
79+
// Some older engines implement setLanguage but do not expose a voice catalog.
80+
if (!active.selectLanguage(locale)) continue
81+
playbackFailed = true
82+
if (withTimeoutOrNull(10_000) { active.speak(word) } == true) return PronunciationResult.SPOKEN
83+
} else {
84+
for (voice in voices) {
85+
if (!active.selectVoice(voice)) continue
86+
playbackFailed = true
87+
if (withTimeoutOrNull(10_000) { active.speak(word) } == true) return PronunciationResult.SPOKEN
88+
}
89+
}
90+
} catch (cancel: CancellationException) {
91+
throw cancel
92+
} catch (_: Exception) {
93+
// A broken vendor engine must not prevent trying the next installed engine.
94+
playbackFailed = true
95+
} finally {
96+
runCatching { engine?.close() }
97+
}
98+
}
99+
return when {
100+
playbackFailed -> PronunciationResult.PLAYBACK_FAILED
101+
initialized -> PronunciationResult.NO_VOICE
102+
else -> PronunciationResult.UNAVAILABLE
103+
}
104+
}

‎app/src/main/java/com/kienhoang/dualsubreplay/ui/SavedWordsScreen.kt‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,14 @@ internal fun SavedWordsScreen(
126126
selected.reading?.let { Text(it) }
127127
TextButton(onClick = { onPause(); pronouncer.speak(selected.word, selected.wordLanguage) }) { Text("Pronounce") }
128128
pronouncer.message?.let { Text(it) }
129+
if (pronouncer.showSpeechSettings) {
130+
TextButton(
131+
onClick = pronouncer::openSpeechSettings,
132+
modifier = Modifier.testTag("speech_settings"),
133+
) {
134+
Text("Speech settings")
135+
}
136+
}
129137
if (practice && !revealed) {
130138
Button(onClick = { revealed = true }, modifier = Modifier.testTag("reveal_meaning")) { Text("Show meaning") }
131139
} else {

‎app/src/main/java/com/kienhoang/dualsubreplay/ui/WordLearningSheet.kt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ internal fun WordLearningDialog(
2323
speechMessage: String?,
2424
onDismiss: () -> Unit,
2525
existingWord: com.kienhoang.dualsubreplay.data.SavedWord? = null,
26+
onSpeechSettings: (() -> Unit)? = null,
2627
) {
2728
var meaning by remember(selection) { mutableStateOf(existingWord?.meaning.orEmpty()) }
2829
var loading by remember(selection) { mutableStateOf(true) }
@@ -50,6 +51,9 @@ internal fun WordLearningDialog(
5051
selection.token.reading?.takeIf { it.isNotBlank() }?.let { Text(it) }
5152
TextButton(onClick = onSpeak, modifier = Modifier.testTag("pronounce_word")) { Text("Pronounce") }
5253
speechMessage?.let { Text(it) }
54+
onSpeechSettings?.let { open ->
55+
TextButton(onClick = open, modifier = Modifier.testTag("speech_settings")) { Text("Speech settings") }
56+
}
5357
if (loading) LinearProgressIndicator(Modifier.fillMaxWidth())
5458
OutlinedTextField(meaning, { meaning = it; saved = false }, label = { Text("Meaning (${selection.meaningLanguage})") },
5559
enabled = !loading && !saving, modifier = Modifier.testTag("word_meaning"))

0 commit comments

Comments
 (0)