Merge pull request #71 from hoangkien1703/codex/fix-caption-highlight-regression

Prevent caption highlights from jumping backward

Author: hoangkien1703

.github/workflows/release-on-main.yml

@@ -162,23 +162,21 @@ jobs:
 
             Preview builds install separately as **DualSub Replay Preview**, so installing an official release does not require removing the preview app.
 
-            ## v1.0.3 highlights
+            ## v1.0.4 highlights
 
-            - **Adjustable portrait panel**: Use the Higher–Lower slider in More settings to move the portrait transcript panel from 20% higher to 20% lower (#70).
-            - **Improved default placement**: The panel now starts 1% lower to leave slightly more of the video visible (#70).
-            - **Safe resizing**: Raising the panel preserves its usual height, while lowering it reduces the visible subtitle area without moving content behind the navigation bar (#70).
-            - **Saved position**: The selected position persists across restarts, video changes, panel hide/show, and rotation, with dedicated and global reset support (#70).
-            - **Reachable subtitles**: The full bilingual timeline remains scrollable at every position, including replay controls on the final row (#70).
+            - **Steady caption highlights**: Captions no longer jump to a previous sentence when live caption tracking briefly falls back to transcript timing or the WebView clock corrects backward (#71).
+            - **Intentional rewinds still work**: Playback sessions reset highlight continuity after seeks, including small rewinds (#71).
+            - **Natural caption gaps**: Highlighting still clears between captions and resets when the transcript is replaced (#71).
 
             ## Validation
 
-            - PR #70 passed formatting, complexity, unit tests, lint, debug and Android-test builds, optimized release checks, and API 36 managed-device tests.
-            - Regression coverage includes panel geometry limits, small windows, invalid saved values, individual and global resets, persistence, settings interaction, and final-row reachability.
+            - PR #71 passed formatting, complexity, unit tests, lint, debug and Android-test builds, optimized release checks, and API 36 managed-device tests.
+            - Regression coverage includes clock corrections, live-to-timestamp fallback, delayed live captions, regressive words, caption gaps, and small rewinds.
             - The official APK is built with production signing and verified by this release workflow.
 
             ## Known limitations
 
-            - The panel-position control applies to the portrait transcript panel. Landscape layouts and compact/fullscreen captions retain their existing positioning behavior.
+            - Spoken-word tracking on auto-generated captions depends on YouTube's webpage and may vary by video or device. Transcript timing remains available as a fallback.
 
             The `.sha256` file can be used to verify the APK download.
           files: |

app/build.gradle.kts

@@ -3,8 +3,8 @@ plugins {
     id("org.jetbrains.kotlin.plugin.compose")
 }
 
-val appVersionCode = 34
-val appVersionName = "1.0.3"
+val appVersionCode = 35
+val appVersionName = "1.0.4"
 val releaseStoreFile = providers.environmentVariable("ANDROID_RELEASE_STORE_FILE").orNull
 val releaseStorePassword = providers.environmentVariable("ANDROID_RELEASE_STORE_PASSWORD").orNull
 val releaseKeyAlias = providers.environmentVariable("ANDROID_RELEASE_KEY_ALIAS").orNull

app/src/androidTest/java/com/kienhoang/dualsubreplay/ui/RestoredHighlightPlaybackTest.kt

@@ -116,8 +116,16 @@ class RestoredHighlightPlaybackTest {
                     }
                     listOf(Configuration.ORIENTATION_PORTRAIT, Configuration.ORIENTATION_LANDSCAPE).forEach { value ->
                         compose.runOnIdle { orientation.value = value }
-                        listOf(0.1f to "One", 1.1f to "two", 2.1f to "three.", 0.1f to "One").forEach { (second, word) ->
-                            compose.runOnIdle { vm.onWebPlaybackSecond("abcdefghijk", second, null) }
+                        listOf(
+                            Triple(0.1f, "One", "steady"),
+                            Triple(1.1f, "two", "steady"),
+                            Triple(2.1f, "three.", "steady"),
+                            Triple(1.9f, "two", "small-rewind"),
+                            Triple(0.1f, "One", "large-rewind"),
+                        ).forEach { (second, word, sessionId) ->
+                            compose.runOnIdle {
+                                vm.onWebPlaybackSecond("abcdefghijk", second, null, sessionId)
+                            }
                             assertSpokenWord(word)
                         }
                     }

app/src/main/java/com/kienhoang/dualsubreplay/ui/AppViewModel.kt

@@ -276,7 +276,9 @@ class AppViewModel internal constructor(
     private var playbackKnown = false
     private var appVisible = true
     private val liveCaptionTracker = LiveCaptionTracker()
+    private val captionHighlightResolver = CaptionHighlightResolver()
     private var liveCaptionProgress: LiveCaptionProgress? = null
+    private var playbackSessionId: String? = null
     private val liveTranslationGate = LiveTranslationGate()
     private var liveTranslationJob: Job? = null
     private var rejectedLiveRevision: Long? = null
@@ -480,15 +482,23 @@ class AppViewModel internal constructor(
         videoId: String,
         second: Float,
         liveCaption: LiveCaptionSample? = null,
+        sessionId: String = "",
     ) {
         val current = _state.value
         if (current.activeVideoId != videoId || !second.isFinite()) return
         val timeMs = (second.coerceAtLeast(0f) * 1_000).toLong()
+        val normalizedSessionId = sessionId.takeIf(String::isNotBlank)
+        val sessionChanged =
+            normalizedSessionId != null &&
+                playbackSessionId?.let { it != normalizedSessionId } == true
+        if (normalizedSessionId != null) playbackSessionId = normalizedSessionId
         val seek =
-            timeMs + LIVE_CAPTION_BACKWARD_SEEK_RESET_MS < latestPlaybackSecondMs ||
+            sessionChanged ||
+                timeMs + LIVE_CAPTION_BACKWARD_SEEK_RESET_MS < latestPlaybackSecondMs ||
                 timeMs > latestPlaybackSecondMs + 2_000L
         if (seek) {
             liveCaptionTracker.reset()
+            captionHighlightResolver.reset()
             liveCaptionProgress = null
         }
         latestPlaybackSecondMs = timeMs
@@ -542,18 +552,15 @@ class AppViewModel internal constructor(
                 null
             }
         val timedWordIndex = activeWordIndex(current.segments, timedIndex, timeMs)
-        val timedPosition =
-            timedIndex.takeIf { it >= 0 && timedWordIndex >= 0 }?.let {
-                KaraokePosition(it, timedWordIndex)
-            }
         val position =
-            effectiveKaraokePosition(
-                current.generatedCaptions,
-                current.wordHighlightEnabled,
-                timedPosition,
-                livePosition,
+            captionHighlightResolver.resolve(
+                generatedCaptions = current.generatedCaptions,
+                wordHighlightEnabled = current.wordHighlightEnabled,
+                timedSegmentIndex = timedIndex,
+                timedWordIndex = timedWordIndex,
+                livePosition = livePosition,
             )
-        val index = position?.segmentIndex ?: timedIndex
+        val index = position?.segmentIndex ?: -1
         val wordIndex = position?.wordIndex ?: -1
         if (index != current.currentIndex || wordIndex != current.activeWordIndex) {
             _state.update { it.copy(currentIndex = index, activeWordIndex = wordIndex) }
@@ -741,6 +748,7 @@ class AppViewModel internal constructor(
     fun setWordHighlightEnabled(enabled: Boolean) {
         preferences.edit().putBoolean(WORD_HIGHLIGHT_ENABLED_PREFERENCE, enabled).apply()
         liveCaptionTracker.reset()
+        captionHighlightResolver.reset()
         liveCaptionProgress = null
         _state.update { it.copy(wordHighlightEnabled = enabled, activeWordIndex = -1) }
     }
@@ -755,6 +763,7 @@ class AppViewModel internal constructor(
         preferences.edit().putString(CAPTION_FORMAT_PREFERENCE, format.storageValue).apply()
         _state.update { it.copy(captionFormat = format) }
         liveCaptionTracker.reset()
+        captionHighlightResolver.reset()
         liveCaptionProgress = null
     }
 
@@ -873,6 +882,7 @@ class AppViewModel internal constructor(
         _state.update { it.copy(autoPronounce = true) }
         latestPlaybackSecondMs = 0L
         liveCaptionTracker.reset()
+        captionHighlightResolver.reset()
         liveCaptionProgress = null
         _state.update { current ->
             current.copy(
@@ -924,7 +934,9 @@ class AppViewModel internal constructor(
         liveTranslationGate.reset()
         latestPlaybackSecondMs = 0L
         liveCaptionTracker.reset()
+        captionHighlightResolver.reset()
         liveCaptionProgress = null
+        playbackSessionId = null
         playbackKnown = false
         playbackRequests.value = CaptionPlaybackRequest()
         _state.update {
@@ -967,6 +979,8 @@ class AppViewModel internal constructor(
         if (shouldResetPlaybackClock(_state.value.activeVideoId, videoId)) {
             latestPlaybackSecondMs = 0L
             playbackKnown = false
+            playbackSessionId = null
+            captionHighlightResolver.reset()
         }
         playbackRequests.value = CaptionPlaybackRequest()
         _state.update {
@@ -1077,6 +1091,7 @@ class AppViewModel internal constructor(
     ) {
         var displayStore: SubtitleStore? = null
         liveCaptionTracker.reset()
+        captionHighlightResolver.reset()
         _state.update {
             it.copy(
                 segments = emptyList(),
@@ -1145,6 +1160,7 @@ class AppViewModel internal constructor(
                 ?.id != rows.firstOrNull()?.id
         ) {
             liveCaptionTracker.reset()
+            captionHighlightResolver.reset()
         }
         _state.update { current ->
             if (!isCurrentLoad(current, videoId, generation)) return@update current

app/src/main/java/com/kienhoang/dualsubreplay/ui/CaptionPlaybackClock.kt

@@ -8,6 +8,8 @@ internal class CaptionPlaybackClock {
     private var sampledAt = 0L
     private var lastWallSample = 0L
     private var positionMs = 0L
+    private var lastEmittedPositionMs: Long? = null
+    private var lastEmittedSessionId: String? = null
     private val retiredSessions = ArrayDeque<String>()
 
     fun accept(
@@ -43,7 +45,14 @@ internal class CaptionPlaybackClock {
         // Do not march through words when the renderer stops reporting playback.
         if (age !in 0..PLAYBACK_CLOCK_STALE_MS || sample.seeking) return null
         val advance = if (sample.paused || sample.buffering) 0 else (age * sample.playbackRate).toLong()
-        return positionMs + advance
+        val projectedPositionMs = positionMs + advance
+        if (lastEmittedSessionId != sample.sessionId) {
+            lastEmittedSessionId = sample.sessionId
+            lastEmittedPositionMs = null
+        }
+        return maxOf(projectedPositionMs, lastEmittedPositionMs ?: projectedPositionMs).also {
+            lastEmittedPositionMs = it
+        }
     }
 
     fun sample(): WebPlaybackSnapshot? = snapshot

app/src/main/java/com/kienhoang/dualsubreplay/ui/DualSubApp.kt

@@ -233,7 +233,7 @@ private fun DualSubExperience(
     fullscreenLearningOverlay: (@Composable BoxScope.() -> Unit)?,
     onNavigationVisibilityChange: (Boolean) -> Unit,
     onPageChanged: (String) -> Unit,
-    onPlaybackSecond: (String, Float, LiveCaptionSample?) -> Unit,
+    onPlaybackSecond: (String, Float, LiveCaptionSample?, String) -> Unit,
     onPlaybackPaused: (String, Boolean) -> Unit,
     onShowSubtitles: () -> Unit,
     onHideSubtitles: () -> Unit,

app/src/main/java/com/kienhoang/dualsubreplay/ui/KaraokeTiming.kt

@@ -42,6 +42,48 @@ internal fun effectiveKaraokePosition(
     else -> timedPosition
 }
 
+internal data class CaptionHighlightPosition(
+    val segmentIndex: Int,
+    val wordIndex: Int,
+) : Comparable<CaptionHighlightPosition> {
+    override fun compareTo(other: CaptionHighlightPosition): Int =
+        compareValuesBy(this, other, CaptionHighlightPosition::segmentIndex, CaptionHighlightPosition::wordIndex)
+}
+
+/** Keeps timestamp and live-caption arbitration monotonic between real playback discontinuities. */
+internal class CaptionHighlightResolver {
+    private var lastPosition: CaptionHighlightPosition? = null
+
+    fun reset() {
+        lastPosition = null
+    }
+
+    fun resolve(
+        generatedCaptions: Boolean,
+        wordHighlightEnabled: Boolean,
+        timedSegmentIndex: Int,
+        timedWordIndex: Int,
+        livePosition: KaraokePosition?,
+    ): CaptionHighlightPosition? {
+        val timedPosition =
+            timedSegmentIndex.takeIf { it >= 0 }?.let {
+                CaptionHighlightPosition(
+                    segmentIndex = it,
+                    wordIndex = timedWordIndex.takeIf { wordHighlightEnabled && it >= 0 } ?: -1,
+                )
+            }
+        val selected =
+            when {
+                wordHighlightEnabled && generatedCaptions && livePosition != null ->
+                    CaptionHighlightPosition(livePosition.segmentIndex, livePosition.wordIndex)
+                else -> timedPosition
+            } ?: return null
+        val resolved = lastPosition?.takeIf { selected < it } ?: selected
+        lastPosition = resolved
+        return resolved
+    }
+}
+
 internal data class LiveCaptionProgress(
     val text: String,
     val tokens: List<String>,

app/src/main/java/com/kienhoang/dualsubreplay/ui/YouTubeBrowserScreen.kt

@@ -261,7 +261,7 @@ internal fun SingleYouTubePage(
     navigationRequestId: Long,
     controller: YouTubeWebController,
     onPageChanged: (String) -> Unit,
-    onPlaybackSecond: (videoId: String, second: Float, liveCaption: LiveCaptionSample?) -> Unit,
+    onPlaybackSecond: (videoId: String, second: Float, liveCaption: LiveCaptionSample?, sessionId: String) -> Unit,
     onPlaybackPaused: (String, Boolean) -> Unit = { _, _ -> },
     liveCaptionCaptureEnabled: Boolean = false,
     suppressPageCaptions: Boolean = false,
@@ -527,7 +527,12 @@ internal fun SingleYouTubePage(
                     val selection = snapshot?.url?.takeIf { it == webView.url }?.let(::browseVideoSelection)
                     if (selection != null && position != null) {
                         currentOnPlaybackPaused(selection.videoId, snapshot.paused)
-                        currentOnPlaybackSecond(selection.videoId, position / 1000f, snapshot.liveCaption)
+                        currentOnPlaybackSecond(
+                            selection.videoId,
+                            position / 1000f,
+                            snapshot.liveCaption,
+                            snapshot.sessionId,
+                        )
                     }
                     delay(33)
                 }

app/src/test/java/com/kienhoang/dualsubreplay/ui/CaptionPlaybackClockTest.kt

@@ -46,6 +46,19 @@ class CaptionPlaybackClockTest {
         assertEquals(2033L, clock.position(333))
     }
 
+    @Test fun smallClockCorrectionsCannotMoveHighlightTimeBackwardWithinSession() {
+        val clock = CaptionPlaybackClock()
+        assertTrue(clock.accept(sample(10f, 1000, "playing"), 100, 100, 1000))
+        assertEquals(10099L, clock.position(199))
+
+        assertTrue(clock.accept(sample(10.04f, 1100, "playing"), 200, 200, 1100))
+        assertEquals(10099L, clock.position(200))
+        assertEquals(10100L, clock.position(260))
+
+        assertTrue(clock.accept(sample(2f, 1200, "seek"), 300, 300, 1200))
+        assertEquals(2000L, clock.position(300))
+    }
+
     @Test fun controlledSteadyPlaybackAddsLessThan150msAcrossPresentationRestarts() {
         repeat(3) {
             val clock = CaptionPlaybackClock()

app/src/test/java/com/kienhoang/dualsubreplay/ui/KaraokeTimingTest.kt

@@ -22,6 +22,51 @@ class KaraokeTimingTest {
         assertNull(effectiveKaraokePosition(true, false, timed, live))
     }
 
+    @Test
+    fun liveToTimestampFallbackCannotReturnToPreviousSentence() {
+        val resolver = CaptionHighlightResolver()
+        assertEquals(
+            CaptionHighlightPosition(2, 0),
+            resolver.resolve(true, true, 1, 3, KaraokePosition(2, 0)),
+        )
+        assertEquals(
+            CaptionHighlightPosition(2, 0),
+            resolver.resolve(true, true, 1, 3, null),
+        )
+    }
+
+    @Test
+    fun delayedLiveCaptionAndRegressiveWordMatchCannotMoveHighlightBackward() {
+        val resolver = CaptionHighlightResolver()
+        assertEquals(
+            CaptionHighlightPosition(3, 2),
+            resolver.resolve(true, true, 3, 2, null),
+        )
+        assertEquals(
+            CaptionHighlightPosition(3, 2),
+            resolver.resolve(true, true, 3, 3, KaraokePosition(2, 4)),
+        )
+        assertEquals(
+            CaptionHighlightPosition(3, 2),
+            resolver.resolve(true, true, 3, 1, KaraokePosition(3, 1)),
+        )
+    }
+
+    @Test
+    fun genuineGapClearsHighlightAndResetAllowsIntentionalReplay() {
+        val resolver = CaptionHighlightResolver()
+        assertEquals(
+            CaptionHighlightPosition(4, 1),
+            resolver.resolve(false, true, 4, 1, null),
+        )
+        assertNull(resolver.resolve(false, true, -1, -1, null))
+        resolver.reset()
+        assertEquals(
+            CaptionHighlightPosition(1, 0),
+            resolver.resolve(false, true, 1, 0, null),
+        )
+    }
+
     @Test
     fun rollingCaptionProgressNeverFlashesBackToTheFirstWord() {
         val first = reconcileLiveCaptionProgress(