Merge pull request #25 from luongndcoder/fix/soniox-realtime-speaker-timestamp

fix(stt): Soniox realtime speaker attribution + segment timestamps

Author: luongndcoder

package.json

@@ -1,7 +1,7 @@
 {
   "name": "voicescribe",
   "private": true,
-  "version": "1.2.8",
+  "version": "1.2.9",
   "type": "module",
   "scripts": {
     "dev": "vite",

src-python/stt.py

@@ -763,11 +763,19 @@ def stop(self):
     def results(self):
         """Blocking generator that yields transcript results from Soniox.
 
-        Uses chunk_id-based replace-in-place strategy:
-        - Accumulates final tokens globally
-        - Each event: yields (final + non-final) text with same chunk_id
-        - Frontend replaceLastPartText updates in-place -> smooth real-time
-        - Speaker change -> new chunk_id -> new transcript block
+        Per-token streaming with proper speaker-change handling:
+        - Process each token sequentially; current_speaker updates IMMEDIATELY
+          when a final token from a new speaker arrives (the previous version
+          only updated current_speaker at end-of-event, so tokens mid-event
+          from a different speaker got mis-attributed to the last speaker).
+        - On speaker change, flush the accumulated segment (yielding its full
+          text under the OLD speaker) and start a fresh chunk_id for the new
+          one. Frontend `sameChunk` then produces a new transcript block.
+        - Each segment carries its own `start_ms` / `end_ms` (from the actual
+          Soniox token offsets) so the UI shows accurate time ranges instead
+          of falling back to the recording-timer's current second.
+        - Each event also re-yields the in-progress segment so non-final
+          tokens flow into the UI live (frontend `replaceLastPartText`).
         """
         if not self._session:
             return
@@ -778,8 +786,13 @@ def results(self):
         # Running accumulator of ALL final token texts for current segment
         accumulated_final = []
         accumulated_translation = []  # Translation tokens for current segment
-        current_speaker = 0
+        # Use a sentinel so the FIRST final token can initialize without
+        # triggering a spurious "speaker change" flush against an empty buffer.
+        current_speaker = -1
         current_chunk_id = f"soniox-{int(time.time() * 1000)}-{uuid4().hex[:6]}"
+        # Segment time range (in ms, relative to Soniox stream start)
+        segment_start_ms = None
+        segment_end_ms = None
 
         event_count = 0
         try:
@@ -810,12 +823,11 @@ def results(self):
                 if not event.tokens:
                     continue
 
-                # Separate final and non-final tokens from this event
-                new_final = []
                 non_final = []
-                new_translation = []  # Translation tokens from this event
-                non_final_translation = []  # Non-final translation tokens
-                final_speaker = None  # Track speaker from final tokens only
+                non_final_translation = []
+                # Track the latest end_ms across non-final tokens so the live
+                # display reflects the running edge of the segment.
+                latest_non_final_end_ms = None
 
                 for token in event.tokens:
                     speaker_id = int(getattr(token, "speaker", 0) or 0)
@@ -827,45 +839,76 @@ def results(self):
                     translation_status = getattr(token, "translation_status", "none") or "none"
                     if translation_status == "translation":
                         if token.is_final:
-                            new_translation.append(token_text)
+                            accumulated_translation.append(token_text)
                         else:
                             non_final_translation.append(token_text)
                         continue
 
                     # STT tokens (translation_status: "none" or "original")
+                    tok_start = getattr(token, "start_ms", None)
+                    tok_end = getattr(token, "end_ms", None)
+                    if tok_start is not None:
+                        tok_start = int(tok_start)
+                    if tok_end is not None:
+                        tok_end = int(tok_end)
+
                     if token.is_final:
-                        # Speaker changed — flush current segment, start new
-                        if accumulated_final and speaker_id != current_speaker:
+                        # First-ever final token: just adopt its speaker.
+                        if current_speaker == -1:
+                            current_speaker = speaker_id
+
+                        # Speaker change INSIDE the event: flush whatever's
+                        # been accumulated under the OLD speaker before we
+                        # touch the new token. Without this, tokens from S1
+                        # and S2 within the same event used to collapse onto
+                        # whichever speaker happened to be last.
+                        elif speaker_id != current_speaker and accumulated_final:
                             full_text = "".join(accumulated_final).strip()
                             if full_text:
-                                log.info("[stt:soniox-stream] Speaker change: S%d -> S%d, flushing: '%s...'", current_speaker+1, speaker_id+1, full_text[:50])
+                                log.info(
+                                    "[stt:soniox-stream] Speaker change mid-event: S%d -> S%d, flushing: '%s...'",
+                                    current_speaker + 1, speaker_id + 1, full_text[:50],
+                                )
                                 result = {
                                     "text": full_text,
                                     "is_final": True,
                                     "chunk_id": current_chunk_id,
                                     "speaker": f"Speaker {current_speaker + 1}",
                                     "speaker_id": current_speaker,
                                 }
-                                # Attach accumulated translation
+                                if segment_start_ms is not None:
+                                    result["start_ms"] = segment_start_ms
+                                if segment_end_ms is not None:
+                                    result["end_ms"] = segment_end_ms
                                 tl_text = "".join(accumulated_translation).strip()
                                 if tl_text:
                                     result["translation"] = tl_text
                                 yield result
-                            # Start new segment
+
+                            # Reset for the new speaker's segment
                             accumulated_final = []
                             accumulated_translation = []
+                            segment_start_ms = None
+                            segment_end_ms = None
                             current_chunk_id = f"soniox-{int(time.time() * 1000)}-{uuid4().hex[:6]}"
+                            current_speaker = speaker_id
+
+                        # Update current_speaker even when no flush happened
+                        # (e.g. token's speaker_id matches the existing one,
+                        # or we just initialized from -1)
+                        current_speaker = speaker_id
 
-                        new_final.append(token_text)
-                        final_speaker = speaker_id
+                        # Track segment time range from the actual token offsets
+                        if tok_start is not None and segment_start_ms is None:
+                            segment_start_ms = tok_start
+                        if tok_end is not None:
+                            segment_end_ms = tok_end
+
+                        accumulated_final.append(token_text)
                     else:
                         non_final.append(token_text)
-
-                # Add new tokens to accumulators
-                accumulated_final.extend(new_final)
-                accumulated_translation.extend(new_translation)
-                if final_speaker is not None:
-                    current_speaker = final_speaker
+                        if tok_end is not None:
+                            latest_non_final_end_ms = tok_end
 
                 # Build display text: all final so far + current non-final
                 display_text = "".join(accumulated_final + non_final).strip()
@@ -875,14 +918,30 @@ def results(self):
                 # Build translation text
                 translation_text = "".join(accumulated_translation + non_final_translation).strip()
 
+                # Resolve display end_ms: prefer the live non-final token's
+                # offset so the time range grows in real time while the user
+                # is still speaking. Fall back to the latest final token's
+                # end_ms otherwise.
+                display_end_ms = (
+                    latest_non_final_end_ms if latest_non_final_end_ms is not None
+                    else segment_end_ms
+                )
+
                 # Yield with same chunk_id -> frontend replaceLastPartText
+                speaker_label = (
+                    f"Speaker {current_speaker + 1}" if current_speaker >= 0 else "Speaker 1"
+                )
                 result = {
                     "text": display_text,
                     "is_final": True,
                     "chunk_id": current_chunk_id,
-                    "speaker": f"Speaker {current_speaker + 1}",
-                    "speaker_id": current_speaker,
+                    "speaker": speaker_label,
+                    "speaker_id": max(0, current_speaker),
                 }
+                if segment_start_ms is not None:
+                    result["start_ms"] = segment_start_ms
+                if display_end_ms is not None:
+                    result["end_ms"] = display_end_ms
                 # Only include translation when it has actually changed (reduce frontend re-renders)
                 if translation_text:
                     result["translation"] = translation_text
@@ -892,13 +951,20 @@ def results(self):
             if accumulated_final:
                 full_text = "".join(accumulated_final).strip()
                 if full_text:
+                    speaker_label = (
+                        f"Speaker {current_speaker + 1}" if current_speaker >= 0 else "Speaker 1"
+                    )
                     result = {
                         "text": full_text,
                         "is_final": True,
                         "chunk_id": current_chunk_id,
-                        "speaker": f"Speaker {current_speaker + 1}",
-                        "speaker_id": current_speaker,
+                        "speaker": speaker_label,
+                        "speaker_id": max(0, current_speaker),
                     }
+                    if segment_start_ms is not None:
+                        result["start_ms"] = segment_start_ms
+                    if segment_end_ms is not None:
+                        result["end_ms"] = segment_end_ms
                     tl_text = "".join(accumulated_translation).strip()
                     if tl_text:
                         result["translation"] = tl_text

src-tauri/Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "voicescribe"
-version = "1.2.8"
+version = "1.2.9"
 description = "Scribble - Meeting Minutes"
 authors = ["you"]
 edition = "2021"

src-tauri/Cargo.toml

@@ -1,7 +1,7 @@
 {
   "$schema": "https://schema.tauri.app/config/2",
   "productName": "Scribble",
-  "version": "1.2.8",
+  "version": "1.2.9",
   "identifier": "com.scribble.app",
   "build": {
     "beforeDevCommand": "pnpm dev",

src-tauri/tauri.conf.json

@@ -194,6 +194,17 @@ function handleFinalTranscript(data: SttWsMessage, text: string, ts: string, sta
     useAppStore.getState().setInterimText("");
     useAppStore.getState().setInterimSpeaker(speaker, speakerId);
 
+    // Prefer the backend's token-level offsets (Soniox sends start_ms / end_ms
+    // per segment, which is the truth). Fall back to the recording timer for
+    // STT sources that don't provide ms offsets, with a 3-second look-back on
+    // segment start to roughly match the spoken phrase boundary.
+    const startSec = data.start_ms != null
+        ? Math.floor(data.start_ms / 1000)
+        : Math.max(0, state.seconds - 3);
+    const endSec = data.end_ms != null
+        ? Math.floor(data.end_ms / 1000)
+        : state.seconds;
+
     const lastPart = state.transcriptParts[state.transcriptParts.length - 1];
     const lastChunkIds = new Set<string>();
     if (lastPart?.chunkId) lastChunkIds.add(lastPart.chunkId);
@@ -204,10 +215,10 @@ function handleFinalTranscript(data: SttWsMessage, text: string, ts: string, sta
     const sameSpeaker = Boolean(lastPart) && Number(lastPart.speakerId) === Number(speakerId);
 
     if (sameChunk) {
-        useAppStore.getState().replaceLastPartText(text, String(state.seconds), chunkId || undefined);
+        useAppStore.getState().replaceLastPartText(text, String(endSec), chunkId || undefined);
     } else if (sameSpeaker) {
         // Don't clear interimTranslation here — accumulated translation will update via handleTranslationEvent
-        useAppStore.getState().appendToLastPart(text, String(state.seconds), chunkId || undefined);
+        useAppStore.getState().appendToLastPart(text, String(endSec), chunkId || undefined);
     } else {
         // New speaker — commit pending translation
         const prevTranslation = useAppStore.getState().interimTranslation;
@@ -221,8 +232,8 @@ function handleFinalTranscript(data: SttWsMessage, text: string, ts: string, sta
             speakerId,
             chunkId: chunkId || undefined,
             chunkIds: chunkId ? [chunkId] : undefined,
-            startTime: String(Math.max(0, state.seconds - 3)),
-            endTime: String(state.seconds),
+            startTime: String(startSec),
+            endTime: String(endSec),
             timestamp: ts,
             translation: "",
         });

src/components/recording/use-streaming-stt.ts

@@ -25,6 +25,10 @@ export interface SttWsMessage {
     append?: boolean;
     error?: string;
     segments?: SttSegment[];
+    /** Soniox token-level offsets (ms from stream start). Present on
+     *  real-time STT events when the upstream tokens carry timestamps. */
+    start_ms?: number;
+    end_ms?: number;
 }
 
 export interface DiagnoseResult {