-
Notifications
You must be signed in to change notification settings - Fork 27
Expand file tree
/
Copy pathwebsocket_controller.py
More file actions
1716 lines (1467 loc) · 63.8 KB
/
Copy pathwebsocket_controller.py
File metadata and controls
1716 lines (1467 loc) · 63.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""
WebSocket controller for Chronicle backend.
This module handles WebSocket connections for audio streaming.
"""
import asyncio
import concurrent.futures
import json
import logging
import os
import time
import uuid
from functools import partial
from typing import Optional
import redis.asyncio as redis
from fastapi import Query, WebSocket, WebSocketDisconnect
from friend_lite.decoder import OmiOpusDecoder
from starlette.websockets import WebSocketState
from advanced_omi_backend.auth import websocket_auth
from advanced_omi_backend.client_manager import generate_client_id, get_client_manager
from advanced_omi_backend.constants import (
OMI_CHANNELS,
OMI_SAMPLE_RATE,
OMI_SAMPLE_WIDTH,
)
from advanced_omi_backend.controllers.session_controller import mark_session_complete
from advanced_omi_backend.services.audio_stream import AudioStreamProducer
from advanced_omi_backend.services.audio_stream.producer import (
get_audio_stream_producer,
)
from advanced_omi_backend.utils.omi_codec_utils import is_opus_header_stripped
# Thread pool executors for audio decoding
_DEC_IO_EXECUTOR = concurrent.futures.ThreadPoolExecutor(
max_workers=os.cpu_count() or 4,
thread_name_prefix="opus_io",
)
# Logging setup
logger = logging.getLogger(__name__)
application_logger = logging.getLogger("audio_processing")
# Track pending WebSocket connections to prevent race conditions
pending_connections: set[str] = set()
async def subscribe_to_interim_results(websocket: WebSocket, session_id: str) -> None:
"""
Subscribe to interim transcription results from Redis Pub/Sub and forward to client WebSocket.
Runs as background task during WebSocket connection. Listens for interim and final
transcription results published by the Deepgram streaming consumer and forwards them
to the connected client for real-time transcript display.
Args:
websocket: Connected WebSocket client
session_id: Session ID (client_id) to subscribe to
Note:
This task runs continuously until the WebSocket disconnects or the task is cancelled.
Results are published to Redis Pub/Sub channel: transcription:interim:{session_id}
"""
redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0")
try:
# Create Redis client for Pub/Sub
redis_client = await redis.from_url(redis_url, decode_responses=True)
# Create Pub/Sub instance
pubsub = redis_client.pubsub()
# Subscribe to interim results channel for this session
channel = f"transcription:interim:{session_id}"
await pubsub.subscribe(channel)
logger.info(f"📢 Subscribed to interim results channel: {channel}")
# Listen for messages
while True:
try:
message = await pubsub.get_message(
ignore_subscribe_messages=True, timeout=1.0
)
if message and message["type"] == "message":
# Parse result data
try:
result_data = json.loads(message["data"])
# Forward to client WebSocket
await websocket.send_json(
{"type": "interim_transcript", "data": result_data}
)
# Log for debugging
is_final = result_data.get("is_final", False)
text_preview = result_data.get("text", "")[:50]
result_type = "FINAL" if is_final else "interim"
logger.debug(
f"✉️ Forwarded {result_type} result to client {session_id}: {text_preview}..."
)
except json.JSONDecodeError as e:
logger.error(f"Failed to parse interim result JSON: {e}")
except Exception as send_error:
logger.error(
f"Failed to send interim result to client {session_id}: {send_error}"
)
# WebSocket might be closed, exit loop
break
except asyncio.TimeoutError:
# No message received, continue waiting
continue
except asyncio.CancelledError:
logger.info(
f"Interim results subscriber cancelled for session {session_id}"
)
break
except Exception as e:
logger.error(
f"Error in interim results subscriber for {session_id}: {e}",
exc_info=True,
)
break
except Exception as e:
logger.error(
f"Failed to initialize interim results subscriber for {session_id}: {e}",
exc_info=True,
)
finally:
try:
# Unsubscribe and close connections
await pubsub.unsubscribe(channel)
await pubsub.close()
await redis_client.aclose()
logger.info(f"🔕 Unsubscribed from interim results channel: {channel}")
except Exception as cleanup_error:
logger.error(
f"Error cleaning up interim results subscriber: {cleanup_error}"
)
async def parse_wyoming_protocol(ws: WebSocket) -> tuple[dict, Optional[bytes]]:
"""Parse Wyoming protocol: JSON header line followed by optional binary payload.
Returns:
Tuple of (header_dict, payload_bytes or None)
"""
# Read data from WebSocket
logger.debug(f"parse_wyoming_protocol: About to call ws.receive()")
message = await ws.receive()
logger.debug(
f"parse_wyoming_protocol: Received message with keys: {message.keys() if message else 'None'}"
)
# Handle WebSocket close frame
if "type" in message and message["type"] == "websocket.disconnect":
# This is a normal WebSocket close event
code = message.get("code", 1000)
reason = message.get("reason", "")
logger.info(
f"📴 WebSocket disconnect received in parse_wyoming_protocol. Code: {code}, Reason: {reason}"
)
raise WebSocketDisconnect(code=code, reason=reason)
# Handle text message (JSON header)
if "text" in message:
header_text = message["text"]
# Wyoming protocol uses newline-terminated JSON
if not header_text.endswith("\n"):
header_text += "\n"
# Parse JSON header
json_line = header_text.strip()
header = json.loads(json_line)
# If payload is expected, read binary data
payload = None
payload_length = header.get("payload_length")
if payload_length is not None and payload_length > 0:
payload_msg = await ws.receive()
if "bytes" in payload_msg:
payload = payload_msg["bytes"]
else:
logger.warning(f"Expected binary payload but got: {payload_msg.keys()}")
return header, payload
# Handle binary message (invalid - Wyoming protocol requires JSONL headers)
elif "bytes" in message:
raise ValueError(
"Raw binary messages not supported - Wyoming protocol requires JSONL headers"
)
else:
raise ValueError(f"Unexpected WebSocket message type: {message.keys()}")
async def create_client_state(client_id: str, user, device_name: Optional[str] = None):
"""Create and register a new client state."""
# Get client manager
client_manager = get_client_manager()
# Directory where WAV chunks are written
from pathlib import Path
CHUNK_DIR = Path(
"./audio_chunks"
) # This will be mounted to ./data/audio_chunks by Docker
# Use ClientManager for atomic client creation and registration
client_state = client_manager.create_client(
client_id, CHUNK_DIR, user.user_id, user.email
)
# Also track in persistent mapping (for database queries + cross-container Redis)
from advanced_omi_backend.client_manager import track_client_user_relationship_async
await track_client_user_relationship_async(client_id, user.user_id)
# Register client in user model (persistent)
from advanced_omi_backend.users import register_client_to_user
await register_client_to_user(user, client_id, device_name)
return client_state
async def cleanup_client_state(client_id: str):
"""
Clean up and remove client state, marking session complete.
Note: We do NOT cancel the speech detection job here because:
1. The job needs to process all audio data that was already sent
2. If speech was detected, it should create a conversation
3. The job will complete naturally when it sees session status = "finalizing"
4. The job has a grace period (15s) to wait for final transcription
5. RQ's job_timeout (24h) prevents jobs from hanging forever
"""
# Note: Previously we cancelled the speech detection job here, but this prevented
# conversations from being created when WebSocket disconnects mid-recording.
# The speech detection job now monitors session status and completes naturally.
import redis.asyncio as redis
logger.info(
f"🔄 Letting speech detection job complete naturally for client {client_id} (if running)"
)
# Mark all active sessions for this client as complete AND delete Redis streams
try:
# Get async Redis client
redis_url = os.getenv("REDIS_URL", "redis://localhost:6379/0")
async_redis = redis.from_url(redis_url, decode_responses=False)
# Get audio stream producer for finalization
from advanced_omi_backend.services.audio_stream.producer import (
get_audio_stream_producer,
)
audio_stream_producer = get_audio_stream_producer()
# Find all session keys for this client and mark them complete
pattern = f"audio:session:*"
cursor = 0
sessions_closed = 0
while True:
cursor, keys = await async_redis.scan(cursor, match=pattern, count=100)
for key in keys:
# Check if this session belongs to this client
client_id_bytes = await async_redis.hget(key, "client_id")
if client_id_bytes and client_id_bytes.decode() == client_id:
session_id = key.decode().replace("audio:session:", "")
# Check session status
status_bytes = await async_redis.hget(key, "status")
status = status_bytes.decode() if status_bytes else None
# If session is still active, finalize it first (sets status + completion_reason atomically)
if status in ["active", None]:
logger.info(
f"📊 Finalizing active session {session_id[:12]} due to WebSocket disconnect"
)
await audio_stream_producer.finalize_session(
session_id, completion_reason="websocket_disconnect"
)
# Mark session as complete (WebSocket disconnected)
await mark_session_complete(
async_redis, session_id, "websocket_disconnect"
)
sessions_closed += 1
if cursor == 0:
break
if sessions_closed > 0:
logger.info(
f"✅ Closed {sessions_closed} active session(s) for client {client_id}"
)
# Set TTL on Redis Streams for this client (allows consumer groups to finish processing)
stream_pattern = f"audio:stream:{client_id}"
stream_key = await async_redis.exists(stream_pattern)
if stream_key:
# Check how many messages are in the stream
stream_length = await async_redis.xlen(stream_pattern)
# Check for pending messages in consumer groups
pending_count = 0
try:
# Check streaming-transcription consumer group for pending messages
pending_info = await async_redis.xpending(
stream_pattern, "streaming-transcription"
)
if pending_info:
pending_count = pending_info.get("pending", 0)
except Exception as e:
# Consumer group might not exist yet - that's ok
logger.debug(f"No consumer group for {stream_pattern}: {e}")
if stream_length > 0 or pending_count > 0:
logger.warning(
f"⚠️ Closing {stream_pattern} with unprocessed data: "
f"{stream_length} messages in stream, {pending_count} pending in consumer group"
)
await async_redis.expire(
stream_pattern, 60
) # 60 second TTL for consumer group fan-out
logger.info(f"⏰ Set 60s TTL on Redis stream: {stream_pattern}")
else:
logger.debug(f"No Redis stream found for client {client_id}")
await async_redis.close()
except Exception as session_error:
logger.warning(
f"⚠️ Error marking sessions complete for client {client_id}: {session_error}"
)
# Use ClientManager for atomic client removal with cleanup
client_manager = get_client_manager()
removed = await client_manager.remove_client_with_cleanup(client_id)
if removed:
logger.info(f"Client {client_id} cleaned up successfully")
else:
logger.warning(f"Client {client_id} was not found for cleanup")
# Shared helper functions for WebSocket handlers
async def _setup_websocket_connection(
ws: WebSocket,
token: Optional[str],
device_name: Optional[str],
pending_client_id: str,
connection_type: str,
) -> tuple[Optional[str], Optional[object], Optional[object]]:
"""
Setup WebSocket connection: accept, authenticate, create client state.
Args:
ws: WebSocket connection
token: JWT authentication token
device_name: Optional device name for client ID
pending_client_id: Temporary tracking ID
connection_type: "OMI" or "PCM" for logging
Returns:
tuple: (client_id, client_state, user) or (None, None, None) on failure
"""
# Accept WebSocket first (required before any send/close operations)
await ws.accept()
# Authenticate user after accepting connection
user = await websocket_auth(ws, token)
if not user:
# Send error message to client before closing
try:
error_msg = (
json.dumps(
{
"type": "error",
"error": "authentication_failed",
"message": "Authentication failed. Please log in again and ensure your token is valid.",
"code": 1008,
}
)
+ "\n"
)
await ws.send_text(error_msg)
application_logger.info("Sent authentication error message to client")
except Exception as send_error:
application_logger.warning(f"Failed to send error message: {send_error}")
# Close connection with appropriate code
await ws.close(code=1008, reason="Authentication failed")
return None, None, None
# Generate proper client_id using user and device_name
client_id = generate_client_id(user, device_name)
# Remove from pending now that we have real client_id
pending_connections.discard(pending_client_id)
application_logger.info(
f"🔌 {connection_type} WebSocket connection accepted - User: {user.user_id} ({user.email}), Client: {client_id}"
)
# Send ready message to confirm connection is established
try:
ready_msg = (
json.dumps({"type": "ready", "message": "WebSocket connection established"})
+ "\n"
)
await ws.send_text(ready_msg)
application_logger.debug(f"✅ Sent ready message to {client_id}")
except Exception as e:
application_logger.error(f"Failed to send ready message to {client_id}: {e}")
# Create client state
client_state = await create_client_state(client_id, user, device_name)
return client_id, client_state, user
async def _initialize_streaming_session(
client_state,
audio_stream_producer,
user_id: str,
user_email: str,
client_id: str,
audio_format: dict,
websocket: Optional[WebSocket] = None,
) -> Optional[asyncio.Task]:
"""
Initialize streaming session with Redis and enqueue processing jobs.
Args:
client_state: Client state object
audio_stream_producer: Audio stream producer instance
user_id: User ID
user_email: User email
client_id: Client ID
audio_format: Audio format dict from audio-start event
websocket: Optional WebSocket connection to launch interim results subscriber
Returns:
Interim results subscriber task if websocket provided and session initialized, None otherwise
"""
application_logger.info(
f"🔴 BACKEND: _initialize_streaming_session called for {client_id}"
)
if hasattr(client_state, "stream_session_id"):
application_logger.debug(f"Session already initialized for {client_id}")
return None
# Initialize stream session - use client_id as session_id for predictable lookup
# All other session metadata goes to Redis (single source of truth)
client_state.stream_session_id = client_state.client_id
application_logger.info(
f"🆔 Created stream session: {client_state.stream_session_id}"
)
# Determine transcription provider from config.yml
from advanced_omi_backend.model_registry import get_models_registry
registry = get_models_registry()
if not registry:
raise ValueError(
"config.yml not found - cannot determine transcription provider"
)
stt_model = registry.get_default("stt")
if not stt_model:
raise ValueError("No default STT model configured in config.yml (defaults.stt)")
# Use model_provider for session tracking (generic, not validated against hardcoded list)
provider = (
stt_model.model_provider.lower() if stt_model.model_provider else stt_model.name
)
application_logger.info(
f"📋 Using STT provider: {provider} (model: {stt_model.name})"
)
# Initialize session tracking in Redis (SINGLE SOURCE OF TRUTH for session metadata)
# This includes user_email, connection info, audio format, chunk counters, job IDs, etc.
connection_id = f"ws_{client_id}_{int(time.time())}"
await audio_stream_producer.init_session(
session_id=client_state.stream_session_id,
user_id=user_id,
client_id=client_id,
user_email=user_email,
connection_id=connection_id,
mode="streaming",
provider=provider,
)
# Store audio format in Redis session (not in ClientState)
import json
from advanced_omi_backend.services.audio_stream.producer import (
get_audio_stream_producer,
)
session_key = f"audio:session:{client_state.stream_session_id}"
redis_client = audio_stream_producer.redis_client
await redis_client.hset(session_key, "audio_format", json.dumps(audio_format))
# Enqueue streaming jobs (speech detection + audio persistence)
from advanced_omi_backend.controllers.queue_controller import start_streaming_jobs
job_ids = start_streaming_jobs(
session_id=client_state.stream_session_id, user_id=user_id, client_id=client_id
)
# Store job IDs in Redis session (not in ClientState)
await audio_stream_producer.update_session_job_ids(
session_id=client_state.stream_session_id,
speech_detection_job_id=job_ids["speech_detection"],
audio_persistence_job_id=job_ids["audio_persistence"],
)
# Note: Placeholder conversation creation is handled by the audio persistence job,
# which reads the always_persist_enabled setting from global config.
# Launch interim results subscriber if WebSocket provided
subscriber_task = None
if websocket:
subscriber_task = asyncio.create_task(
subscribe_to_interim_results(websocket, client_state.stream_session_id)
)
application_logger.info(
f"📡 Launched interim results subscriber for session {client_state.stream_session_id}"
)
return subscriber_task
async def _finalize_streaming_session(
client_state, audio_stream_producer, user_id: str, user_email: str, client_id: str
) -> None:
"""
Finalize streaming session: flush buffer, signal workers, enqueue finalize job, cleanup.
Args:
client_state: Client state object
audio_stream_producer: Audio stream producer instance
user_id: User ID
user_email: User email
client_id: Client ID
"""
if not hasattr(client_state, "stream_session_id"):
application_logger.debug(f"No active session to finalize for {client_id}")
return
session_id = client_state.stream_session_id
try:
# Flush any remaining buffered audio
audio_format = getattr(client_state, "stream_audio_format", {})
await audio_stream_producer.flush_session_buffer(
session_id=session_id,
sample_rate=audio_format.get("rate", 16000),
channels=audio_format.get("channels", 1),
sample_width=audio_format.get("width", 2),
)
# Send end-of-session signal to workers
await audio_stream_producer.send_session_end_signal(session_id)
# Mark session as finalizing with user_stopped reason (audio-stop event)
await audio_stream_producer.finalize_session(
session_id, completion_reason="user_stopped"
)
# Store markers in Redis so open_conversation_job can persist them
if client_state.markers:
session_key = f"audio:session:{session_id}"
await audio_stream_producer.redis_client.hset(
session_key, "markers", json.dumps(client_state.markers)
)
client_state.markers.clear()
# NOTE: Finalize job disabled - open_conversation_job now handles everything
# The open_conversation_job will:
# 1. Detect the "finalizing" status
# 2. Enter 5-second grace period
# 3. Get audio file path
# 4. Mark session complete
# 5. Clean up Redis streams
# 6. Enqueue batch transcription and memory processing
#
# If no speech was detected (open_conversation_job never started):
# - Audio is discarded (intentional - we only create conversations with speech)
# - Redis streams are cleaned up by TTL
#
# TODO: Consider adding cleanup for no-speech scenarios if needed
application_logger.info(
f"✅ Session {session_id[:12]} marked as finalizing - open_conversation_job will handle cleanup"
)
# Clear session state from ClientState (only stream_session_id is stored there now)
# All other session metadata lives in Redis (single source of truth)
if hasattr(client_state, "stream_session_id"):
delattr(client_state, "stream_session_id")
except Exception as finalize_error:
application_logger.error(
f"❌ Failed to finalize streaming session: {finalize_error}", exc_info=True
)
async def _publish_audio_to_stream(
client_state,
audio_stream_producer,
audio_data: bytes,
user_id: str,
client_id: str,
sample_rate: int,
channels: int,
sample_width: int,
) -> None:
"""
Publish audio chunk to Redis Stream with chunk tracking.
Args:
client_state: Client state object
audio_stream_producer: Audio stream producer instance
audio_data: Raw PCM audio bytes
user_id: User ID
client_id: Client ID
sample_rate: Sample rate (Hz)
channels: Number of channels
sample_width: Bytes per sample
"""
if not hasattr(client_state, "stream_session_id"):
application_logger.warning(
f"⚠️ Received audio chunk before session initialized for {client_id}"
)
return
session_id = client_state.stream_session_id
# Publish to Redis Stream using producer (producer owns chunk counting)
await audio_stream_producer.add_audio_chunk(
audio_data=audio_data,
session_id=session_id,
user_id=user_id,
client_id=client_id,
sample_rate=sample_rate,
channels=channels,
sample_width=sample_width,
)
async def _handle_omi_audio_chunk(
client_state,
audio_stream_producer,
opus_payload: bytes,
decode_packet_fn,
strip_header: bool,
user_id: str,
client_id: str,
packet_count: int,
) -> None:
"""
Handle OMI audio chunk: decode Opus to PCM, then publish to stream.
Args:
client_state: Client state object
audio_stream_producer: Audio stream producer instance
opus_payload: Opus-encoded audio bytes
decode_packet_fn: Opus decoder function
strip_header: Whether to strip 3-byte BLE header before decoding
user_id: User ID
client_id: Client ID
packet_count: Current packet number for logging
"""
# Decode Opus to PCM
start_time = time.time()
loop = asyncio.get_running_loop()
pcm_data = await loop.run_in_executor(
_DEC_IO_EXECUTOR, decode_packet_fn, opus_payload, strip_header
)
decode_time = time.time() - start_time
if pcm_data:
if packet_count <= 5 or packet_count % 1000 == 0:
application_logger.debug(
f"🎵 Decoded OMI packet #{packet_count}: {len(opus_payload)} bytes -> "
f"{len(pcm_data)} PCM bytes (took {decode_time:.3f}s)"
)
# Publish decoded PCM to Redis Stream
await _publish_audio_to_stream(
client_state,
audio_stream_producer,
pcm_data,
user_id,
client_id,
OMI_SAMPLE_RATE,
OMI_CHANNELS,
OMI_SAMPLE_WIDTH,
)
else:
# Log decode failures for first 5 packets
if packet_count <= 5:
application_logger.warning(
f"❌ Failed to decode OMI packet #{packet_count}: {len(opus_payload)} bytes"
)
async def _handle_streaming_mode_audio(
client_state,
audio_stream_producer,
audio_data: bytes,
audio_format: dict,
user_id: str,
user_email: str,
client_id: str,
websocket: Optional[WebSocket] = None,
) -> Optional[asyncio.Task]:
"""
Handle audio chunk in streaming mode.
Args:
client_state: Client state object
audio_stream_producer: Audio stream producer instance
audio_data: Raw PCM audio bytes
audio_format: Audio format dict (rate, width, channels)
user_id: User ID
user_email: User email
client_id: Client ID
websocket: Optional WebSocket connection to launch interim results subscriber
Returns:
Interim results subscriber task if websocket provided and session initialized, None otherwise
"""
# Initialize session if needed
subscriber_task = None
if not hasattr(client_state, "stream_session_id"):
subscriber_task = await _initialize_streaming_session(
client_state,
audio_stream_producer,
user_id,
user_email,
client_id,
audio_format,
websocket=websocket, # Pass WebSocket to launch interim results subscriber
)
# Publish to Redis Stream
await _publish_audio_to_stream(
client_state,
audio_stream_producer,
audio_data,
user_id,
client_id,
audio_format.get("rate", 16000),
audio_format.get("channels", 1),
audio_format.get("width", 2),
)
return subscriber_task
async def _handle_batch_mode_audio(
client_state, audio_data: bytes, audio_format: dict, client_id: str
) -> None:
"""
Handle audio chunk in batch mode with rolling 30-minute limit.
Args:
client_state: Client state object
audio_data: Raw PCM audio bytes
audio_format: Audio format dict
client_id: Client ID
"""
# Initialize batch accumulator if needed
if not hasattr(client_state, "batch_audio_chunks"):
client_state.batch_audio_chunks = []
client_state.batch_audio_format = audio_format
client_state.batch_audio_bytes = 0 # Track total bytes
client_state.batch_chunks_processed = 0 # Track how many batches processed
application_logger.info(f"📦 Started batch audio accumulation for {client_id}")
# Accumulate audio
client_state.batch_audio_chunks.append(audio_data)
client_state.batch_audio_bytes += len(audio_data)
application_logger.debug(
f"📦 Accumulated chunk #{len(client_state.batch_audio_chunks)} ({len(audio_data)} bytes) for {client_id}"
)
# Calculate duration: sample_rate * width * channels = bytes/second
sample_rate = audio_format.get("rate", 16000)
width = audio_format.get("width", 2)
channels = audio_format.get("channels", 1)
bytes_per_second = sample_rate * width * channels
accumulated_seconds = client_state.batch_audio_bytes / bytes_per_second
MAX_BATCH_SECONDS = 30 * 60 # 30 minutes
# Check if we've hit the 30-minute limit
if accumulated_seconds >= MAX_BATCH_SECONDS:
application_logger.warning(
f"⚠️ Batch accumulation reached 30-minute limit "
f"({accumulated_seconds:.1f}s, {client_state.batch_audio_bytes / 1024 / 1024:.1f} MB). "
f"Processing batch #{client_state.batch_chunks_processed + 1}..."
)
# Process this batch (will create conversation and transcribe)
await _process_rolling_batch(
client_state,
user_id=client_state.user_id, # Need to store these on session start
user_email=client_state.user_email,
client_id=client_state.client_id,
batch_number=client_state.batch_chunks_processed + 1,
)
# Clear buffer for next batch
client_state.batch_audio_chunks = []
client_state.batch_audio_bytes = 0
client_state.batch_chunks_processed += 1
application_logger.info(
f"✅ Rolled batch #{client_state.batch_chunks_processed}. "
f"Starting fresh accumulation for next 30 minutes."
)
async def _handle_audio_chunk(
client_state,
audio_stream_producer,
audio_data: bytes,
audio_format: dict,
user_id: str,
user_email: str,
client_id: str,
websocket: Optional[WebSocket] = None,
) -> Optional[asyncio.Task]:
"""
Route audio chunk to appropriate mode handler (streaming or batch).
Args:
client_state: Client state object
audio_stream_producer: Audio stream producer instance
audio_data: Raw PCM audio bytes
audio_format: Audio format dict
user_id: User ID
user_email: User email
client_id: Client ID
websocket: Optional WebSocket connection to launch interim results subscriber
Returns:
Interim results subscriber task if websocket provided and streaming mode, None otherwise
"""
recording_mode = getattr(client_state, "recording_mode", "batch")
if recording_mode == "streaming":
return await _handle_streaming_mode_audio(
client_state,
audio_stream_producer,
audio_data,
audio_format,
user_id,
user_email,
client_id,
websocket=websocket,
)
else:
await _handle_batch_mode_audio(
client_state, audio_data, audio_format, client_id
)
return None
async def _handle_audio_session_start(
client_state,
audio_format: dict,
client_id: str,
websocket: Optional[WebSocket] = None,
) -> tuple[bool, str]:
"""
Handle audio-start event - validate mode and set recording mode.
Args:
client_state: Client state object
audio_format: Audio format dict with mode
client_id: Client ID
websocket: Optional WebSocket connection (for WebUI error messages)
Returns:
(audio_streaming_flag, recording_mode)
"""
from advanced_omi_backend.services.transcription import is_transcription_available
recording_mode = audio_format.get("mode", "batch")
application_logger.info(
f"🔴 BACKEND: Received audio-start for {client_id} - "
f"mode={recording_mode}, full format={audio_format}"
)
# Store on client state for later use
client_state.recording_mode = recording_mode
# VALIDATION: Check if streaming mode is available
if recording_mode == "streaming":
if not is_transcription_available("streaming"):
error_msg = (
"Streaming transcription not available. "
"Please use Batch mode or configure a streaming STT provider (defaults.stt_stream in config.yml)."
)
application_logger.warning(
f"⚠️ Streaming mode requested but stt_stream not configured for {client_id}"
)
# Send error to WebSocket client (for WebUI display)
if websocket and websocket.client_state == WebSocketState.CONNECTED:
try:
error_response = {
"type": "error",
"error": "streaming_not_configured",
"message": error_msg,
"code": 400,
}
await websocket.send_json(error_response)
application_logger.info(
f"📤 Sent streaming error to WebUI client {client_id}"
)
# Close the websocket connection after sending error
await websocket.close(
code=1008, reason="Streaming transcription not configured"
)
application_logger.info(
f"🔌 Closed WebSocket connection for {client_id} due to streaming config error"
)
# Raise ValueError to exit the handler completely
raise ValueError(error_msg)
except ValueError:
# Re-raise ValueError to exit handler
raise
except Exception as e:
application_logger.error(f"Failed to send error to client: {e}")
# Still raise ValueError to exit handler
raise ValueError(error_msg)
# For OMI devices (no websocket), fall back to batch mode silently
if not websocket:
application_logger.warning(
f"🔄 OMI device {client_id} requested streaming but falling back to batch mode"
)
recording_mode = "batch"
client_state.recording_mode = recording_mode
application_logger.info(
f"🎙️ Audio session started for {client_id} - "
f"Format: {audio_format.get('rate')}Hz, "
f"{audio_format.get('width')}bytes, "
f"{audio_format.get('channels')}ch, "
f"Mode: {recording_mode}"
)
return True, recording_mode # Switch to audio streaming mode
async def _handle_audio_session_stop(
client_state, audio_stream_producer, user_id: str, user_email: str, client_id: str
) -> bool:
"""
Handle audio-stop event - finalize session based on mode.
Args:
client_state: Client state object
audio_stream_producer: Audio stream producer instance
user_id: User ID
user_email: User email
client_id: Client ID
Returns:
False to switch back to control mode
"""
recording_mode = getattr(client_state, "recording_mode", "batch")
application_logger.info(
f"🛑 Audio session stopped for {client_id} (mode: {recording_mode})"
)