add multi-speaker vc support

Author: ncoop57

cordslite/_modidx.py

@@ -76,9 +76,11 @@
                                 'cordslite.core.VoiceClient.__init__': ('core.html#voiceclient.__init__', 'cordslite/core.py'),
                                 'cordslite.core.VoiceClient.__repr__': ('core.html#voiceclient.__repr__', 'cordslite/core.py'),
                                 'cordslite.core.VoiceClient._connect': ('core.html#voiceclient._connect', 'cordslite/core.py'),
+                                'cordslite.core.VoiceClient._get_proc': ('core.html#voiceclient._get_proc', 'cordslite/core.py'),
                                 'cordslite.core.VoiceClient._recv_audio': ('core.html#voiceclient._recv_audio', 'cordslite/core.py'),
                                 'cordslite.core.VoiceClient._udp': ('core.html#voiceclient._udp', 'cordslite/core.py'),
                                 'cordslite.core.VoiceClient._vhb': ('core.html#voiceclient._vhb', 'cordslite/core.py'),
+                                'cordslite.core.VoiceClient._write_audio': ('core.html#voiceclient._write_audio', 'cordslite/core.py'),
                                 'cordslite.core.VoiceClient.join': ('core.html#voiceclient.join', 'cordslite/core.py'),
                                 'cordslite.core.VoiceClient.leave': ('core.html#voiceclient.leave', 'cordslite/core.py'),
                                 'cordslite.core.VoiceClient.start_recording': ( 'core.html#voiceclient.start_recording',
@@ -89,5 +91,6 @@
                                 'cordslite.core.VoiceUDP.datagram_received': ('core.html#voiceudp.datagram_received', 'cordslite/core.py'),
                                 'cordslite.core.decrypt_pkt': ('core.html#decrypt_pkt', 'cordslite/core.py'),
                                 'cordslite.core.get_ip': ('core.html#get_ip', 'cordslite/core.py'),
+                                'cordslite.core.silence': ('core.html#silence', 'cordslite/core.py'),
                                 'cordslite.core.websockets.asyncio.client.ClientConnection.send': ( 'core.html#websockets.asyncio.client.clientconnection.send',
                                                                                                     'cordslite/core.py')}}}

cordslite/core.py

@@ -3,9 +3,9 @@
 # AUTOGENERATED! DO NOT EDIT! File to edit: ../nbs/00_core.ipynb.
 
 # %% auto #0
-__all__ = ['evt_typs', 'sr', 'DiscordClient', 'DiscordObject', 'DiscordError', 'Guild', 'Channel', 'Channels', 'Message',
+__all__ = ['evt_typs', 'spf', 'sr', 'DiscordClient', 'DiscordObject', 'DiscordError', 'Guild', 'Channel', 'Channels', 'Message',
            'Messages', 'User', 'Member', 'Members', 'GatewayClient', 'Op', 'Event', 'VoiceClient', 'get_ip', 'VoiceUDP',
-           'decrypt_pkt', 'Bot']
+           'decrypt_pkt', 'silence', 'Bot']
 
 # %% ../nbs/00_core.ipynb #9868997a
 from fastcore.utils import *
@@ -176,6 +176,7 @@ async def send(self:websockets.asyncio.client.ClientConnection, msg, **kw):
 # %% ../nbs/00_core.ipynb #130c739c
 @patch
 async def _connect(self:GatewayClient):
+    if self.ws: await self.ws.close()
     self.ws = await websockets.connect(self.url)
     hello = json.loads(await self.ws.recv())
     self.hb_int = hello['d']['heartbeat_interval']
@@ -327,26 +328,50 @@ def decrypt_pkt(pkt, secret):
     ext_sz = int.from_bytes(pkt[14:16], 'big') * 4 # extension data size in bytes
     return decrypted[ext_sz:]
 
-# %% ../nbs/00_core.ipynb #96f699af
+# %% ../nbs/00_core.ipynb #c06edb1d
+spf = 960
 sr = 48_000
+def silence(n_smpls:int): return b'\x00' * (n_smpls * 2) # 2 bytes per sample (s16le)
+
+@patch
+def _get_proc(self:VoiceClient, ssrc):
+    if ssrc not in self._procs:
+        pth = str(self._rec_path.with_stem(f"{self._rec_path.stem}_{ssrc}"))
+        proc = (ffmpeg.input('pipe:', f='s16le', ar=sr, ac=1)
+                      .output(pth).overwrite_output()
+                      .run_async(pipe_stdin=True, quiet=True))
+        proc.stdin.write(silence(int((time.time() - self._rec_start) * sr)))
+        self._procs[ssrc] = proc
+    return self._procs[ssrc]
+
+@patch
+def _write_audio(self:VoiceClient, ssrc, ts, data):
+    proc = self._get_proc(ssrc)
+    # Fill silence gaps
+    if ssrc in self._last_ts:
+        gap = ts - self._last_ts[ssrc] - spf
+        if gap > 0: proc.stdin.write(silence(gap))
+    self._last_ts[ssrc] = ts
+    proc.stdin.write(self._decoders[ssrc].decode(data, spf))
 
 @patch
 async def _recv_audio(self:VoiceClient):
     while self._recording:
         try:
             pkt = await asyncio.wait_for(self.proto.packets.get(), timeout=0.1)
-            if pkt[1] != 0x78: continue
-            opus_data = decrypt_pkt(pkt, self.secret)
-            if opus_data: self._proc.stdin.write(self._decoder.decode(opus_data, 960))
+            if pkt[1] != 0x78: continue # filter out RTP non-voice packets 
+            ssrc = int.from_bytes(pkt[8:12], 'big')
+            ts = int.from_bytes(pkt[4:8], 'big')
+            if ssrc not in self._decoders: self._decoders[ssrc] = opuslib_next.Decoder(sr, 1)
+            if data := decrypt_pkt(pkt, self.secret): self._write_audio(ssrc, ts, data)
         except asyncio.TimeoutError: continue
 
+# %% ../nbs/00_core.ipynb #b17c0541
 @patch
 def start_recording(self:VoiceClient, path='recording.mp3'):
-    self._rec_path = path
-    self._decoder = opuslib_next.Decoder(sr, 1)
-    self._proc = (ffmpeg.input('pipe:', f='s16le', ar=sr, ac=1)
-                        .output(path).overwrite_output()
-                        .run_async(pipe_stdin=True, quiet=True))
+    while not self.proto.packets.empty(): self.proto.packets.get_nowait()
+    self._rec_path,self._decoders,self._procs,self._last_ts = Path(path),{},{},{}
+    self._rec_start = time.time()
     self._recording = True
     self._rec_task = asyncio.create_task(self._recv_audio())
     return path
@@ -355,8 +380,8 @@ def start_recording(self:VoiceClient, path='recording.mp3'):
 def stop_recording(self:VoiceClient):
     self._recording = False
     self._rec_task.cancel()
-    self._proc.communicate()
-    return self._rec_path
+    for p in self._procs.values(): p.communicate()
+    return [str(self._rec_path.with_stem(f"{self._rec_path.stem}_{ssrc}")) for ssrc in self._procs]
 
 # %% ../nbs/00_core.ipynb #a79084f1
 @patch
@@ -387,7 +412,9 @@ async def _on_msg(self, msg):
         if not parts or not parts[0].startswith('!'): return
         name = parts[0][1:]
         if name not in self.cmds: return
-        try: await self.cmds[name](msg, parts[1] if len(parts) > 1 else '')
+        try:
+            res = self.cmds[name](msg, parts[1] if len(parts) > 1 else '')
+            if asyncio.iscoroutine(res): await res
         except Exception as e:
             self.errors.append(e)
             if self._on_err: await self._on_err(msg, e)