Implement webRTC Transport layer for screen sharing and for Low-Latency Input

README.md

@@ -7,7 +7,7 @@
 
 # Rein
 
-A cross-platform, remote desktop (started as a couch keyboard replacement utilizing touch-screen devices following the **KISS principle**. It allows touchscreen devices to act as a trackpad and keyboard for a desktop system through a locally served web interface. This can serve as a standard open interface for upcoming cloud PC/gaming services. The project also bring better STT for Linux via phone and other platforms.
+A cross-platform, remote desktop (started as a couch keyboard replacement utilizing touch-screen devices following the **KISS principle**. It allows touchscreen devices and non touch desktop to act as a trackpad and keyboard for a desktop system through a locally served web interface. Think this could become some sort of standardization for cloud PC or cloud gaming interfaces, where all providers can push improvements and make them directly available to all platforms. Then upcoming providers would only need to think about the infrastructure. The project also bring better STT for Linux via phone and other platforms.
 
 > Contributions are welcome! Please leave a star ⭐ to show your support.
 

src/components/Trackpad/ScreenMirror.tsx

@@ -21,22 +21,27 @@ export const ScreenMirror = ({
 	isTracking,
 	handlers,
 }: ScreenMirrorProps) => {
-	const { wsRef, status } = useConnection()
-	const canvasRef = useRef<HTMLCanvasElement>(null)
-	const { hasFrame } = useMirrorStream(wsRef, canvasRef, status)
+	const { status } = useConnection()
+	const videoRef = useRef<HTMLVideoElement>(null)
+	const { hasStream } = useMirrorStream(videoRef, status)
 
 	return (
 		<div className="absolute inset-0 flex items-center justify-center bg-black overflow-hidden select-none touch-none">
 			{/* Mirror Canvas */}
-			<canvas
-				ref={canvasRef}
+			<video
+				ref={videoRef}
+				aria-label="Screen Share Video"
+				autoPlay
+				playsInline
+				muted
+				onError={handleVideoError}
 				className={`w-full h-full object-contain transition-opacity duration-500 ${
-					hasFrame ? "opacity-100" : "opacity-0"
+					hasStream ? "opacity-100" : "opacity-0"
 				}`}
 			/>
 
 			{/* Standby UI */}
-			{!hasFrame && (
+			{!hasStream && (
 				<div className="absolute inset-0 flex flex-col items-center justify-center text-gray-400 gap-4">
 					<div className="loading loading-spinner loading-lg text-primary" />
 					<div className="text-center px-6">
@@ -57,3 +62,7 @@ export const ScreenMirror = ({
 		</div>
 	)
 }
+
+const handleVideoError = (e: React.SyntheticEvent<HTMLVideoElement, Event>) => {
+	console.error("[RTC] Video playback error", e.currentTarget.error)
+}

src/contexts/ConnectionProvider.tsx

@@ -1,16 +1,30 @@
 "use client"
-
 import type React from "react"
-import { createContext, useContext, useEffect, useRef, useState } from "react"
-
+import {
+	createContext,
+	useCallback,
+	useContext,
+	useEffect,
+	useRef,
+	useState,
+} from "react"
+import { useWebRTC } from "../hooks/useWebRTC"
 type ConnectionStatus = "connecting" | "connected" | "disconnected"
 
 interface ConnectionContextType {
 	wsRef: React.RefObject<WebSocket | null>
 	status: ConnectionStatus
 	platform: string | null
 	latency: number | null
+	pcRef: React.RefObject<RTCPeerConnection | null>
+	subscribeMirrorStream: (
+		cb: (stream: MediaStream | null) => void,
+	) => () => void
+	createPeerConnection: () => RTCPeerConnection
+	closePeerConnection: () => void
 	send: (msg: unknown) => void
+	sendInput: (msg: unknown) => void
+	configureMediaSender: (pc: RTCPeerConnection, sender: RTCRtpSender) => void
 	subscribe: (type: string, callback: (msg: unknown) => void) => () => void
 }
 
@@ -34,21 +48,48 @@ export function ConnectionProvider({
 	const [latency, setLatency] = useState<number | null>(null)
 	const isMountedRef = useRef(true)
 	const subscribersRef = useRef<Record<string, Set<(msg: unknown) => void>>>({})
+	const mirrorStreamSubscribersRef = useRef<
+		Set<(stream: MediaStream | null) => void>
+	>(new Set())
 
-	const subscribe = (type: string, callback: (msg: unknown) => void) => {
-		if (!subscribersRef.current[type]) {
-			subscribersRef.current[type] = new Set()
-		}
+	const reconnectCountRef = useRef(0)
+	const reconnectTimerRef = useRef<number | null>(null)
+
+	const subscribe = useCallback(
+		(type: string, callback: (msg: unknown) => void) => {
+			if (!subscribersRef.current[type]) {
+				subscribersRef.current[type] = new Set()
+			}
 
-		subscribersRef.current[type].add(callback)
+			subscribersRef.current[type].add(callback)
 
-		return () => {
-			subscribersRef.current[type].delete(callback)
+			return () => {
+				subscribersRef.current[type].delete(callback)
+			}
+		},
+		[],
+	)
+
+	const send = useCallback((msg: unknown) => {
+		if (wsRef.current?.readyState === WebSocket.OPEN) {
+			wsRef.current.send(JSON.stringify(msg))
 		}
-	}
+	}, [])
 
-	const reconnectCountRef = useRef(0)
-	const reconnectTimerRef = useRef<number | null>(null)
+	const handleTrackReceived = useCallback((stream: MediaStream | null) => {
+		for (const cb of mirrorStreamSubscribersRef.current) {
+			cb(stream)
+		}
+	}, [])
+
+	const {
+		pcRef,
+		createPeerConnection,
+		handleSignalingMessage,
+		sendInput,
+		configureMediaSender,
+		closePeerConnection,
+	} = useWebRTC(send, handleTrackReceived)
 
 	useEffect(() => {
 		isMountedRef.current = true
@@ -69,17 +110,15 @@ export function ConnectionProvider({
 			try {
 				storedToken = localStorage.getItem("rein_auth_token")
 			} catch (_e) {
-				// Restricted context (e.g. private mode)
+				// Restricted context
 			}
 
 			const token = urlToken || storedToken
 
 			if (urlToken && urlToken !== storedToken) {
 				try {
 					localStorage.setItem("rein_auth_token", urlToken)
-				} catch (_e) {
-					// Failed to store
-				}
+				} catch (_e) {}
 			}
 
 			let wsUrl = `${protocol}//${host}/ws`
@@ -97,50 +136,53 @@ export function ConnectionProvider({
 			setStatus("connecting")
 			const socket = new WebSocket(wsUrl)
 
-			socket.onopen = () => {
+			socket.onopen = async () => {
 				if (isMountedRef.current) {
 					setStatus("connected")
-					reconnectCountRef.current = 0 // Reset on successful connect
+					reconnectCountRef.current = 0
+
 					if (reconnectTimerRef.current) {
 						clearTimeout(reconnectTimerRef.current)
 						reconnectTimerRef.current = null
 					}
 				}
 			}
 
-			socket.onmessage = (event) => {
+			socket.onmessage = async (event) => {
 				if (!isMountedRef.current) return
 
-				// Handle binary data separately (frames)
 				if (event.data instanceof Blob) {
-					// Relayed directly via internal listeners in hooks
 					return
 				}
 
 				try {
 					const msg = JSON.parse(event.data)
+
 					if (msg.type === "connected") {
 						setPlatform(msg.platform || null)
 					}
+
+					if (
+						msg.type === "offer" ||
+						msg.type === "answer" ||
+						msg.type === "ice-candidate"
+					) {
+						handleSignalingMessage(msg)
+					}
+
 					const typeSubscribers = subscribersRef.current[msg.type]
 					if (typeSubscribers) {
 						for (const callback of typeSubscribers) {
 							callback(msg)
 						}
 					}
-				} catch (_e) {
-					// Not JSON or silent error
-				}
+				} catch (_e) {}
 			}
 
 			socket.onclose = () => {
 				if (isMountedRef.current) {
 					setStatus("disconnected")
-					// Exponential Backoff
-					const delay = Math.min(
-						1000 * 2 ** reconnectCountRef.current,
-						30000, // Max 30s
-					)
+					const delay = Math.min(1000 * 2 ** reconnectCountRef.current, 30000)
 					reconnectCountRef.current += 1
 
 					if (reconnectTimerRef.current) {
@@ -157,6 +199,7 @@ export function ConnectionProvider({
 			wsRef.current = socket
 		}
 		connect()
+
 		return () => {
 			isMountedRef.current = false
 			if (reconnectTimerRef.current) {
@@ -171,15 +214,9 @@ export function ConnectionProvider({
 				wsRef.current = null
 			}
 		}
-	}, [])
-
-	const send = (msg: unknown) => {
-		if (wsRef.current?.readyState === WebSocket.OPEN) {
-			wsRef.current.send(JSON.stringify(msg))
-		}
-	}
+	}, [handleSignalingMessage]) // Depend on handleSignalingMessage
 
-	// Ping/Pong heartbeat for latency measurement
+	// Ping/Pong heartbeat
 	useEffect(() => {
 		if (status !== "connected") {
 			setLatency(null)
@@ -199,7 +236,6 @@ export function ConnectionProvider({
 		}
 		subscribersRef.current.pong.add(handlePong)
 
-		// Send a ping immediately, then every 2 seconds
 		const sendPing = () => {
 			if (ws.readyState === WebSocket.OPEN) {
 				ws.send(
@@ -221,7 +257,23 @@ export function ConnectionProvider({
 
 	return (
 		<ConnectionContext.Provider
-			value={{ wsRef, status, platform, latency, send, subscribe }}
+			value={{
+				wsRef,
+				status,
+				platform,
+				latency,
+				pcRef,
+				subscribeMirrorStream: (cb) => {
+					mirrorStreamSubscribersRef.current.add(cb)
+					return () => mirrorStreamSubscribersRef.current.delete(cb)
+				},
+				createPeerConnection,
+				closePeerConnection,
+				send,
+				sendInput,
+				configureMediaSender,
+				subscribe,
+			}}
 		>
 			{children}
 		</ConnectionContext.Provider>