diff --git a/electron-build/entitlements.mac.inherit.plist b/electron-build/entitlements.mac.inherit.plist new file mode 100644 index 000000000..46c76069a --- /dev/null +++ b/electron-build/entitlements.mac.inherit.plist @@ -0,0 +1,27 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.allow-dyld-environment-variables + + com.apple.security.device.camera + + com.apple.security.device.audio-input + + com.apple.security.device.usb + + com.apple.security.device.serial + + com.apple.security.network.server + + com.apple.security.network.client + + + + + + diff --git a/electron-build/entitlements.mac.plist b/electron-build/entitlements.mac.plist new file mode 100644 index 000000000..46c76069a --- /dev/null +++ b/electron-build/entitlements.mac.plist @@ -0,0 +1,27 @@ + + + + + com.apple.security.cs.allow-jit + + com.apple.security.cs.allow-unsigned-executable-memory + + com.apple.security.cs.allow-dyld-environment-variables + + com.apple.security.device.camera + + com.apple.security.device.audio-input + + com.apple.security.device.usb + + com.apple.security.device.serial + + com.apple.security.network.server + + com.apple.security.network.client + + + + + + diff --git a/package.json b/package.json index 06e3d0bbe..d87aedf05 100644 --- a/package.json +++ b/package.json @@ -120,7 +120,11 @@ "target": [ "dmg" ], - "icon": "electron-build/icon.icns" + "icon": "electron-build/icon.icns", + "entitlements": "electron-build/entitlements.mac.plist", + "entitlementsInherit": "electron-build/entitlements.mac.inherit.plist", + "hardenedRuntime": true, + "gatekeeperAssess": false }, "dmg": { "background": "electron-build/background.png", @@ -490,4 +494,4 @@ } }, "packageManager": "yarn@1.22.22" -} +} \ No newline at end of file diff --git a/src/app/src/App.tsx b/src/app/src/App.tsx index e8c968231..05135212d 100644 --- a/src/app/src/App.tsx +++ b/src/app/src/App.tsx @@ -1,6 +1,7 @@ import { useEffect } from 'react'; import { Provider as ReduxProvider } from 'react-redux'; import { HashRouter } from 'react-router'; +import isElectron from 'is-electron'; import { store as reduxStore } from 'app/store/redux'; import rootSaga from 'app/store/redux/sagas'; @@ -9,6 +10,7 @@ import store from 'app/store'; import * as user from 'app/lib/user'; import controller from 'app/lib/controller'; import ConfirmationDialog from 'app/components/ConfirmationDialog/ConfirmationDialog'; +import { initializeGlobalCameraService } from 'app/lib/camera/globalCameraService'; import { Toaster } from './components/shadcn/Sonner'; import { ReactRoutes } from './react-routes'; @@ -28,6 +30,24 @@ function App() { query: 'token=' + token, }; controller.connect(host, options); + + // Initialize camera service after controller connection (only on main client) + controller.addListener('connect', () => { + // Main client check: Either Electron app OR localhost browser + // Remote clients are browser-based connections to external servers + const isMainClient = isElectron() || + window.location.hostname === 'localhost' || + window.location.hostname === '127.0.0.1' || + window.location.hostname === '0.0.0.0'; + + if (isMainClient) { + console.log('[App] Initializing camera service on main client'); + initializeGlobalCameraService().catch((error) => { + console.error('Failed to initialize camera service:', error); + }); + } + }); + return; } else { console.log('no auth'); diff --git a/src/app/src/features/Camera/index.tsx b/src/app/src/features/Camera/index.tsx new file mode 100644 index 000000000..42ffe6bfd --- /dev/null +++ b/src/app/src/features/Camera/index.tsx @@ -0,0 +1,654 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import React, { useState, useEffect, useRef } from 'react'; +import { FaCamera, FaExclamationTriangle, FaInfoCircle } from 'react-icons/fa'; +import { toast } from 'app/lib/toaster'; +import isElectron from 'is-electron'; + +import { Switch } from 'app/components/shadcn/Switch'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from 'app/components/shadcn/Select'; +import { CameraSettings, CameraStatus } from 'app/services/CameraService'; +import store from 'app/store'; +import controller from 'app/lib/controller'; +import { getGlobalCameraService } from 'app/lib/camera/globalCameraService'; + +const RESOLUTION_OPTIONS = [ + { value: 'low', label: '640×480 (Low)' }, + { value: 'medium', label: '1280×720 (Medium)' }, + { value: 'high', label: '1920×1080 (High)' }, +]; + +const FRAME_RATE_OPTIONS = [ + { value: 15, label: '15 FPS' }, + { value: 30, label: '30 FPS' }, +]; + +const Camera: React.FC = () => { + // Removed spam logs that were causing re-render loop detection issues + + // Check if we're in headless mode (remote client) + const [isHeadlessMode, setIsHeadlessMode] = useState(false); + const [isCheckingHeadless, setIsCheckingHeadless] = useState(true); + + useEffect(() => { + // Check if we're in headless mode (remote client connecting to external server) + // We need to detect if this app is connecting to an EXTERNAL server, not if it's SERVING to remote clients + const checkHeadlessMode = async () => { + try { + // Main client = Electron app OR localhost browser + // Remote client = browser accessing external server (non-localhost) + const isMainClient = isElectron() || + window.location.hostname === 'localhost' || + window.location.hostname === '127.0.0.1' || + window.location.hostname === '0.0.0.0'; + + // Only consider it headless mode if we're NOT the main client + // (i.e., we're a browser accessing a remote server from another device) + setIsHeadlessMode(!isMainClient); + } catch (error) { + console.error('[Camera] Failed to check headless mode:', error); + setIsHeadlessMode(false); + } finally { + setIsCheckingHeadless(false); + } + }; + + checkHeadlessMode(); + }, []); + + // Helper function to safely send controller commands + const sendControllerCommand = (command: string, ...args: any[]) => { + if (controller.socket && controller.socket.connected) { + controller.command(command, ...args); + } else { + console.warn(`Cannot send ${command}: socket not connected`); + } + }; + + // Get the global camera service instance (only if not in headless mode) + const cameraService = !isHeadlessMode ? getGlobalCameraService() : null; + + // Helper function to safely use camera service + const withCameraService = (fn: (service: NonNullable, ...args: T) => void | Promise) => { + return async (...args: T) => { + let service = getGlobalCameraService(); + + // If no service, wait a bit and try again + if (!service) { + await new Promise(resolve => setTimeout(resolve, 1000)); + service = getGlobalCameraService(); + } + + if (!service) { + toast.error('Camera service not available. Please wait for initialization or refresh the page.'); + return; + } + return fn(service, ...args); + }; + }; + + const [devices, setDevices] = useState([]); + const [settings, setSettings] = useState(() => { + if (cameraService) { + return cameraService.getSettings(); + } + return store.get('workspace.camera', { + enabled: false, + deviceId: '', + constraints: { width: 1280, height: 720, frameRate: 30 }, + qualityPreset: 'medium' as const, + }); + }); + const [status, setStatus] = useState(() => { + if (cameraService) { + return cameraService.getStatus(); + } + return { + enabled: false, + available: false, + streaming: false, + error: null, + metrics: { fps: 0, bitrateKbps: 0, viewers: 0, droppedFrames: 0 }, + }; + }); + const [isInitialized, setIsInitialized] = useState(!!cameraService); + const videoRef = useRef(null); + + const [, setCameraState] = useState(() => store.get('workspace.camera', { + enabled: false, + deviceId: '', + constraints: { + width: 1280, + height: 720, + frameRate: 30, + }, + qualityPreset: 'medium' as const, + })); + + // Effect to attach stream to video element when streaming status changes + useEffect(() => { + const service = getGlobalCameraService(); + if (!service || !videoRef.current) { + return; + } + + if (status.streaming) { + const stream = service.getStream(); + console.log('[Camera] Attaching stream to video element, stream:', stream, 'tracks:', stream?.getTracks().length); + if (stream) { + videoRef.current.srcObject = stream; + videoRef.current.play().catch((err) => { + console.error('[Camera] Preview video play failed:', err); + }); + } + } else { + console.log('[Camera] Clearing video element srcObject'); + videoRef.current.srcObject = null; + } + }, [status.streaming]); + + useEffect(() => { + // Camera service is now initialized globally, just set up event listeners + const initializeCameraComponent = async () => { + let service = cameraService; + + // If service not available, wait for it or try to initialize + if (!service) { + // Try to get the service multiple times with increasing delays + for (let attempt = 0; attempt < 10; attempt++) { + await new Promise(resolve => setTimeout(resolve, attempt * 500 + 500)); + service = getGlobalCameraService(); + if (service) { + break; + } + } + + // If still no service, try to trigger global initialization + if (!service) { + try { + const { initializeGlobalCameraService } = await import('app/lib/camera/globalCameraService'); + service = await initializeGlobalCameraService(); + } catch (error) { + console.error('Failed to initialize camera service:', error); + return; + } + } + } + + if (service) { + // Wait a bit to ensure service has loaded settings from store + await new Promise(resolve => setTimeout(resolve, 100)); + + // Get current settings and status + const currentSettings = service.getSettings(); + const currentStatus = service.getStatus(); + + // Update local state with current service state + setSettings(currentSettings); + setStatus(currentStatus); + setIsInitialized(true); + + // Initialize the service if not already done (requests permission) + // This only happens when user opens Camera page for the first time + try { + if (service.getDevices().length === 0) { + console.log('[Camera] Initializing camera service and requesting permission...'); + await service.initialize(); + } + } catch (error) { + console.error('[Camera] Failed to initialize camera service:', error); + } + + // Enumerate devices immediately + try { + await service.enumerateDevices(); + // Also directly get devices in case event hasn't fired yet + const currentDevices = service.getDevices(); + if (currentDevices.length > 0) { + setDevices(currentDevices); + } + } catch (error) { + console.error('Failed to enumerate camera devices:', error); + } + + // Set up event listeners + const handleDevicesChanged = (newDevices: MediaDeviceInfo[]) => { + setDevices(newDevices); + }; + + const handleSettingsChanged = (newSettings: CameraSettings) => { + setSettings(newSettings); + }; + + const handleStatusChanged = (newStatus: CameraStatus) => { + setStatus(newStatus); + }; + + const handleStreamStarted = (stream: MediaStream) => { + if (videoRef.current) { + videoRef.current.srcObject = stream; + } + }; + + const handleStreamStopped = () => { + if (videoRef.current) { + videoRef.current.srcObject = null; + } + }; + + const handleError = (error: string) => { + toast.error(error); + }; + + const handleMetricsUpdated = (metrics: any) => { + setStatus(prev => ({ ...prev, metrics })); + // Only send metrics to server if streaming is enabled + if (status.streaming && settings.enabled) { + sendControllerCommand('camera:metrics', metrics); + } + }; + + service.on('devicesChanged', handleDevicesChanged); + service.on('settingsChanged', handleSettingsChanged); + service.on('statusChanged', handleStatusChanged); + service.on('streamStarted', handleStreamStarted); + service.on('streamStopped', handleStreamStopped); + service.on('metricsUpdated', handleMetricsUpdated); + service.on('error', handleError); + + // Store cleanup function + return () => { + service.off('devicesChanged', handleDevicesChanged); + service.off('settingsChanged', handleSettingsChanged); + service.off('statusChanged', handleStatusChanged); + service.off('streamStarted', handleStreamStarted); + service.off('streamStopped', handleStreamStopped); + service.off('metricsUpdated', handleMetricsUpdated); + service.off('error', handleError); + }; + } + }; + + // Call the async function and handle cleanup + let cleanupFunction: (() => void) | undefined; + + initializeCameraComponent().then(cleanup => { + cleanupFunction = cleanup; + }); + + // Return cleanup function for useEffect + return () => { + if (cleanupFunction) { + cleanupFunction(); + } + }; + }, []); + + // Separate effect to connect video element when it becomes available and stream is already active + useEffect(() => { + if (isInitialized && status.streaming && videoRef.current && !videoRef.current.srcObject) { + const service = getGlobalCameraService(); + if (service) { + const stream = service.getStream(); + if (stream) { + videoRef.current.srcObject = stream; + videoRef.current.play().catch((err) => { + console.warn('[Camera] Late-bind preview video play failed:', err); + }); + } + } + } + }, [isInitialized, status.streaming]); + + // Additional effect to connect video on render when streaming is active + // This runs every render to catch the case where videoRef becomes available + useEffect(() => { + if (isInitialized && status.streaming && videoRef.current && !videoRef.current.srcObject) { + const service = getGlobalCameraService(); + if (service) { + const stream = service.getStream(); + if (stream) { + videoRef.current.srcObject = stream; + videoRef.current.play().catch((err) => { + console.warn('[Camera] Video play on render failed:', err); + }); + } + } + } + }); + + useEffect(() => { + if (isInitialized && cameraService) { + cameraService.enumerateDevices(); + } + }, [isInitialized, cameraService]); + + const handleToggleStreaming = withCameraService(async (service, enabled: boolean) => { + try { + const currentSettings = service.getSettings(); + + if (enabled) { + // Validate device is selected before starting + if (!currentSettings.deviceId) { + toast.error('Please select a camera device first'); + return; + } + + // Start the stream first + await service.startStream(); + + // Update settings with enabled=true + const newSettings = { ...currentSettings, enabled: true }; + service.updateSettings(newSettings); + + // Persist to store + store.set('workspace.camera', newSettings); + setCameraState(newSettings); + setSettings(newSettings); + + // Notify server + sendControllerCommand('camera:startStream'); + sendControllerCommand('camera:updateSettings', newSettings); + + toast.success('Camera streaming started'); + } else { + // Stop the stream first + await service.stopStream(); + + // Update settings with enabled=false + const newSettings = { ...currentSettings, enabled: false }; + service.updateSettings(newSettings); + + // Persist to store + store.set('workspace.camera', newSettings); + setCameraState(newSettings); + setSettings(newSettings); + + // Notify server + sendControllerCommand('camera:stopStream'); + sendControllerCommand('camera:updateSettings', newSettings); + + toast.success('Camera streaming stopped'); + } + } catch (error) { + console.error('Failed to toggle streaming:', error); + toast.error(`Failed to ${enabled ? 'start' : 'stop'} streaming`); + } + }); + + const handleDeviceChange = withCameraService((service, deviceId: string | number) => { + const newSettings = { ...settings, deviceId: String(deviceId) }; + service.updateSettings(newSettings); + + // Update store and notify server + store.set('workspace.camera', newSettings); + setCameraState(newSettings); + sendControllerCommand('camera:updateSettings', newSettings); + }); + + const handleQualityChange = withCameraService((service, qualityPreset: string | number) => { + const newSettings = { ...settings, qualityPreset: qualityPreset as 'low' | 'medium' | 'high' }; + service.updateSettings(newSettings); + + // Update store and notify server + store.set('workspace.camera', newSettings); + setCameraState(newSettings); + sendControllerCommand('camera:updateSettings', newSettings); + }); + + const handleFrameRateChange = withCameraService((service, frameRate: string | number) => { + const constraints = { ...settings.constraints, frameRate: Number(frameRate) }; + const newSettings = { ...settings, constraints }; + service.updateSettings(newSettings); + + // Update store and notify server + store.set('workspace.camera', newSettings); + setCameraState(newSettings); + sendControllerCommand('camera:updateSettings', newSettings); + }); + + const deviceOptions = devices.map(device => ({ + value: device.deviceId, + label: device.label || `Camera ${device.deviceId.slice(0, 8)}...`, + })); + + const hasDevices = devices.length > 0; + const canStream = hasDevices && settings.deviceId; + + // Show loading state while checking headless mode + if (isCheckingHeadless) { + return ( +
+
+
Loading camera settings...
+
+
+ ); + } + + // Show message if in headless mode + if (isHeadlessMode) { + return ( +
+
+
+

+ Camera streaming is managed by the main server +

+
+
+ +
+
+ +
+

+ Remote Client Mode +

+

+ This device is running in Remote Client mode and is connected to an external gSender server. + Camera streaming can only be configured on the main server device. +

+

+ If you want to control camera settings, please access the Camera page on the main server device + (the one physically connected to your CNC machine). +

+
+
+
+
+ ); + } + + return ( +
+
+
+

+ Stream your camera to remote clients for workflow monitoring +

+
+
+ + {status.error && ( +
+
+ + Error +
+

{status.error}

+
+ )} + +
+
+
+
+ +

+ When enabled, streaming starts automatically on app launch +

+
+ +
+ +
+ + {!hasDevices ? ( +
+ No camera devices detected. Please connect a camera and refresh the page. +
+ ) : ( + + )} +
+ +
+
+ + +
+
+ + +
+
+
+
+ + {canStream && ( +
+

Live Preview

+
+
+
+ )} + + {status.streaming && ( +
+

Streaming Status

+
+
+
{status.metrics.fps}
+
FPS
+
+
+
{status.metrics.bitrateKbps}
+
kbps
+
+
+
{status.metrics.viewers}
+
Viewers
+
+
+
{status.metrics.droppedFrames}
+
Dropped
+
+
+
+ )} + +
+
+ +
+

Getting Started

+
    +
  • • Select a camera device and enable streaming
    • +
  • • Remote clients on your network can view the stream on their workflow page
    • +
  • • First-time access may require accepting a security certificate
    • +
  • • Streaming automatically starts when you launch the app (if enabled)
    • +
+
+
+
+
+ ); +}; + +export default Camera; diff --git a/src/app/src/features/RemoteCameraPanel/index.tsx b/src/app/src/features/RemoteCameraPanel/index.tsx new file mode 100644 index 000000000..35a951d16 --- /dev/null +++ b/src/app/src/features/RemoteCameraPanel/index.tsx @@ -0,0 +1,469 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import React, { useState, useEffect, useRef } from 'react'; +import { FaCamera, FaExclamationTriangle } from 'react-icons/fa'; +import cx from 'classnames'; +import controller from 'app/lib/controller'; + +interface CameraStatus { + enabled: boolean; + available: boolean; + transport: string; + viewers: number; + constraints: { + width: number; + height: number; + frameRate: number; + }; +} + +interface ConnectionState { + state: 'disconnected' | 'connecting' | 'connected' | 'failed'; + error?: string; +} + +const RemoteCameraPanel: React.FC = () => { + + const [cameraStatus, setCameraStatus] = useState(null); + const [connectionState, setConnectionState] = useState({ state: 'disconnected' }); + const [peerConnection, setPeerConnection] = useState(null); + const peerConnectionRef = useRef(null); + const videoRef = useRef(null); + const streamRef = useRef(null); + + // Keep ref in sync with state + useEffect(() => { + peerConnectionRef.current = peerConnection; + }, [peerConnection]); + + /** + * Sets up unmute event handlers for initially muted video tracks. + * This handles cases where tracks are muted at connection time and need + * a video element reload when they become unmuted. + */ + const setupUnmuteHandlers = (stream: MediaStream) => { + const videoTracks = stream.getVideoTracks(); + const UNMUTE_TIMEOUT = 10000; // Stop waiting after 10 seconds + + videoTracks.forEach((track) => { + if (!track.muted) return; + + const handleUnmute = () => { + // Reload video element to display newly unmuted track + if (videoRef.current && streamRef.current) { + const currentStream = streamRef.current; + videoRef.current.srcObject = null; + videoRef.current.load(); + + // Restore stream and play after brief delay + setTimeout(() => { + if (videoRef.current) { + videoRef.current.srcObject = currentStream; + videoRef.current.play().catch((error) => { + console.debug('RemoteCameraPanel: Play after unmute failed:', error.message); + }); + } + }, 100); + } + + track.removeEventListener('unmute', handleUnmute); + }; + + track.addEventListener('unmute', handleUnmute); + + // Clean up listener if track doesn't unmute within timeout + setTimeout(() => { + track.removeEventListener('unmute', handleUnmute); + }, UNMUTE_TIMEOUT); + }); + }; + + useEffect(() => { + // Check camera status on mount with retry for socket connection + const checkWithRetry = async (attempts = 3) => { + await checkCameraStatus(); + + // If socket is not connected and we have attempts left, retry after a delay + if (!controller.socket?.connected && attempts > 1) { + setTimeout(() => checkWithRetry(attempts - 1), 1000); + } + }; + + checkWithRetry(); + + // Set up periodic auto-connect check + const autoConnectInterval = setInterval(() => { + // Check connection state using refs to avoid stale closures + const currentPeer = peerConnectionRef.current; + const shouldConnect = !currentPeer || currentPeer.connectionState === 'closed' || currentPeer.connectionState === 'failed'; + + // Try to connect if we don't have an active connection and camera is available + if (shouldConnect && controller.socket?.connected) { + // Check camera status first + checkCameraStatus(); + } + }, 2000); // Check every 2 seconds + + // Set up socket listeners + const handleCameraAvailability = (data: { available: boolean; transport: string }) => { + setCameraStatus(prev => prev ? { ...prev, available: data.available } : null); + + if (data.available && controller.socket?.connected) { + const currentPeer = peerConnectionRef.current; + const shouldConnect = !currentPeer || currentPeer.connectionState === 'closed' || currentPeer.connectionState === 'failed'; + if (shouldConnect) { + connectToCamera(); + } + } else if (!data.available) { + disconnectFromCamera(); + } + }; + + const handleCameraOffer = async (data: { sdp: string; clientId: string }) => { + // data.clientId is the MAIN client's socket ID (who we send answer/ICE candidates to) + const mainClientId = data.clientId; + + // If we already have a peer connection, check its state + const currentPeer = peerConnectionRef.current; + if (currentPeer) { + const state = currentPeer.signalingState; + const iceState = currentPeer.iceConnectionState; + + // If connection is active and working, ignore new offers + if ((state === 'stable' && (iceState === 'connected' || iceState === 'completed')) || + (state !== 'closed' && state !== 'stable')) { + return; + } + + // Only close if connection is failed or closed + if (state !== 'closed') { + currentPeer.close(); + } + setPeerConnection(null); + } + + try { + // Create a new peer connection + const pc = new RTCPeerConnection({ iceServers: [] }); + + // Set up handlers + pc.ontrack = (event) => { + if (videoRef.current && event.streams[0]) { + const stream = event.streams[0]; + + // Set video element properties and stream + videoRef.current.muted = true; + videoRef.current.volume = 1.0; + videoRef.current.srcObject = stream; + streamRef.current = stream; + + setConnectionState({ state: 'connected' }); + + // Attempt to auto-play + if (videoRef.current) { + videoRef.current.play().catch((error) => { + console.warn('[RemoteCamera] Video play failed:', error); + }); + } + } else { + console.warn('[RemoteCamera] ontrack event but no video ref or stream'); + } + }; + + pc.onicecandidate = (event) => { + if (event.candidate && controller.socket) { + controller.socket.emit('camera:ice', { + candidate: event.candidate.toJSON(), + clientId: mainClientId // Send to MAIN client + }); + } + }; + + pc.oniceconnectionstatechange = () => { + const state = pc.iceConnectionState; + if (state === 'failed' || state === 'disconnected') { + setConnectionState({ state: 'failed', error: `Connection ${state}` }); + } + }; + + // Update state with new peer connection + setPeerConnection(pc); + + // Handle the offer + await pc.setRemoteDescription(new RTCSessionDescription({ + type: 'offer', + sdp: data.sdp + })); + + const answer = await pc.createAnswer(); + await pc.setLocalDescription(answer); + + controller.socket.emit('camera:answer', { + sdp: answer.sdp, + clientId: mainClientId // Send to MAIN client + }); + } catch (error) { + console.error('RemoteCameraPanel: Failed to handle camera offer:', error); + setConnectionState({ state: 'failed', error: error instanceof Error ? error.message : 'Unknown error' }); + } + }; + + const handleCameraAnswer = async (data: { sdp: string; clientId: string }) => { + const currentPeer = peerConnectionRef.current; + if (!currentPeer) return; + + try { + await currentPeer.setRemoteDescription(new RTCSessionDescription({ + type: 'answer', + sdp: data.sdp + })); + } catch (error) { + console.error('Failed to handle camera answer:', error); + setConnectionState({ state: 'failed', error: error instanceof Error ? error.message : 'Unknown error' }); + } + }; + + const handleCameraIce = async (data: { candidate: RTCIceCandidateInit; clientId: string }) => { + const currentPeer = peerConnectionRef.current; + if (!currentPeer) { + console.warn('[RemoteCamera] No peer connection available for ICE candidate'); + return; + } + + // Check if peer connection is in a valid state for ICE candidates + const state = currentPeer.signalingState; + if (state === 'closed' || currentPeer.remoteDescription === null) { + // Silently ignore ICE candidates that arrive before the connection is ready + return; + } + + try { + await currentPeer.addIceCandidate(new RTCIceCandidate(data.candidate)); + } catch (error) { + console.error('[RemoteCamera] Failed to add ICE candidate:', error); + } + }; + + const handleCameraError = (data: { message: string }) => { + setConnectionState({ state: 'failed', error: data.message }); + }; + + const handleSocketConnect = () => { + // Reset connection state when socket reconnects + setConnectionState({ state: 'disconnected' }); + + // Check camera status after reconnection + checkCameraStatus(); + }; + + controller.socket.on('camera:availability', handleCameraAvailability); + controller.socket.on('camera:offer', handleCameraOffer); + controller.socket.on('camera:answer', handleCameraAnswer); + controller.socket.on('camera:ice', handleCameraIce); + controller.socket.on('camera:error', handleCameraError); + controller.socket.on('connect', handleSocketConnect); + + return () => { + clearInterval(autoConnectInterval); + + controller.socket.off('camera:availability', handleCameraAvailability); + controller.socket.off('camera:offer', handleCameraOffer); + controller.socket.off('camera:answer', handleCameraAnswer); + controller.socket.off('camera:ice', handleCameraIce); + controller.socket.off('camera:error', handleCameraError); + controller.socket.off('connect', handleSocketConnect); + + // Clean up peer connection on unmount + const currentPeer = peerConnectionRef.current; + if (currentPeer && currentPeer.connectionState !== 'closed') { + currentPeer.close(); + setPeerConnection(null); + + // Notify server that we're disconnecting + if (controller.socket?.connected) { + controller.socket.emit('camera:viewerDisconnect'); + } + } + + // Clean up video element + if (videoRef.current) { + videoRef.current.srcObject = null; + } + streamRef.current = null; + }; + }, []); + + const checkCameraStatus = async () => { + try { + const response = await fetch('/api/camera/status'); + if (response.ok) { + const status = await response.json(); + setCameraStatus(status); + + // Check if we should initiate connection + if (status.available && controller.socket?.connected) { + const currentPeer = peerConnectionRef.current; + const shouldConnect = !currentPeer || currentPeer.connectionState === 'closed' || currentPeer.connectionState === 'failed'; + + if (shouldConnect) { + connectToCamera(); + } + } + } else { + console.warn('RemoteCameraPanel: Failed to fetch camera status:', response.status); + } + } catch (error) { + console.warn('RemoteCameraPanel: Failed to check camera status:', error); + } + }; + + const connectToCamera = async () => { + // Use refs to check current state instead of closure values + const currentPeer = peerConnectionRef.current; + + // Prevent multiple simultaneous connection attempts + if (currentPeer && currentPeer.connectionState !== 'closed' && currentPeer.connectionState !== 'failed') { + return; + } + + // Check socket connection first + if (!controller.socket?.connected) { + console.error('[RemoteCamera] Cannot request stream - socket not connected'); + setConnectionState({ state: 'failed', error: 'Socket not connected' }); + return; + } + + setConnectionState({ state: 'connecting' }); + + try { + // Remote client requests to join the stream + // The peer connection will be created when we receive the offer from the main client + controller.socket.emit('camera:requestStream', { + clientId: controller.socket.id + }); + + } catch (error) { + console.error('[RemoteCamera] Error in connectToCamera:', error); + setConnectionState({ state: 'failed', error: error instanceof Error ? error.message : 'Unknown error' }); + } + }; + + const disconnectFromCamera = () => { + const currentPeer = peerConnectionRef.current; + + if (currentPeer && currentPeer.connectionState !== 'closed') { + currentPeer.close(); + setPeerConnection(null); + } + + if (videoRef.current) { + videoRef.current.srcObject = null; + } + + streamRef.current = null; + setConnectionState({ state: 'disconnected' }); + }; + + // Always render the panel to show status + const renderConnectionStatus = () => { + // If camera status is not available + if (!cameraStatus?.available) { + return ( + + Waiting for stream + + ); + } + + switch (connectionState.state) { + case 'connecting': + return ( + +
+ Connecting... + + ); + case 'failed': + return ( + + + Failed + + ); + case 'disconnected': + return ( + +
+ Initializing... + + ); + default: + return null; + } + }; + + return ( +
+
+
+ + Camera Stream + {connectionState.state === 'connected' ? ( + + LIVE + + ) : ( + renderConnectionStatus() + )} +
+
+ +
+
+
+ ); +}; + +export default RemoteCameraPanel; diff --git a/src/app/src/features/RemoteMode/ConditionalRemoteCameraPanel.tsx b/src/app/src/features/RemoteMode/ConditionalRemoteCameraPanel.tsx new file mode 100644 index 000000000..98ec960c4 --- /dev/null +++ b/src/app/src/features/RemoteMode/ConditionalRemoteCameraPanel.tsx @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import { useState, useEffect } from 'react'; +import isElectron from 'is-electron'; +import RemoteCameraPanel from 'app/features/RemoteCameraPanel'; + +/** + * Component to conditionally render RemoteCameraPanel based on streaming settings. + * Only shows camera panel when: + * 1. This is a remote client + * 2. Camera streaming is enabled on the server + */ +export const ConditionalRemoteCameraPanel = () => { + const [cameraAvailable, setCameraAvailable] = useState(false); + const [isRemoteClient, setIsRemoteClient] = useState(false); + const [isCheckingInitialStatus, setIsCheckingInitialStatus] = useState(true); + + useEffect(() => { + // Check if this is a remote client + // Main client = Electron app OR localhost browser + // Remote client = browser accessing external server (non-localhost) + const isMainClient = isElectron() || + window.location.hostname === 'localhost' || + window.location.hostname === '127.0.0.1' || + window.location.hostname === '0.0.0.0'; + + const isRemote = !isMainClient; + + setIsRemoteClient(isRemote); + + if (!isRemote) { + // If running locally (main server), don't show the camera panel here + setCameraAvailable(false); + setIsCheckingInitialStatus(false); + return; + } + + // Check server camera status via API + const checkCameraStatus = async () => { + try { + const response = await fetch('/api/camera/status'); + if (response.ok) { + const status = await response.json(); + // Show panel if camera is enabled and available for streaming + const isAvailable = status.enabled && status.available; + setCameraAvailable(isAvailable); + setIsCheckingInitialStatus(false); + } else { + console.warn('[ConditionalRemoteCameraPanel] Failed to fetch camera status:', response.status); + setCameraAvailable(false); + setIsCheckingInitialStatus(false); + } + } catch (error) { + console.warn('[ConditionalRemoteCameraPanel] Error fetching camera status:', error); + setCameraAvailable(false); + setIsCheckingInitialStatus(false); + } + }; + + // Check immediately + checkCameraStatus(); + + // Poll every 5 seconds to detect when camera becomes available + // Once available, keep rendering the panel (don't unmount it) + const interval = setInterval(async () => { + try { + const response = await fetch('/api/camera/status'); + if (response.ok) { + const status = await response.json(); + const isAvailable = status.enabled && status.available; + // Only update if changing from unavailable to available + // Never unmount once mounted by not setting to false + if (isAvailable && !cameraAvailable) { + setCameraAvailable(true); + } + } + } catch (error) { + // Silently ignore polling errors to keep component mounted + } + }, 10000); + + return () => { + clearInterval(interval); + }; + }, [cameraAvailable]); + + if (!isRemoteClient) { + return null; + } + + // Once camera is available, always render the panel (don't unmount it) + // The panel itself will handle connection states + if (!cameraAvailable && !isCheckingInitialStatus) { + return null; + } + + if (cameraAvailable) { + return ; + } + + return null; +}; + diff --git a/src/app/src/features/StatusIcons/index.tsx b/src/app/src/features/StatusIcons/index.tsx index f804d5729..dccbd4bde 100644 --- a/src/app/src/features/StatusIcons/index.tsx +++ b/src/app/src/features/StatusIcons/index.tsx @@ -1,7 +1,7 @@ import { useEffect, useState } from 'react'; import cx from 'classnames'; import { Link } from 'react-router'; -import { LuGamepad2 } from 'react-icons/lu'; +import { LuGamepad2, LuVideo } from 'react-icons/lu'; import { FaRegKeyboard } from 'react-icons/fa6'; import { RemoteModeDialog } from 'app/features/RemoteMode'; @@ -10,6 +10,7 @@ import actions, { } from 'app/features/RemoteMode/apiActions.ts'; import RemoteIndicator from 'app/features/RemoteMode/components/RemoteIndicator.tsx'; import Tooltip from 'app/components/Tooltip'; +import store from 'app/store'; const StatusIcons = () => { const [gamepadConnected, setGamePadConnected] = useState(false); @@ -19,6 +20,7 @@ const StatusIcons = () => { headlessStatus: false, }); const [showRemoteDialog, setShowRemoteDialog] = useState(false); + const [cameraStreamingEnabled, setCameraStreamingEnabled] = useState(false); function toggleRemoteModeDialog(e: React.MouseEvent) { e.preventDefault(); @@ -28,6 +30,23 @@ const StatusIcons = () => { useEffect(() => { actions.fetchSettings(setHeadlessSettings); + // Load initial camera streaming state + const loadCameraState = () => { + const cameraSettings = store.get('workspace.camera', { enabled: false }); + setCameraStreamingEnabled(cameraSettings.enabled || false); + }; + + loadCameraState(); + + // Set up store listener for camera state changes + const handleStoreChange = () => { + const cameraSettings = store.get('workspace.camera', { enabled: false }); + setCameraStreamingEnabled(cameraSettings.enabled || false); + }; + + // Listen for store changes + store.on('change', handleStoreChange); + const gameConnectHandler = () => { const gamepads = navigator.getGamepads(); const hasGamepad = gamepads.some((gamepad) => gamepad !== null); @@ -46,6 +65,7 @@ const StatusIcons = () => { window.addEventListener('gamepaddisconnected', gameDisconnectHandler); return () => { + store.off('change', handleStoreChange); window.removeEventListener('gamepadconnected', gameConnectHandler); window.removeEventListener( 'gamepaddisconnected', @@ -90,6 +110,17 @@ const StatusIcons = () => { /> + + + + + setShowRemoteDialog(false)} diff --git a/src/app/src/features/Visualizer/index.tsx b/src/app/src/features/Visualizer/index.tsx index ad2e36eb5..a3464d9ae 100644 --- a/src/app/src/features/Visualizer/index.tsx +++ b/src/app/src/features/Visualizer/index.tsx @@ -108,6 +108,13 @@ class Visualizer extends Component { state = this.getInitialState(); + displayWebGLErrorMessage = () => { + const errorElement = WebGL.getWebGLErrorMessage(); + toast.error('WebGL is not available. Please ensure your graphics drivers are up to date and WebGL is enabled in your browser settings.'); + // Optionally display the error element in a modal or append it to the DOM + console.error('WebGL Error:', errorElement.innerHTML); + }; + actions = { dismissNotification: () => { this.setState((state) => ({ @@ -419,7 +426,7 @@ class Visualizer extends Component { }, toggle3DView: () => { if (!WebGL.isWebGLAvailable() && this.state.disabled) { - displayWebGLErrorMessage(); + this.displayWebGLErrorMessage(); return; } @@ -687,7 +694,7 @@ class Visualizer extends Component { this.subscribe(); if (!WebGL.isWebGLAvailable() && !this.state.disabled) { - displayWebGLErrorMessage(); + this.displayWebGLErrorMessage(); setTimeout(() => { this.setState((state) => ({ diff --git a/src/app/src/lib/camera/globalCameraService.ts b/src/app/src/lib/camera/globalCameraService.ts new file mode 100644 index 000000000..6327d105b --- /dev/null +++ b/src/app/src/lib/camera/globalCameraService.ts @@ -0,0 +1,135 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import CameraService from 'app/services/CameraService'; +import store from 'app/store'; +import controller from 'app/lib/controller'; +import { initializeCameraWebRTC } from './webrtcHandlers'; + +// Global camera service instance +let globalCameraService: CameraService | null = null; + +// Initialize the global camera service +export const initializeGlobalCameraService = async (): Promise => { + if (globalCameraService) { + return globalCameraService; + } + + globalCameraService = new CameraService(); + + try { + // Load settings from store FIRST to check if camera is enabled + const storedSettings = store.get('workspace.camera'); + + if (storedSettings) { + globalCameraService.updateSettings(storedSettings); + } + + // Only initialize (request permission) if camera is enabled + // This prevents requesting permission on first launch when feature is disabled + if (storedSettings?.enabled) { + // Initialize the camera service (enumerate devices and request permission) + await globalCameraService.initialize(); + } + + // Initialize WebRTC handlers AFTER settings are loaded + initializeCameraWebRTC(globalCameraService); + + // Auto-start streaming if enabled and device is selected + if (storedSettings?.enabled && storedSettings?.deviceId) { + try { + // Re-enumerate devices to ensure we have current device IDs + await globalCameraService.enumerateDevices(); + const availableDevices = globalCameraService.getDevices(); + + // Check if the stored deviceId still exists + const deviceExists = availableDevices.some(device => device.deviceId === storedSettings.deviceId); + + if (!deviceExists && availableDevices.length > 0) { + console.warn('[GlobalCameraService] Stored device no longer available, using first available device'); + // Update to first available device + const firstDevice = availableDevices[0]; + const updatedSettings = { + ...storedSettings, + deviceId: firstDevice.deviceId, + }; + globalCameraService.updateSettings(updatedSettings); + store.set('workspace.camera', updatedSettings); + } else if (!deviceExists) { + console.warn('[GlobalCameraService] No camera devices available, skipping auto-start'); + return globalCameraService; + } + + console.log('[GlobalCameraService] Attempting to auto-start camera streaming with device:', globalCameraService.getSettings().deviceId); + await globalCameraService.startStream(); + console.log('[GlobalCameraService] Camera streaming auto-started successfully'); + + // Notify server about the streaming status + if (controller.socket?.connected) { + // Send startStream command to mark stream as available on server + controller.socket.emit('camera:startStream'); + + // Also send settings + const updatedSettings = globalCameraService.getSettings(); + controller.socket.emit('camera:updateSettings', updatedSettings); + } else { + console.warn('[GlobalCameraService] Socket not connected, will notify server when connection is established'); + } + } catch (error) { + console.error('[GlobalCameraService] Failed to auto-start camera streaming:', error); + + // Extract user-friendly error message + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + + // Disable camera streaming in settings since auto-start failed + globalCameraService.updateSettings({ ...globalCameraService.getSettings(), enabled: false }); + store.set('workspace.camera.enabled', false); + + // Log warning to help user understand what happened + console.warn( + '[GlobalCameraService] Camera streaming disabled due to auto-start failure.', + 'You can manually enable it from the Camera page.', + 'Error details:', errorMessage + ); + } + } + + return globalCameraService; + } catch (error) { + console.error('[GlobalCameraService] Failed to initialize:', error); + throw error; + } +}; + +// Get the global camera service instance +export const getGlobalCameraService = (): CameraService | null => { + return globalCameraService; +}; + +// Cleanup function (for completeness) +export const cleanupGlobalCameraService = (): void => { + if (globalCameraService) { + globalCameraService.stopStream(); + globalCameraService = null; + } +}; diff --git a/src/app/src/lib/camera/webrtcHandlers.ts b/src/app/src/lib/camera/webrtcHandlers.ts new file mode 100644 index 000000000..a6fe7c795 --- /dev/null +++ b/src/app/src/lib/camera/webrtcHandlers.ts @@ -0,0 +1,192 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import controller from 'app/lib/controller'; +import CameraService from 'app/services/CameraService'; + +// Global camera service instance +let cameraService: CameraService | null = null; + +// Track active peer connections by client ID to prevent duplicates +const activePeerConnections = new Map(); + +// WebRTC handlers that need to persist globally +const handleStreamRequest = async (data: { requesterId: string }) => { + if (!cameraService) { + console.warn('[MainCamera] No camera service available for stream request'); + return; + } + + // Check if we already have an active connection for this client + const existingPeer = activePeerConnections.get(data.requesterId); + if (existingPeer) { + // Always close and recreate connection for new requests + // This ensures reconnections work as fast as first load + existingPeer.close(); + activePeerConnections.delete(data.requesterId); + } + + const freshStatus = cameraService.getStatus(); + + if (!freshStatus.streaming) { + console.warn('[MainCamera] Not streaming, ignoring stream request from', data.requesterId); + return; + } + + try { + // Create peer connection for the requesting client + const peerConnection = await cameraService.createPeerConnection(); + + // Store the peer connection for this client + activePeerConnections.set(data.requesterId, peerConnection); + + // Listen for ICE candidates on this specific peer connection, not the service + peerConnection.onicecandidate = (event) => { + if (event.candidate && controller.socket?.connected) { + controller.socket.emit('camera:ice', { + candidate: event.candidate.toJSON(), + clientId: data.requesterId + }); + } + }; + + // Create and send offer + const offer = await peerConnection.createOffer(); + await peerConnection.setLocalDescription(offer); + + if (controller.socket?.connected) { + controller.socket.emit('camera:offer', { + sdp: offer.sdp, + clientId: data.requesterId + }); + } else { + console.error('[MainCamera] Socket not connected, cannot send offer'); + } + } catch (error) { + console.error('[MainCamera] Failed to create offer for stream request:', error); + } +}; + +const handleCameraAnswer = async (data: { sdp: string; clientId: string }) => { + if (!cameraService) { + console.warn('[MainCamera] No camera service available for answer'); + return; + } + + // Get the peer connection for this specific client + const peerConnection = activePeerConnections.get(data.clientId); + if (!peerConnection) { + console.warn('[MainCamera] No peer connection found for client:', data.clientId); + return; + } + + try { + const state = peerConnection.signalingState; + + // Only set remote description if we're expecting an answer + if (state !== 'have-local-offer') { + console.warn('[MainCamera] Cannot set answer - peer connection in wrong state:', state); + return; + } + + await peerConnection.setRemoteDescription(new RTCSessionDescription({ + type: 'answer', + sdp: data.sdp + })); + } catch (error) { + console.error('[MainCamera] Failed to handle camera answer:', error); + } +}; + +const handleCameraIce = async (data: { candidate: RTCIceCandidateInit; clientId: string }) => { + if (!cameraService) { + console.warn('[MainCamera] No camera service available for ICE candidate'); + return; + } + + // Get the peer connection for this specific client + const peerConnection = activePeerConnections.get(data.clientId); + if (!peerConnection) { + console.warn('[MainCamera] No peer connection found for client:', data.clientId); + return; + } + + try { + await peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate)); + } catch (error) { + console.error('[MainCamera] Failed to add ICE candidate:', error); + } +}; + +// Handle viewer disconnect - clean up peer connections for that viewer +const handleViewerDisconnect = (data: { viewerId: string }) => { + const peerConnection = activePeerConnections.get(data.viewerId); + if (peerConnection) { + peerConnection.close(); + activePeerConnections.delete(data.viewerId); + } +}; + +// Initialize global camera WebRTC handlers +export const initializeCameraWebRTC = (cameraServiceInstance: CameraService) => { + cameraService = cameraServiceInstance; + + // Register socket event handlers that will persist globally + if (controller.socket?.connected) { + registerHandlers(); + } + + // Also register on future socket connections + controller.addListener('connect', registerHandlers); +}; + +// Register the handlers with the socket +const registerHandlers = () => { + if (!controller.socket?.connected) { + return; + } + + // Remove existing handlers first to avoid duplicates + controller.socket.off('camera:streamRequest', handleStreamRequest); + controller.socket.off('camera:answer', handleCameraAnswer); + controller.socket.off('camera:ice', handleCameraIce); + controller.socket.off('camera:viewerDisconnected', handleViewerDisconnect); + + // Register the handlers + controller.socket.on('camera:streamRequest', handleStreamRequest); + controller.socket.on('camera:answer', handleCameraAnswer); + controller.socket.on('camera:ice', handleCameraIce); + controller.socket.on('camera:viewerDisconnected', handleViewerDisconnect); +}; + +// Cleanup function (optional, for completeness) +export const cleanupCameraWebRTC = () => { + if (controller.socket) { + controller.socket.off('camera:streamRequest', handleStreamRequest); + controller.socket.off('camera:answer', handleCameraAnswer); + controller.socket.off('camera:ice', handleCameraIce); + } + + controller.removeListener('connect', registerHandlers); + cameraService = null; +}; diff --git a/src/app/src/lib/controller.ts b/src/app/src/lib/controller.ts index 6110e9668..ba4376edb 100644 --- a/src/app/src/lib/controller.ts +++ b/src/app/src/lib/controller.ts @@ -522,6 +522,13 @@ class Controller { // controller.command('watchdir:load', '/path/to/file', callback) command(cmd: string, ...args: Array): void { const { port } = this; + + // Handle camera commands specially (they don't need a port) + if (cmd.startsWith('camera:')) { + this.socket && this.socket.emit(cmd, ...args); + return; + } + if (!port) { return; } diff --git a/src/app/src/lib/controller/camera.ts b/src/app/src/lib/controller/camera.ts new file mode 100644 index 000000000..46ef54740 --- /dev/null +++ b/src/app/src/lib/controller/camera.ts @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import store from 'app/store'; +import { CameraSettings } from 'app/services/CameraService'; + +export const updateCameraSettings = (settings: Partial) => { + const currentSettings = store.get('workspace.camera', {}); + const newSettings = { ...currentSettings, ...settings }; + + store.set('workspace.camera', newSettings); + + return newSettings; +}; + +export const getCameraSettings = (): CameraSettings => { + return store.get('workspace.camera', { + enabled: false, + deviceId: '', + constraints: { + width: 1280, + height: 720, + frameRate: 30, + }, + qualityPreset: 'medium' as const, + }); +}; + diff --git a/src/app/src/react-routes.tsx b/src/app/src/react-routes.tsx index 2a676f18b..914d69641 100644 --- a/src/app/src/react-routes.tsx +++ b/src/app/src/react-routes.tsx @@ -12,7 +12,7 @@ import { StatParent } from './features/Stats/StatParent'; import Surfacing from './features/Surfacing'; import ToolCard from './components/ToolCard'; import { GiFlatPlatform } from 'react-icons/gi'; -import { FaGamepad, FaKeyboard, FaMicrochip } from 'react-icons/fa'; +import { FaGamepad, FaKeyboard, FaMicrochip, FaVideo } from 'react-icons/fa'; import { TbRulerMeasure } from 'react-icons/tb'; import { MdSquareFoot } from 'react-icons/md'; import { Alarms } from './features/Stats/Alarms'; @@ -26,8 +26,6 @@ import { WorkspaceSelector } from './features/WorkspaceSelector'; import DRO from './features/DRO'; import { RemoteWidget } from './components/RemoteWidget'; import Coolant from './features/Coolant'; -import FileControl from './features/FileControl'; -import JobControl from './features/JobControl'; import { Jogging } from './features/Jogging'; import Macros from './features/Macros'; import Probe from './features/Probe'; @@ -40,6 +38,10 @@ import { TopBar } from 'app/workspace/TopBar'; import Console from 'app/features/Console'; import Profile from './features/Gamepad/Profile'; import RotarySurfacing from './features/Rotary/RotarySurfacing'; +import Camera from './features/Camera'; +import FileControl from './features/FileControl'; +import JobControl from './features/JobControl'; +import { ConditionalRemoteCameraPanel } from './features/RemoteMode/ConditionalRemoteCameraPanel'; export const ReactRoutes = () => { return ( @@ -113,6 +115,13 @@ export const ReactRoutes = () => { link="/tools/gamepad" /> + + { } /> + + + + } + /> } + element={} /> { withFixedArea >
- + {}} />
} @@ -293,8 +315,9 @@ export const ReactRoutes = () => { +
+
} diff --git a/src/app/src/services/CameraService.ts b/src/app/src/services/CameraService.ts new file mode 100644 index 000000000..ed91aed4f --- /dev/null +++ b/src/app/src/services/CameraService.ts @@ -0,0 +1,538 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import { EventEmitter } from 'events'; + +export interface CameraConstraints { + width: number; + height: number; + frameRate: number; +} + +export interface CameraSettings { + enabled: boolean; + deviceId: string; + constraints: CameraConstraints; + qualityPreset: 'low' | 'medium' | 'high'; +} + +export interface CameraMetrics { + fps: number; + bitrateKbps: number; + viewers: number; + droppedFrames: number; +} + +export interface CameraStatus { + enabled: boolean; + available: boolean; + streaming: boolean; + error: string | null; + metrics: CameraMetrics; +} + +export const QUALITY_PRESETS = { + low: { width: 640, height: 480, frameRate: 15, bitrate: 500 }, + medium: { width: 1280, height: 720, frameRate: 30, bitrate: 1500 }, + high: { width: 1920, height: 1080, frameRate: 30, bitrate: 3000 }, +}; + +class CameraService extends EventEmitter { + private stream: MediaStream | null = null; + private peerConnection: RTCPeerConnection | null = null; + private settings: CameraSettings; + private status: CameraStatus; + private devices: MediaDeviceInfo[] = []; + private metricsInterval: NodeJS.Timeout | null = null; + private lastBytesReceived = 0; + private lastTimestamp = 0; + + constructor() { + super(); + this.settings = { + enabled: false, + deviceId: '', + constraints: QUALITY_PRESETS.medium, + qualityPreset: 'medium', + }; + this.status = { + enabled: false, + available: false, + streaming: false, + error: null, + metrics: { + fps: 0, + bitrateKbps: 0, + viewers: 0, + droppedFrames: 0, + }, + }; + } + + /** + * Initializes the camera service by requesting permissions and enumerating devices. + * This method should be called once before attempting to start streaming. + * + * @throws Will set an error status if initialization fails + * @emits initialized - When initialization completes successfully + * @emits devicesChanged - When camera devices are enumerated + */ + async initialize(): Promise { + try { + // Request temporary camera access to trigger permission prompt + // This allows enumerateDevices() to return device labels + try { + const tempStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: false }); + // Immediately stop the temporary stream + tempStream.getTracks().forEach(track => track.stop()); + } catch (permissionError) { + console.warn('[CameraService] Camera permission not granted, device labels may be unavailable:', permissionError); + // Continue anyway - devices will be enumerated but may have generic labels + } + + await this.enumerateDevices(); + this.emit('initialized'); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + this.setError(`Failed to initialize camera service: ${message}`); + } + } + + /** + * Enumerates all available video input devices (cameras). + * + * @returns Promise resolving to array of available camera devices + * @emits devicesChanged - When the device list is updated + */ + async enumerateDevices(): Promise { + try { + const devices = await navigator.mediaDevices.enumerateDevices(); + this.devices = devices.filter(device => device.kind === 'videoinput'); + this.emit('devicesChanged', this.devices); + return this.devices; + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + this.setError(`Failed to enumerate devices: ${message}`); + return []; + } + } + + /** + * Gets the list of available camera devices. + * @returns Array of video input device info + */ + getDevices(): MediaDeviceInfo[] { + return this.devices; + } + + /** + * Gets the current camera settings. + * @returns Copy of current settings object + */ + getSettings(): CameraSettings { + return { ...this.settings }; + } + + /** + * Gets the current camera status including streaming state and metrics. + * @returns Copy of current status object + */ + getStatus(): CameraStatus { + return { ...this.status }; + } + + /** + * Updates camera settings. If streaming is active and device/constraints change, + * automatically restarts the stream with new settings. + * + * @param newSettings - Partial settings object to merge with current settings + * @emits settingsChanged - When settings are updated + * @emits statusChanged - When enabled state changes + */ + updateSettings(newSettings: Partial): void { + const oldSettings = { ...this.settings }; + this.settings = { ...this.settings, ...newSettings }; + + // Apply quality preset if changed + if (newSettings.qualityPreset && newSettings.qualityPreset !== oldSettings.qualityPreset) { + this.settings.constraints = { ...QUALITY_PRESETS[newSettings.qualityPreset] }; + } + + // Update status to match settings + if (newSettings.enabled !== undefined) { + this.status.enabled = newSettings.enabled; + } + + this.emit('settingsChanged', this.settings); + this.emit('statusChanged', this.status); + + // Restart streaming if device or constraints changed while streaming + if (this.status.streaming && ( + newSettings.deviceId !== oldSettings.deviceId || + JSON.stringify(newSettings.constraints) !== JSON.stringify(oldSettings.constraints) + )) { + this.restartStream(); + } + } + + /** + * Starts camera streaming with the currently configured device and settings. + * Acquires camera access and begins broadcasting to connected viewers. + * + * @throws Error if camera cannot be accessed or constraints cannot be satisfied + * @emits streamStarted - When stream starts successfully + * @emits statusChanged - When streaming status changes + */ + async startStream(): Promise { + if (this.status.streaming) { + return; + } + + try { + this.clearError(); + + if (!this.settings.deviceId) { + throw new Error('No camera device selected'); + } + + const constraints: MediaStreamConstraints = { + video: { + deviceId: { exact: this.settings.deviceId }, + width: { ideal: this.settings.constraints.width }, + height: { ideal: this.settings.constraints.height }, + frameRate: { ideal: this.settings.constraints.frameRate }, + }, + audio: false, + }; + + try { + this.stream = await navigator.mediaDevices.getUserMedia(constraints); + } catch (constraintError: any) { + // Try fallback device if specific device constraint fails + this.stream = await this.tryFallbackDevice(constraintError); + } + + this.status.streaming = true; + this.status.available = true; + + this.startMetricsCollection(); + this.emit('streamStarted', this.stream); + this.emit('statusChanged', this.status); + } catch (error) { + // Better error message extraction with user-friendly messages + let message = 'Unknown error'; + let userFriendlyMessage = ''; + + if (error instanceof Error) { + message = error.message; + + // Provide helpful messages for common errors + if (error.name === 'NotReadableError') { + userFriendlyMessage = 'Camera is already in use by another application. Please close other apps using the camera and try again.'; + } else if (error.name === 'NotAllowedError') { + userFriendlyMessage = 'Camera access denied. Please grant camera permission in your browser settings.'; + } else if (error.name === 'NotFoundError') { + userFriendlyMessage = 'Camera device not found. Please check that your camera is connected.'; + } else if (error.name === 'OverconstrainedError') { + userFriendlyMessage = 'Camera does not support the requested settings. Try a different resolution or frame rate.'; + } + } else if (error && typeof error === 'object' && 'name' in error) { + const errorWithMessage = error as { name: unknown; message?: string }; + message = `${errorWithMessage.name}: ${errorWithMessage.message || 'No message'}`; + } else if (typeof error === 'string') { + message = error; + } + + console.error('[CameraService] Failed to start stream:', error); + const errorMessage = userFriendlyMessage || `Failed to start camera stream: ${message}`; + this.setError(errorMessage); + throw error; + } + } + + /** + * Stops camera streaming and releases all camera resources. + * Closes peer connections and stops all media tracks. + * + * @emits streamStopped - When stream stops successfully + * @emits statusChanged - When streaming status changes + */ + async stopStream(): Promise { + if (!this.status.streaming) { + return; + } + + try { + this.stopMetricsCollection(); + + if (this.peerConnection) { + this.peerConnection.close(); + this.peerConnection = null; + } + + if (this.stream) { + this.stream.getTracks().forEach(track => track.stop()); + this.stream = null; + } + + this.status.streaming = false; + this.status.available = false; + this.status.metrics = { + fps: 0, + bitrateKbps: 0, + viewers: 0, + droppedFrames: 0, + }; + + this.emit('streamStopped'); + this.emit('statusChanged', this.status); + } catch (error) { + const message = error instanceof Error ? error.message : 'Unknown error'; + this.setError(`Failed to stop camera stream: ${message}`); + } + } + + /** + * Restarts the camera stream (stops then starts). + * Useful when settings change that require re-initialization. + */ + async restartStream(): Promise { + await this.stopStream(); + await this.startStream(); + } + + /** + * Gets the current MediaStream object. + * @returns Current camera stream or null if not streaming + */ + getStream(): MediaStream | null { + return this.stream; + } + + /** + * Creates a new RTCPeerConnection for streaming to a remote viewer. + * Adds the current camera stream tracks to the connection. + * + * @param iceServers - Optional ICE servers for peer connection (defaults to LAN-only) + * @returns Promise resolving to configured RTCPeerConnection + * @emits iceCandidate - When a new ICE candidate is generated + * @emits iceConnectionStateChange - When ICE connection state changes + */ + async createPeerConnection(iceServers: RTCIceServer[] = []): Promise { + const newPeerConnection = new RTCPeerConnection({ + iceServers, + }); + + // Add stream to peer connection + if (this.stream) { + this.stream.getTracks().forEach((track) => { + newPeerConnection.addTrack(track, this.stream!); + }); + } else { + console.warn('CameraService: No stream available when creating peer connection'); + } + + // Configure sender encodings for quality control + const sender = newPeerConnection.getSenders().find(s => s.track?.kind === 'video'); + if (sender) { + const params = sender.getParameters(); + + if (params.encodings.length === 0) { + params.encodings.push({}); + } + + const preset = QUALITY_PRESETS[this.settings.qualityPreset]; + params.encodings[0].maxBitrate = preset.bitrate * 1000; // Convert to bps + + await sender.setParameters(params); + } + + // Handle ICE candidates - send them to remote client via signaling server + newPeerConnection.onicecandidate = (event) => { + if (event.candidate) { + this.emit('iceCandidate', event.candidate); + } + }; + + newPeerConnection.oniceconnectionstatechange = () => { + const state = newPeerConnection.iceConnectionState; + this.emit('iceConnectionStateChange', state); + + if (state === 'connected' || state === 'completed') { + this.status.metrics.viewers = 1; + } else if (state === 'disconnected' || state === 'failed' || state === 'closed') { + this.status.metrics.viewers = 0; + } + + this.emit('statusChanged', this.status); + }; + + // Store reference to the main peer connection (for metrics) + if (!this.peerConnection) { + this.peerConnection = newPeerConnection; + } + + return newPeerConnection; + } + + getPeerConnection(): RTCPeerConnection | null { + return this.peerConnection; + } + + private startMetricsCollection(): void { + this.stopMetricsCollection(); + + this.metricsInterval = setInterval(async () => { + await this.updateMetrics(); + }, 1000); + } + + private stopMetricsCollection(): void { + if (this.metricsInterval) { + clearInterval(this.metricsInterval); + this.metricsInterval = null; + } + } + + private async updateMetrics(): Promise { + if (!this.stream || !this.status.streaming || !this.settings.enabled) { + return; + } + + try { + let fps = 0; + let bitrateKbps = 0; + let droppedFrames = 0; + let viewers = 0; + + // If we have a peer connection, get WebRTC stats + if (this.peerConnection) { + const stats = await this.peerConnection.getStats(); + stats.forEach(report => { + if (report.type === 'outbound-rtp' && report.mediaType === 'video') { + fps = report.framesPerSecond || 0; + + // Calculate bitrate from bytes sent + const currentTime = Date.now(); + const currentBytes = report.bytesSent || 0; + + if (this.lastTimestamp > 0) { + const timeDiff = (currentTime - this.lastTimestamp) / 1000; // seconds + const bytesDiff = currentBytes - this.lastBytesReceived; + bitrateKbps = Math.round((bytesDiff * 8) / (timeDiff * 1000)); // kbps + } + + this.lastBytesReceived = currentBytes; + this.lastTimestamp = currentTime; + droppedFrames = report.framesDropped || 0; + } + }); + + viewers = this.peerConnection.connectionState === 'connected' ? 1 : 0; + } else { + // No peer connection, but we can still show basic stream info + const videoTrack = this.stream.getVideoTracks()[0]; + if (videoTrack) { + const settings = videoTrack.getSettings(); + fps = settings.frameRate || this.settings.constraints.frameRate; + + // Estimate bitrate based on quality preset + const preset = QUALITY_PRESETS[this.settings.qualityPreset]; + bitrateKbps = preset.bitrate; + } + viewers = 0; + } + + this.status.metrics = { + fps, + bitrateKbps, + viewers, + droppedFrames, + }; + + this.emit('metricsUpdated', this.status.metrics); + } catch (error) { + console.error('CameraService: Failed to update camera metrics:', error); + } + } + + private setError(message: string): void { + this.status.error = message; + this.status.available = false; + this.emit('error', message); + this.emit('statusChanged', this.status); + } + + private clearError(): void { + this.status.error = null; + this.emit('statusChanged', this.status); + } + + /** + * Attempts to acquire camera stream using fallback constraints when specific device fails. + * This handles cases where the requested device is no longer available. + * + * @param constraintError - The error thrown by getUserMedia with exact device constraints + * @returns Promise resolving to a MediaStream from any available camera + * @throws The original error if it's not a device availability issue + */ + private async tryFallbackDevice(constraintError: any): Promise { + const canFallback = constraintError.name === 'OverconstrainedError' || + constraintError.name === 'NotFoundError'; + + if (!canFallback) { + throw constraintError; + } + + console.warn('[CameraService] Exact device constraint failed, trying with any available camera'); + + const fallbackConstraints: MediaStreamConstraints = { + video: { + width: { ideal: this.settings.constraints.width }, + height: { ideal: this.settings.constraints.height }, + frameRate: { ideal: this.settings.constraints.frameRate }, + }, + audio: false, + }; + + const stream = await navigator.mediaDevices.getUserMedia(fallbackConstraints); + + // Update the deviceId to the one actually being used + const videoTrack = stream.getVideoTracks()[0]; + if (videoTrack) { + const actualDeviceId = videoTrack.getSettings().deviceId; + if (actualDeviceId && actualDeviceId !== this.settings.deviceId) { + this.settings.deviceId = actualDeviceId; + } + } + + return stream; + } + + async cleanup(): Promise { + await this.stopStream(); + this.removeAllListeners(); + } +} + +export default CameraService; diff --git a/src/app/src/store/defaultState/index.ts b/src/app/src/store/defaultState/index.ts index fc46dbbf4..60bdcdab3 100644 --- a/src/app/src/store/defaultState/index.ts +++ b/src/app/src/store/defaultState/index.ts @@ -141,6 +141,16 @@ const defaultState: State = { terminal: { inputHistory: [], }, + camera: { + enabled: false, + deviceId: '', + constraints: { + width: 1280, + height: 720, + frameRate: 30, + }, + qualityPreset: 'medium' as 'low' | 'medium' | 'high', + }, mode: WORKSPACE_MODE.DEFAULT, rotaryAxis: { firmwareSettings: ROTARY_MODE_FIRMWARE_SETTINGS, diff --git a/src/app/src/store/index.ts b/src/app/src/store/index.ts index f8f1c3c57..43c49464c 100644 --- a/src/app/src/store/index.ts +++ b/src/app/src/store/index.ts @@ -54,7 +54,7 @@ const getConfig = (): string => { content = fs.readFileSync(userData!.path, 'utf8') || '{}'; } } else { - content = localStorage.getItem('sienci') || '{}'; + content = (typeof localStorage !== 'undefined' ? localStorage.getItem('sienci') : null) || '{}'; } if (content === '{}') { @@ -83,7 +83,9 @@ const persist = (data: StoreData): void => { const fs = window.require('fs'); // Use window.require to require fs module in Electron fs.writeFileSync(userData!.path, value); } else { - localStorage.setItem('sienci', value); + if (typeof localStorage !== 'undefined') { + localStorage.setItem('sienci', value); + } } } catch (e) { log.error(e); @@ -247,7 +249,9 @@ const backupPreviousState = (data: any): void => { const fs = window.require('fs'); // Use window.require to require fs module in Electron fs.writeFileSync(backupPath, value); } else { - localStorage.setItem('sienci-backup', value); + if (typeof localStorage !== 'undefined') { + localStorage.setItem('sienci-backup', value); + } } }; diff --git a/src/main.js b/src/main.js index 6a850d677..9b8aa7413 100644 --- a/src/main.js +++ b/src/main.js @@ -167,7 +167,9 @@ const main = () => { return; } - const url = `http://${address}:${port}`; + // Always use localhost for Electron to ensure secure context for camera access + // Even when server binds to 0.0.0.0 for remote access, Electron should connect via localhost + const url = `http://localhost:${port}`; // The bounds is a rectangle object with the following properties: // * `x` Number - The x coordinate of the origin of the rectangle. // * `y` Number - The y coordinate of the origin of the rectangle. diff --git a/src/server/api/api.camera.js b/src/server/api/api.camera.js new file mode 100644 index 000000000..dd9f275ee --- /dev/null +++ b/src/server/api/api.camera.js @@ -0,0 +1,136 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import { authorizeIPAddress } from '../access-control'; +import config from '../services/configstore'; +import logger from '../lib/logger'; + +const log = logger('api:camera'); + +// Camera streaming state +let cameraState = { + enabled: false, + available: false, + transport: 'webrtc', + viewers: 0, + constraints: { + width: 1280, + height: 720, + frameRate: 30, + }, +}; + +const isRemoteAccessAllowed = (req) => { + const clientIP = req.ip || req.connection.remoteAddress; + + // Always allow localhost + if (clientIP === '127.0.0.1' || clientIP === '::1') { + return true; + } + + // Allow local network addresses + const isLocalNetwork = + clientIP.startsWith('192.168.') || + clientIP.startsWith('10.') || + /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(clientIP); + + if (isLocalNetwork) { + return true; + } + + // For non-local networks, check remote access settings + const remoteSettings = config.get('remoteSettings', {}); + if (!remoteSettings.headlessStatus) { + return false; + } + + const allowedIP = remoteSettings.ip; + return clientIP === allowedIP || authorizeIPAddress(clientIP); +}; + +const checkOriginAndHost = (req) => { + const remoteSettings = config.get('remoteSettings', {}); + const allowedIP = remoteSettings.ip; + + const origin = req.get('Origin'); + + // Allow localhost for development + if (process.env.NODE_ENV === 'development') { + return true; + } + + // Check if origin matches allowed remote access configuration + if (origin) { + const originUrl = new URL(origin); + const isAllowedOrigin = originUrl.hostname === allowedIP || + originUrl.hostname === 'localhost' || + originUrl.hostname === '127.0.0.1'; + + if (!isAllowedOrigin) { + return false; + } + } + + return true; +}; + +export const getStatus = (req, res) => { + try { + const clientIP = req.ip || req.connection.remoteAddress; + // Reduced logging to prevent spam - only log non-localhost requests + if (clientIP !== '127.0.0.1' && clientIP !== '::1') { + // Only log every 10th request to reduce spam + if (!global.cameraStatusLogCounter) { + global.cameraStatusLogCounter = 0; + } + global.cameraStatusLogCounter++; + if (global.cameraStatusLogCounter % 10 === 0) { + log.debug(`Camera status request from IP: ${clientIP} (${global.cameraStatusLogCounter} total)`); + } + } + + // Check access control + if (!isRemoteAccessAllowed(req)) { + log.warn(`Camera status access denied for IP: ${clientIP}`); + return res.status(403).json({ error: 'Access denied' }); + } + + // Check origin and host headers + if (!checkOriginAndHost(req)) { + log.warn(`Camera status invalid origin for IP: ${clientIP}`); + return res.status(403).json({ error: 'Invalid origin' }); + } + + return res.json(cameraState); + } catch (error) { + log.error('Error getting camera status:', error); + return res.status(500).json({ error: 'Internal server error' }); + } +}; + +export const updateStatus = (newState) => { + cameraState = { ...cameraState, ...newState }; + log.debug('Camera state updated:', cameraState); +}; + +export const getCameraState = () => cameraState; diff --git a/src/server/api/index.js b/src/server/api/index.js index a50df09e7..18edcde94 100644 --- a/src/server/api/index.js +++ b/src/server/api/index.js @@ -40,6 +40,7 @@ import * as jobStats from './api.jobstats'; import * as maintenance from './api.maintenance'; import * as alarmList from './api.alarmList'; import * as releaseNotes from './api.releasenotes'; +import * as camera from './api.camera'; export { version, @@ -61,4 +62,5 @@ export { maintenance, alarmList, releaseNotes, + camera, }; diff --git a/src/server/app.js b/src/server/app.js index 7834bbf23..b6c961398 100644 --- a/src/server/app.js +++ b/src/server/app.js @@ -256,7 +256,8 @@ const appMain = () => { // Check whether the request path is not restricted const whitelist = [ // Also see "src/app/api/index.js" - urljoin(settings.route, 'api/signin') + urljoin(settings.route, 'api/signin'), + urljoin(settings.route, 'api/camera/status') ]; bypass = bypass || whitelist.some(path => { return req.path.indexOf(path) === 0; @@ -367,6 +368,9 @@ const appMain = () => { app.put(urljoin(settings.route, 'api/users/:id'), api.users.update); app.delete(urljoin(settings.route, 'api/users/:id'), api.users.__delete); + // Camera + app.get(urljoin(settings.route, 'api/camera/status'), api.camera.getStatus); + // Watch app.get(urljoin(settings.route, 'api/watch/files'), api.watch.getFiles); app.post(urljoin(settings.route, 'api/watch/files'), api.watch.getFiles); diff --git a/src/server/index.js b/src/server/index.js index 6997bd57f..5c763809f 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -145,9 +145,12 @@ const createServer = (options, callback) => { const setPort = get(remoteSettings, 'port', null); const setIP = get(remoteSettings, 'ip', null); + // When remote access is enabled, use the configured port but ALWAYS bind to 0.0.0.0 + // This allows the server to listen on all network interfaces (localhost + LAN) if (remoteSettings.headlessStatus && !isInDevelopmentMode && setIP && setPort && setIP !== '0.0.0.0' && remoteSettings.ip.length > 0) { port = setPort; - host = setIP; + host = '0.0.0.0'; // Bind to all interfaces so remote clients can connect + // The setIP is only used for display purposes (see callback below) } const mountPoints = uniqWith([ @@ -274,9 +277,15 @@ const createServer = (options, callback) => { // cncengine service cncengine.start(server, options.controller || config.get('controller', '')); - const address = server.address().address; + let address = server.address().address; const port = server.address().port; + // If we're bound to 0.0.0.0 and remote access is enabled, use the configured display IP + // This ensures the UI shows the correct remote access URL + if (address === '0.0.0.0' && remoteSettings.headlessStatus && setIP) { + address = setIP; + } + callback && callback(null, { address, port, diff --git a/src/server/services/camera/CameraSignaling.js b/src/server/services/camera/CameraSignaling.js new file mode 100644 index 000000000..24383e80a --- /dev/null +++ b/src/server/services/camera/CameraSignaling.js @@ -0,0 +1,311 @@ +/* + * Copyright (C) 2021 Sienci Labs Inc. + * + * This file is part of gSender. + * + * gSender is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, under version 3 of the License. + * + * gSender is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with gSender. If not, see . + * + * Contact for information regarding this program and its license + * can be sent through gSender@sienci.com or mailed to the main office + * of Sienci Labs Inc. in Waterloo, Ontario, Canada. + * + */ + +import { authorizeIPAddress } from '../../access-control'; +import config from '../configstore'; +import logger from '../../lib/logger'; +import { updateStatus, getCameraState } from '../../api/api.camera'; + +const log = logger('camera:signaling'); + +/** + * @typedef {Object} CameraOfferData + * @property {string} sdp - Session Description Protocol offer + * @property {string} clientId - ID of the target client + */ + +/** + * @typedef {Object} CameraAnswerData + * @property {string} sdp - Session Description Protocol answer + * @property {string} clientId - ID of the target client + */ + +/** + * @typedef {Object} CameraIceData + * @property {RTCIceCandidateInit} candidate - ICE candidate + * @property {string} clientId - ID of the target client + */ + +/** + * Camera signaling service for WebRTC connections between main client and remote viewers. + * Handles offer/answer exchange and ICE candidate forwarding for camera streaming. + */ +class CameraSignaling { + constructor() { + /** @type {import('socket.io').Server | null} */ + this.io = null; + + /** @type {Map} clientId -> socket */ + this.activeConnections = new Map(); + } + + /** + * Initializes the signaling service with a Socket.IO server instance. + * @param {import('socket.io').Server} io - Socket.IO server instance + */ + initialize(io) { + this.io = io; + this.setupSocketHandlers(); + log.info('Camera signaling service initialized'); + } + + /** + * Sets up socket event handlers for incoming connections. + * @private + */ + setupSocketHandlers() { + if (!this.io) { + return; + } + + this.io.on('connection', (socket) => { + this.handleConnection(socket); + }); + } + + /** + * Handles new client connections and sets up camera event listeners. + * @param {import('socket.io').Socket} socket - Connected socket + * @private + */ + handleConnection(socket) { + const clientIP = socket.handshake.address; + + // Check if client is authorized for camera access + if (!this.isClientAuthorized(clientIP)) { + log.warn(`Unauthorized camera access attempt from ${clientIP}`); + socket.disconnect(); + return; + } + + this.activeConnections.set(socket.id, socket); + + // Set up camera-specific event handlers + socket.on('camera:offer', (data) => this.handleOffer(socket, data)); + socket.on('camera:answer', (data) => this.handleAnswer(socket, data)); + socket.on('camera:ice', (data) => this.handleIceCandidate(socket, data)); + socket.on('camera:requestStream', (data) => this.handleStreamRequest(socket, data)); + socket.on('camera:requestStatus', () => this.handleStatusRequest(socket)); + socket.on('camera:viewerDisconnect', () => this.handleViewerDisconnect(socket)); + + socket.on('disconnect', () => { + this.handleDisconnection(socket); + }); + + // Send current camera availability + this.sendAvailability(socket); + } + + /** + * Handles explicit viewer disconnect event. + * Notifies main client to clean up peer connection. + * @param {import('socket.io').Socket} socket - Disconnecting socket + * @private + */ + handleViewerDisconnect(socket) { + // Broadcast to all clients (especially main client) that this viewer disconnected + this.io.emit('camera:viewerDisconnected', { + viewerId: socket.id + }); + } + + /** + * Handles client disconnection and cleans up connection tracking. + * @param {import('socket.io').Socket} socket - Disconnected socket + * @private + */ + handleDisconnection(socket) { + this.activeConnections.delete(socket.id); + } + + /** + * Handles WebRTC offer from main client (camera source) to remote viewer. + * Forwards the offer to the requesting client to establish peer connection. + * @param {import('socket.io').Socket} socket - Socket of the main client sending the offer + * @param {CameraOfferData} data - Offer data containing SDP and target client ID + * @private + */ + handleOffer(socket, data) { + const { sdp, clientId } = data; + + // Forward offer to the specific remote client that requested the stream + const targetSocket = this.activeConnections.get(clientId); + if (targetSocket) { + targetSocket.emit('camera:offer', { sdp, clientId: socket.id }); + } else { + socket.emit('camera:error', { message: 'Target client not found' }); + } + } + + /** + * Handles WebRTC answer from remote viewer to main client. + * Forwards the answer to complete peer connection establishment. + * @param {import('socket.io').Socket} socket - Socket of the remote client sending the answer + * @param {CameraAnswerData} data - Answer data containing SDP and target client ID + * @private + */ + handleAnswer(socket, data) { + const { sdp, clientId } = data; + + log.debug(`Received camera answer for client ${clientId}`); + + // Forward answer to the specific client + const targetSocket = this.activeConnections.get(clientId); + if (targetSocket) { + targetSocket.emit('camera:answer', { sdp, clientId: socket.id }); + } + } + + /** + * Handles ICE candidate exchange between peers. + * Forwards ICE candidates to establish optimal peer connection. + * @param {import('socket.io').Socket} socket - Socket sending the ICE candidate + * @param {CameraIceData} data - ICE candidate data + * @private + */ + handleIceCandidate(socket, data) { + const { candidate, clientId } = data; + + if (clientId) { + // Forward ICE candidate to specific client + const targetSocket = this.activeConnections.get(clientId); + if (targetSocket) { + targetSocket.emit('camera:ice', { candidate, clientId: socket.id }); + } + } else { + // Broadcast ICE candidate (for offers from main client) + socket.broadcast.emit('camera:ice', { candidate, clientId: socket.id }); + } + } + + /** + * Handles camera status requests from clients. + * @param {import('socket.io').Socket} socket - Requesting socket + * @private + */ + handleStatusRequest(socket) { + const status = getCameraState(); + socket.emit('camera:status', status); + } + + /** + * Handles stream requests from remote viewers. + * Broadcasts the request to all clients so the main client can respond with an offer. + * @param {import('socket.io').Socket} socket - Requesting socket + * @param {Object} data - Request data (currently unused) + * @private + */ + handleStreamRequest(socket, data) { + // Check if camera is available + const status = getCameraState(); + if (!status.available) { + socket.emit('camera:error', { message: 'Camera not available' }); + return; + } + + // Send the request to all clients (including the sender) + // The main client (with camera) should respond with an offer + this.io.emit('camera:streamRequest', { + requesterId: socket.id + }); + } + + /** + * Broadcasts camera availability status to all connected clients. + * @param {boolean} available - Whether camera is available for streaming + */ + broadcastAvailability(available) { + const status = { available, transport: 'webrtc' }; + this.io.emit('camera:availability', status); + updateStatus({ available }); + log.debug(`Broadcasting camera availability: ${available}`); + } + + /** + * Sends camera availability status to a specific client. + * @param {import('socket.io').Socket} socket - Target socket + * @private + */ + sendAvailability(socket) { + const status = getCameraState(); + socket.emit('camera:availability', { + available: status.available, + transport: status.transport + }); + } + + /** + * Updates and broadcasts the current viewer count. + * @param {number} count - Number of active viewers + * @private + */ + updateViewerCount(count) { + updateStatus({ viewers: count }); + this.io.emit('camera:metrics', { viewers: count }); + } + + /** + * Broadcasts camera streaming metrics to all clients. + * @param {Object} metrics - Streaming metrics (fps, bitrate, etc.) + */ + broadcastMetrics(metrics) { + this.io.emit('camera:metrics', metrics); + updateStatus({ ...getCameraState(), ...metrics }); + } + + /** + * Checks if a client IP address is authorized for camera access. + * Allows localhost and local network addresses by default. + * @param {string} clientIP - Client IP address + * @returns {boolean} Whether the client is authorized + * @private + */ + isClientAuthorized(clientIP) { + // Always allow localhost + if (clientIP === '127.0.0.1' || clientIP === '::1') { + return true; + } + + // Allow local network addresses + const isLocalNetwork = + clientIP.startsWith('192.168.') || + clientIP.startsWith('10.') || + /^172\.(1[6-9]|2[0-9]|3[0-1])\./.test(clientIP); + + if (isLocalNetwork) { + return true; + } + + // For non-local networks, check remote access settings + const remoteSettings = config.get('remoteSettings', {}); + if (!remoteSettings.headlessStatus) { + return false; + } + + // Allow configured remote IP + const allowedIP = remoteSettings.ip; + return clientIP === allowedIP || authorizeIPAddress(clientIP); + } +} + +export default CameraSignaling; diff --git a/src/server/services/cncengine/CNCEngine.js b/src/server/services/cncengine/CNCEngine.js index efd7f94ca..e171284da 100644 --- a/src/server/services/cncengine/CNCEngine.js +++ b/src/server/services/cncengine/CNCEngine.js @@ -43,6 +43,7 @@ import { GRBL } from '../../controllers/Grbl/constants'; import { GRBLHAL } from '../../controllers/Grblhal/constants'; import { authorizeIPAddress } from '../../access-control'; import DFUFlasher from '../../lib/Firmware/Flashing/DFUFlasher'; +import CameraSignaling from '../camera/CameraSignaling'; import delay from '../../lib/delay'; import SerialConnection from 'server/lib/SerialConnection'; import Connection from '../../lib/Connection'; @@ -124,6 +125,9 @@ class CNCEngine { } }); + // Camera Signaling + cameraSignaling = new CameraSignaling(); + // @param {object} server The HTTP server instance. // @param {string} controller Specify CNC controller. start(server, controller = '') { @@ -167,6 +171,9 @@ class CNCEngine { maxHttpBufferSize: 40e6 }); + // Initialize camera signaling + this.cameraSignaling.initialize(this.io); + this.io.use(async (socket, next) => { try { // IP Address Access Control @@ -681,6 +688,54 @@ class CNCEngine { log.debug('Socket unload called'); this.unload(); }); + + // Camera command handlers + socket.on('camera:updateSettings', (settings) => { + log.debug('Camera settings updated:', settings); + + // Update the camera API state + // Note: 'available' is NOT updated here - only by startStream/stopStream + const api = require('../../api/api.camera'); + api.updateStatus({ + enabled: settings.enabled, + constraints: settings.constraints + }); + + log.debug('Camera API state updated:', { + enabled: settings.enabled, + deviceId: settings.deviceId ? 'present' : 'missing' + }); + + socket.emit('camera:settingsUpdated', settings); + }); + + socket.on('camera:startStream', (settings) => { + log.debug('Camera stream start requested'); + + // Update API state to mark stream as available + const api = require('../../api/api.camera'); + api.updateStatus({ available: true }); + + // Broadcast to remote clients + this.cameraSignaling.broadcastAvailability(true); + socket.emit('camera:streamStarted'); + }); + + socket.on('camera:stopStream', () => { + log.debug('Camera stream stop requested'); + + // Update API state to mark stream as unavailable + const api = require('../../api/api.camera'); + api.updateStatus({ available: false }); + + // Broadcast to remote clients + this.cameraSignaling.broadcastAvailability(false); + socket.emit('camera:streamStopped'); + }); + + socket.on('camera:metrics', (metrics) => { + this.cameraSignaling.broadcastMetrics(metrics); + }); }); }