diff --git a/src/app/app.component.spec.ts b/src/app/app.component.spec.ts index faf10e60..990c4954 100644 --- a/src/app/app.component.spec.ts +++ b/src/app/app.component.spec.ts @@ -49,10 +49,12 @@ import {MockStreamChatService} from './core/services/testing/mock-stream-chat.se import {MockStringToColorService} from './core/services/testing/mock-string-to-color.service'; import {MockTraceService} from './core/services/testing/mock-trace.service'; import {MockVideoService} from './core/services/testing/mock-video.service'; +import {MockScreenSharingService} from './core/services/testing/mock-screensharing.service'; import {MockWebSocketService} from './core/services/testing/mock-websocket.service'; import {TRACE_SERVICE, TraceService} from './core/services/trace.service'; import {VIDEO_SERVICE, VideoService} from './core/services/video.service'; import {WEBSOCKET_SERVICE, WebSocketService,} from './core/services/websocket.service'; +import { SCREEN_SHARING_SERVICE } from './core/services/screensharing.service'; describe('AppComponent', () => { beforeEach(async () => { @@ -64,6 +66,7 @@ describe('AppComponent', () => { const audioService = new MockAudioService(); const webSocketService = new MockWebSocketService(); const videoService = new MockVideoService(); + const screenSharingService = new MockScreenSharingService(); const streamChatService = new MockStreamChatService(); const eventService = new MockEventService(); const downloadService = new MockDownloadService(); @@ -114,6 +117,10 @@ describe('AppComponent', () => { provide: VIDEO_SERVICE, useValue: videoService, }, + { + provide: SCREEN_SHARING_SERVICE, + useValue: screenSharingService, + }, { provide: STREAM_CHAT_SERVICE, useValue: streamChatService, diff --git a/src/app/components/chat-panel/chat-panel.component.html b/src/app/components/chat-panel/chat-panel.component.html index 97664029..2458a94a 100644 --- a/src/app/components/chat-panel/chat-panel.component.html +++ b/src/app/components/chat-panel/chat-panel.component.html @@ -17,6 +17,7 @@ @if (appName != "") {
+
@for (message of messages; track message; let i = $index) {
+ screen_share + +
diff --git a/src/app/components/chat-panel/chat-panel.component.i18n.ts b/src/app/components/chat-panel/chat-panel.component.i18n.ts index 379d947c..47935448 100644 --- a/src/app/components/chat-panel/chat-panel.component.i18n.ts +++ b/src/app/components/chat-panel/chat-panel.component.i18n.ts @@ -44,6 +44,8 @@ export const CHAT_PANEL_MESSAGES = { turnOffMicTooltip: 'Turn off microphone', useMicTooltip: 'Use microphone', turnOffCamTooltip: 'Turn off camera', + stopScreenShareTooltip: 'Turn off screensharing', + startScreenShareTooltip: 'Turn on screensharing', useCamTooltip: 'Use camera', updatedSessionStateChipLabel: 'Updated session state', }; diff --git a/src/app/components/chat-panel/chat-panel.component.ts b/src/app/components/chat-panel/chat-panel.component.ts index b95cbaed..9c888aaf 100644 --- a/src/app/components/chat-panel/chat-panel.component.ts +++ b/src/app/components/chat-panel/chat-panel.component.ts @@ -68,6 +68,7 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit { @Input() eventData = new Map(); @Input() isAudioRecording: boolean = false; @Input() isVideoRecording: boolean = false; + @Input() isScreenSharing: boolean = false; @Input() hoveredEventMessageIndices: number[] = []; @Output() readonly userInputChange = new EventEmitter(); @@ -94,8 +95,10 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit { @Output() readonly updateState = new EventEmitter(); @Output() readonly toggleAudioRecording = new EventEmitter(); @Output() readonly toggleVideoRecording = new EventEmitter(); + @Output() readonly toggleScreenSharing = new EventEmitter(); @ViewChild('videoContainer', {read: ElementRef}) videoContainer!: ElementRef; + @ViewChild('screenSharingContainer', {read: ElementRef}) screenSharingContainer!: ElementRef; @ViewChild('autoScroll') scrollContainer!: ElementRef; @ViewChild('messageTextarea') public textarea: ElementRef|undefined; scrollInterrupted = false; diff --git a/src/app/components/chat/chat.component.html b/src/app/components/chat/chat.component.html index 59c3a751..f115b559 100644 --- a/src/app/components/chat/chat.component.html +++ b/src/app/components/chat/chat.component.html @@ -223,6 +223,7 @@ [eventData]="eventData" [isAudioRecording]="isAudioRecording" [isVideoRecording]="isVideoRecording" + [isScreenSharing]="isScreenSharing" [hoveredEventMessageIndices]="hoveredEventMessageIndices" (clickEvent)="clickEvent($event)" (handleKeydown)="handleKeydown($event.event, $event.message)" @@ -240,6 +241,7 @@ (updateState)="updateState()" (toggleAudioRecording)="toggleAudioRecording()" (toggleVideoRecording)="toggleVideoRecording()" + (toggleScreenSharing)="toggleScreenSharing()" > } diff --git a/src/app/components/chat/chat.component.spec.ts b/src/app/components/chat/chat.component.spec.ts index 15dd91d3..e3c52ad3 100644 --- a/src/app/components/chat/chat.component.spec.ts +++ b/src/app/components/chat/chat.component.spec.ts @@ -55,9 +55,11 @@ import {MockStreamChatService} from '../../core/services/testing/mock-stream-cha import {MockStringToColorService} from '../../core/services/testing/mock-string-to-color.service'; import {MockTraceService} from '../../core/services/testing/mock-trace.service'; import {MockVideoService} from '../../core/services/testing/mock-video.service'; +import {MockScreenSharingService} from '../../core/services/testing/mock-screensharing.service'; import {MockWebSocketService} from '../../core/services/testing/mock-websocket.service'; import {TRACE_SERVICE, TraceService} from '../../core/services/trace.service'; import {VIDEO_SERVICE, VideoService} from '../../core/services/video.service'; +import {SCREEN_SHARING_SERVICE, ScreenSharingService,} from '../../core/services/screensharing.service'; import {WEBSOCKET_SERVICE, WebSocketService,} from '../../core/services/websocket.service'; import {fakeAsync, tick} from '../../testing/utils'; @@ -110,6 +112,7 @@ describe('ChatComponent', () => { let mockAudioService: MockAudioService; let mockWebSocketService: MockWebSocketService; let mockVideoService: MockVideoService; + let mockScreenSharingService: MockScreenSharingService; let mockStreamChatService: MockStreamChatService; let mockEventService: MockEventService; let mockDownloadService: MockDownloadService; @@ -132,6 +135,7 @@ describe('ChatComponent', () => { mockAudioService = new MockAudioService(); mockWebSocketService = new MockWebSocketService(); mockVideoService = new MockVideoService(); + mockScreenSharingService = new MockScreenSharingService(); mockStreamChatService = new MockStreamChatService(); mockEventService = new MockEventService(); mockDownloadService = new MockDownloadService(); @@ -200,6 +204,7 @@ describe('ChatComponent', () => { {provide: AUDIO_SERVICE, useValue: mockAudioService}, {provide: WEBSOCKET_SERVICE, useValue: mockWebSocketService}, {provide: VIDEO_SERVICE, useValue: mockVideoService}, + {provide: SCREEN_SHARING_SERVICE, useValue: mockScreenSharingService}, {provide: EVENT_SERVICE, useValue: mockEventService}, {provide: STREAM_CHAT_SERVICE, useValue: mockStreamChatService}, {provide: DOWNLOAD_SERVICE, useValue: mockDownloadService}, diff --git a/src/app/components/chat/chat.component.ts b/src/app/components/chat/chat.component.ts index 2d0b3ada..b9b543ae 100644 --- a/src/app/components/chat/chat.component.ts +++ b/src/app/components/chat/chat.component.ts @@ -169,6 +169,7 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { evalSetId = ''; isAudioRecording = false; isVideoRecording = false; + isScreenSharing = false; longRunningEvents: any[] = []; functionCallEventId = ''; redirectUri = URLUtil.getBaseUrlWithoutPath(); @@ -958,6 +959,11 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { this.stopVideoRecording(); this.isVideoRecording = false; } + if (this.isScreenSharing) { + this.stopScreenSharing(); + this.isScreenSharing = false; + } + this.evalTab()?.resetEvalResults(); this.traceData = []; this.bottomPanelVisible = false; @@ -1029,6 +1035,41 @@ export class ChatComponent implements OnInit, AfterViewInit, OnDestroy { this.isVideoRecording = false; } + toggleScreenSharing() { + this.isScreenSharing ? this.stopScreenSharing() : this.startScreenSharing(); + } + + startScreenSharing() { + if (this.sessionHasUsedBidi.has(this.sessionId)) { + this.openSnackBar(BIDI_STREAMING_RESTART_WARNING, 'OK') + return; + } + const screenSharingContainer = this.chatPanel()?.screenSharingContainer; + if (!screenSharingContainer) { + return; + } + this.isScreenSharing = true; + this.streamChatService.startScreenSharingChat({ + appName: this.appName, + userId: this.userId, + sessionId: this.sessionId, + screenSharingContainer, + }); + this.messages.update( + messages => [...messages, {role: 'user', text: 'Sharing Screen...'}]); + this.sessionHasUsedBidi.add(this.sessionId); + } + + stopScreenSharing() { + const screenSharingContainer = this.chatPanel()?.screenSharingContainer; + if (!screenSharingContainer) { + return; + } + + this.streamChatService.stopScreenSharingChat(screenSharingContainer); + this.isScreenSharing = false; + } + private getAsyncFunctionsFromParts(pendingIds: any[], parts: any[], invocationId: string) { for (const part of parts) { if (part.functionCall && pendingIds.includes(part.functionCall.id)) { diff --git a/src/app/components/markdown/markdown.component.scss b/src/app/components/markdown/markdown.component.scss new file mode 100644 index 00000000..e69de29b diff --git a/src/app/components/side-panel/side-panel.component.spec.ts b/src/app/components/side-panel/side-panel.component.spec.ts index ec1da1c4..930b749c 100644 --- a/src/app/components/side-panel/side-panel.component.spec.ts +++ b/src/app/components/side-panel/side-panel.component.spec.ts @@ -41,6 +41,7 @@ import {MockFeatureFlagService} from '../../core/services/testing/mock-feature-f import {MockSafeValuesService} from '../../core/services/testing/mock-safevalues.service'; import {TRACE_SERVICE, TraceService} from '../../core/services/trace.service'; import {VIDEO_SERVICE, VideoService} from '../../core/services/video.service'; +import {SCREEN_SHARING_SERVICE, ScreenSharingService} from '../../core/services/screensharing.service'; import {WEBSOCKET_SERVICE, WebSocketService,} from '../../core/services/websocket.service'; import {SidePanelComponent} from './side-panel.component'; @@ -70,6 +71,7 @@ describe('SidePanelComponent', () => { let mockAudioService: jasmine.SpyObj; let mockWebSocketService: jasmine.SpyObj; let mockVideoService: jasmine.SpyObj; + let mockScreenSharingService: jasmine.SpyObj; let mockEventService: jasmine.SpyObj; let mockDownloadService: jasmine.SpyObj; let mockEvalService: jasmine.SpyObj; @@ -106,6 +108,10 @@ describe('SidePanelComponent', () => { 'VideoService', ['startRecording', 'stopRecording'], ); + mockScreenSharingService = jasmine.createSpyObj( + 'ScreenSharingService', + ['startScreenSharing', 'stopScreenSharing'], + ); mockEventService = jasmine.createSpyObj('EventService', ['getTrace']); mockDownloadService = jasmine.createSpyObj( 'DownloadService', @@ -175,6 +181,7 @@ describe('SidePanelComponent', () => { {provide: AUDIO_SERVICE, useValue: mockAudioService}, {provide: WEBSOCKET_SERVICE, useValue: mockWebSocketService}, {provide: VIDEO_SERVICE, useValue: mockVideoService}, + {provide: SCREEN_SHARING_SERVICE, useValue: mockScreenSharingService}, {provide: EVENT_SERVICE, useValue: mockEventService}, {provide: DOWNLOAD_SERVICE, useValue: mockDownloadService}, {provide: EVAL_SERVICE, useValue: mockEvalService}, diff --git a/src/app/core/services/screensharing.service.ts b/src/app/core/services/screensharing.service.ts new file mode 100644 index 00000000..e862b2bc --- /dev/null +++ b/src/app/core/services/screensharing.service.ts @@ -0,0 +1,173 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { + ElementRef, + Injectable, + InjectionToken, + Renderer2, + RendererFactory2, +} from '@angular/core'; +import {LiveRequest} from '../models/LiveRequest'; +import {WebSocketService} from './websocket.service'; + +export const SCREEN_SHARING_SERVICE = new InjectionToken( + 'ScreenSharingService', +); + +@Injectable({ + providedIn: 'root', +}) +export class ScreenSharingService { + private mediaRecorder!: MediaRecorder; + private stream!: MediaStream; + private renderer: Renderer2; + private videoElement!: HTMLVideoElement; + private videoBuffer: Uint8Array[] = []; + private captureIntervalId: any = null; + + constructor( + private wsService: WebSocketService, + rendererFactory: RendererFactory2, + ) { + this.renderer = rendererFactory.createRenderer(null, null); + } + + createVideoElement(container: ElementRef) { + this.clearVideoElement(container); + + this.videoElement = this.renderer.createElement('video'); + this.renderer.setAttribute(this.videoElement, 'width', '800'); + this.renderer.setAttribute(this.videoElement, 'height', '600'); + this.renderer.setAttribute(this.videoElement, 'autoplay', 'true'); + this.renderer.setAttribute(this.videoElement, 'muted', 'true'); + + this.renderer.appendChild(container.nativeElement, this.videoElement); + } + + async startScreenSharing(container: ElementRef) { + this.createVideoElement(container); + + try { + this.stream = await navigator.mediaDevices.getDisplayMedia({video: true}); + this.videoElement.srcObject = this.stream; + + this.mediaRecorder = new MediaRecorder(this.stream, { + mimeType: 'video/webm', + }); + + this.mediaRecorder.start(1000); + this.captureIntervalId = setInterval( + () => this.captureAndSendFrame(), + 1000, + ); + } catch (error) { + console.error('Error starting screen sharing:', error); + } + } + + private async captureAndSendFrame() { + try { + const frameBlob = await this.captureFrame(); + const frameUint8Array = await this.blobToUint8Array(frameBlob); + const request: LiveRequest = { + blob: { + mime_type: 'image/jpeg', + data: frameUint8Array, + }, + }; + this.wsService.sendMessage(request); + } catch (error) { + console.error('Error capturing/sending screen frame:', error); + } + } + + private async blobToUint8Array(blob: Blob): Promise { + const arrayBuffer = await blob.arrayBuffer(); + return new Uint8Array(arrayBuffer); + } + + private async captureFrame(): Promise { + return new Promise((resolve, reject) => { + try { + const canvas = document.createElement('canvas'); + canvas.width = this.videoElement.videoWidth; + canvas.height = this.videoElement.videoHeight; + const ctx = canvas.getContext('2d'); + + if (!ctx) { + reject(new Error('Canvas context not supported')); + return; + } + + ctx.drawImage(this.videoElement, 0, 0, canvas.width, canvas.height); + + canvas.toBlob( + (blob) => { + if (blob) resolve(blob); + else reject(new Error('Failed to create image blob')); + }, + 'image/png'); + } catch (error) { + reject(error); + } + }); + } + + private sendBufferedVideo() { + if (this.videoBuffer.length === 0) return; + // Concatenate all accumulated chunks into one Uint8Array + const totalLength = this.videoBuffer.reduce( + (sum, chunk) => sum + chunk.length, + 0, + ); + const combinedBuffer = new Uint8Array(totalLength); + + let offset = 0; + for (const chunk of this.videoBuffer) { + combinedBuffer.set(chunk, offset); + offset += chunk.length; + } + + const request: LiveRequest = { + blob: { + mime_type: 'image/jpeg', + data: combinedBuffer, + }, + }; + this.wsService.sendMessage(request); + this.videoBuffer = []; + } + + stopScreenSharing(container: ElementRef) { + if (this.mediaRecorder) { + this.mediaRecorder.stop(); + } + if (this.stream) { + this.stream.getTracks().forEach((track) => track.stop()); + } + clearInterval(this.captureIntervalId); + this.clearVideoElement(container); + } + + private clearVideoElement(container: ElementRef) { + const existingVideo = container.nativeElement.querySelector('video'); + if (existingVideo) { + this.renderer.removeChild(container.nativeElement, existingVideo); + } + } +} diff --git a/src/app/core/services/stream-chat.service.ts b/src/app/core/services/stream-chat.service.ts index 48a8fabf..e3bf9285 100644 --- a/src/app/core/services/stream-chat.service.ts +++ b/src/app/core/services/stream-chat.service.ts @@ -23,6 +23,7 @@ import {LiveRequest} from '../models/LiveRequest'; import {AUDIO_SERVICE, AudioService} from './audio.service'; import {VIDEO_SERVICE, VideoService} from './video.service'; import {WEBSOCKET_SERVICE, WebSocketService} from './websocket.service'; +import { SCREEN_SHARING_SERVICE, ScreenSharingService } from './screensharing.service'; export const STREAM_CHAT_SERVICE = new InjectionToken('StreamChatService'); @@ -36,10 +37,12 @@ export const STREAM_CHAT_SERVICE = export class StreamChatService { private audioIntervalId: number|undefined = undefined; private videoIntervalId: number|undefined = undefined; + private screenSharingIntervalId: number|undefined = undefined; constructor( @Inject(AUDIO_SERVICE) private readonly audioService: AudioService, @Inject(VIDEO_SERVICE) private readonly videoService: VideoService, + @Inject(SCREEN_SHARING_SERVICE) private readonly screenSharingService: ScreenSharingService, @Inject(WEBSOCKET_SERVICE) private readonly webSocketService: WebSocketService, ) {} @@ -111,12 +114,38 @@ export class StreamChatService { await this.startVideoStreaming(videoContainer); } + async startScreenSharingChat({ + appName, + userId, + sessionId, + screenSharingContainer, + }: { + appName: string; userId: string; sessionId: string; + screenSharingContainer: ElementRef + }){ + const protocol = window.location.protocol === 'https:' ? 'wss' : 'ws'; + this.webSocketService.connect( + `${protocol}://${URLUtil.getWSServerUrl()}/run_live?app_name=${ + appName}&user_id=${userId}&session_id=${sessionId}`, + ); + + await this.startAudioStreaming(); + await this.startScreenSharing(screenSharingContainer); + } + stopVideoChat(videoContainer: ElementRef) { this.stopAudioStreaming(); this.stopVideoStreaming(videoContainer); this.webSocketService.closeConnection(); } + stopScreenSharingChat(screenSharingContainer: ElementRef) { + this.stopAudioStreaming(); + this.stopScreenSharing(screenSharingContainer); + this.webSocketService.closeConnection(); + } + + private async startVideoStreaming(videoContainer: ElementRef) { try { await this.videoService.startRecording(videoContainer); @@ -129,6 +158,18 @@ export class StreamChatService { } } + private async startScreenSharing(screenSharingContainer: ElementRef) { + try { + await this.screenSharingService.startScreenSharing(screenSharingContainer); + this.screenSharingIntervalId = setInterval( + async () => await this.sendCapturedFrame(), + 1000, + ); + } catch (error) { + console.error('Error accessing camera:', error); + } + } + private async sendCapturedFrame() { const capturedFrame = await this.videoService.getCapturedFrame(); if (!capturedFrame) return; @@ -148,6 +189,12 @@ export class StreamChatService { this.videoService.stopRecording(videoContainer); } + private stopScreenSharing(screenSharingContainer: ElementRef) { + clearInterval(this.screenSharingIntervalId); + this.screenSharingIntervalId = undefined; + this.screenSharingService.stopScreenSharing(screenSharingContainer); + } + onStreamClose() { return this.webSocketService.onCloseReason(); } diff --git a/src/app/core/services/testing/mock-screensharing.service.ts b/src/app/core/services/testing/mock-screensharing.service.ts new file mode 100644 index 00000000..d61b096c --- /dev/null +++ b/src/app/core/services/testing/mock-screensharing.service.ts @@ -0,0 +1,27 @@ +/** + * @license + * Copyright 2025 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import {Injectable} from '@angular/core'; + +import {ScreenSharingService} from '../screensharing.service'; + +@Injectable() +export class MockScreenSharingService implements Partial { + createVideoElement = jasmine.createSpy('createVideoElement'); + startScreenSharing = jasmine.createSpy('startScreenSharing'); + stopScreenSharing = jasmine.createSpy('stopScreenSharing'); +} \ No newline at end of file diff --git a/src/assets/config/runtime-config.json b/src/assets/config/runtime-config.json index 238c76ca..c6732406 100644 --- a/src/assets/config/runtime-config.json +++ b/src/assets/config/runtime-config.json @@ -1,3 +1,3 @@ { "backendUrl": "http://localhost:8000" -} +} \ No newline at end of file diff --git a/src/main.ts b/src/main.ts index 974d2dba..6696c463 100644 --- a/src/main.ts +++ b/src/main.ts @@ -15,7 +15,6 @@ * limitations under the License. */ - import {HttpClientModule} from '@angular/common/http'; import {importProvidersFrom} from '@angular/core'; import {FormsModule} from '@angular/forms'; @@ -48,6 +47,7 @@ import {STREAM_CHAT_SERVICE, StreamChatService} from './app/core/services/stream import {StringToColorServiceImpl} from './app/core/services/string-to-color.service'; import {TRACE_SERVICE, TraceService} from './app/core/services/trace.service'; import {VIDEO_SERVICE, VideoService} from './app/core/services/video.service'; +import {SCREEN_SHARING_SERVICE, ScreenSharingService} from './app/core/services/screensharing.service'; import {WEBSOCKET_SERVICE, WebSocketService} from './app/core/services/websocket.service'; import {LOGO_COMPONENT} from './app/injection_tokens'; @@ -70,6 +70,7 @@ fetch('./assets/config/runtime-config.json') }, {provide: AUDIO_SERVICE, useClass: AudioService}, {provide: VIDEO_SERVICE, useClass: VideoService}, + {provide: SCREEN_SHARING_SERVICE, useClass: ScreenSharingService}, {provide: STREAM_CHAT_SERVICE, useClass: StreamChatService}, {provide: EVENT_SERVICE, useClass: EventService}, {provide: EVAL_SERVICE, useClass: EvalService},