Skip to content

Commit a510f0b

Browse files
peterkaplancopybara-github
peterkaplan
authored and
copybara-github
committed
feat: Render click overlay for computer use actions in chat
PiperOrigin-RevId: 859276971
1 parent 7205ee4 commit a510f0b

File tree

10 files changed

+724
-140
lines changed

10 files changed

+724
-140
lines changed

‎src/app/components/chat-panel/chat-panel.component.html‎

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -224,28 +224,25 @@
224224
}
225225
</mat-card>
226226
} @if (message.functionCall) {
227-
<button
228-
mat-stroked-button
229-
[ngClass]="{'function-event-button-highlight': shouldMessageHighlighted(i)}"
230-
class="function-event-button"
231-
(click)="clickEvent.emit(i)"
232-
>
233-
<mat-icon>bolt</mat-icon>
234-
{{ message.functionCall.name }}
235-
</button>
236-
} @if (message.functionResponse) {
237-
@if (isComputerUseResponse(message)) {
238-
<div class="computer-use-container" (click)="clickEvent.emit(i)">
239-
<div class="computer-use-header">
240-
<span class="computer-use-tool-name">{{ message.functionResponse.name }}</span>
241-
</div>
242-
<img [src]="getComputerUseScreenshot(message)" class="computer-use-screenshot" alt="Computer Use Screenshot" />
243-
<div class="computer-use-footprint">
244-
<mat-icon class="computer-icon">computer</mat-icon>
245-
<span class="url-text">{{ getComputerUseUrl(message) }}</span>
246-
</div>
247-
</div>
227+
@if (isComputerUseClick(message)) {
228+
<app-computer-action [message]="message" [allMessages]="messages" [index]="i" (clickEvent)="clickEvent.emit($event)"></app-computer-action>
248229
} @else {
230+
<button
231+
mat-stroked-button
232+
[ngClass]="{'function-event-button-highlight': shouldMessageHighlighted(i)}"
233+
class="function-event-button"
234+
(click)="clickEvent.emit(i)"
235+
>
236+
<mat-icon>bolt</mat-icon>
237+
{{ message.functionCall.name }}
238+
</button>
239+
}
240+
}
241+
@if (message.functionResponse) {
242+
@if (isComputerUseResponse(message)) {
243+
<app-computer-action [message]="message" [allMessages]="messages" [index]="i" (clickEvent)="clickEvent.emit($event)"></app-computer-action>
244+
}
245+
@else {
249246
<button mat-stroked-button [ngClass]="{'function-event-button-highlight': shouldMessageHighlighted(i)}"
250247
class="function-event-button" (click)="clickEvent.emit(i)">
251248
<mat-icon>check</mat-icon>

‎src/app/components/chat-panel/chat-panel.component.scss‎

Lines changed: 0 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -441,52 +441,3 @@ button.video-rec-btn {
441441
margin-top: 1em;
442442
margin-bottom: 1em;
443443
}
444-
445-
.computer-use-container {
446-
display: flex;
447-
flex-direction: column;
448-
max-width: 400px;
449-
border-radius: 12px;
450-
border: 1px solid var(--chat-panel-input-field-mat-mdc-text-field-wrapper-border-color);
451-
overflow: hidden;
452-
cursor: pointer;
453-
margin: 5px 5px 10px;
454-
background-color: var(--chat-panel-bot-message-message-card-background-color);
455-
transition: opacity 0.2s;
456-
457-
&:hover {
458-
opacity: 0.9;
459-
}
460-
}
461-
462-
.computer-use-screenshot {
463-
width: 100%;
464-
height: auto;
465-
display: block;
466-
border-bottom: 1px solid var(--chat-panel-input-field-mat-mdc-text-field-wrapper-border-color);
467-
}
468-
469-
.computer-use-footprint {
470-
display: flex;
471-
align-items: center;
472-
padding: 8px 12px;
473-
gap: 8px;
474-
background-color: var(--chat-panel-thought-chip-background-color);
475-
}
476-
477-
.computer-icon {
478-
font-size: 18px;
479-
width: 18px;
480-
height: 18px;
481-
color: var(--chat-panel-header-expected-color);
482-
}
483-
484-
.url-text {
485-
font-size: 11px;
486-
font-family: monospace;
487-
white-space: nowrap;
488-
overflow: hidden;
489-
text-overflow: ellipsis;
490-
color: var(--chat-panel-input-field-textarea-color);
491-
opacity: 0.8;
492-
}

‎src/app/components/chat-panel/chat-panel.component.spec.ts‎

Lines changed: 4 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -695,6 +695,7 @@ describe('ChatPanelComponent', () => {
695695
expect(component.feedback.emit).toHaveBeenCalledWith({direction: 'down'});
696696
});
697697
});
698+
698699
describe('Computer Use', () => {
699700
it(
700701
'isComputerUseResponse should return true when image data and url are present',
@@ -721,50 +722,6 @@ describe('ChatPanelComponent', () => {
721722
}
722723
};
723724
expect(component.isComputerUseResponse(message)).toBeFalse();
724-
});
725-
726-
it(
727-
'isComputerUseResponse should return false when url is missing', () => {
728-
const message = {
729-
functionResponse: {
730-
name: 'computer_use',
731-
response:
732-
{image: {data: 'base64data', mimetype: 'image/png'}, url: ''}
733-
}
734-
};
735-
expect(component.isComputerUseResponse(message)).toBeFalse();
736-
});
737-
738-
it(
739-
'should render computer use container and screenshot', async () => {
740-
component.messages = [{
741-
role: 'bot',
742-
functionResponse: {
743-
name: 'computer',
744-
response: {
745-
image: {data: 'base64data', mimetype: 'image/png'},
746-
url: 'http://google.com'
747-
}
748-
}
749-
}];
750-
fixture.detectChanges();
751-
await fixture.whenStable();
752-
753-
const container =
754-
fixture.debugElement.query(By.css('.computer-use-container'));
755-
expect(container).toBeTruthy();
756-
757-
const img =
758-
fixture.debugElement.query(By.css('.computer-use-screenshot'));
759-
expect(img.nativeElement.src)
760-
.toContain('data:image/png;base64,base64data');
761-
762-
const url = fixture.debugElement.query(By.css('.url-text'));
763-
expect(url.nativeElement.textContent).toContain('http://google.com');
764-
765-
const toolName =
766-
fixture.debugElement.query(By.css('.computer-use-tool-name'));
767-
expect(toolName.nativeElement.textContent).toContain('computer');
768-
});
769-
});
770-
});
725+
});
726+
});
727+
});

‎src/app/components/chat-panel/chat-panel.component.ts‎

Lines changed: 8 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ import {EMPTY, merge, NEVER, of, Subject} from 'rxjs';
3535
import {catchError, filter, first, switchMap, tap} from 'rxjs/operators';
3636

3737
import type {EvalCase} from '../../core/models/Eval';
38-
import {ComputerUsePayload, FunctionResponse} from '../../core/models/types';
38+
import {FunctionCall, FunctionResponse} from '../../core/models/types';
3939
import {AGENT_SERVICE} from '../../core/services/interfaces/agent';
4040
import {FEATURE_FLAG_SERVICE} from '../../core/services/interfaces/feature-flag';
4141
import {SESSION_SERVICE} from '../../core/services/interfaces/session';
@@ -45,8 +45,9 @@ import {UI_STATE_SERVICE} from '../../core/services/interfaces/ui-state';
4545
import {MediaType,} from '../artifact-tab/artifact-tab.component';
4646
import {AudioPlayerComponent} from '../audio-player/audio-player.component';
4747
import {MARKDOWN_COMPONENT, MarkdownComponentInterface} from '../markdown/markdown.component.interface';
48-
48+
import {ComputerActionComponent} from '../computer-action/computer-action.component';
4949
import {ChatPanelMessagesInjectionToken} from './chat-panel.component.i18n';
50+
import {isComputerUseResponse, isVisibleComputerUseClick} from '../../core/models/ComputerUse';
5051

5152
const ROOT_AGENT = 'root_agent';
5253

@@ -71,6 +72,7 @@ const ROOT_AGENT = 'root_agent';
7172
AudioPlayerComponent,
7273
MatTooltipModule,
7374
NgClass,
75+
ComputerActionComponent,
7476
],
7577
})
7678
export class ChatPanelComponent implements OnChanges, AfterViewInit {
@@ -276,28 +278,11 @@ export class ChatPanelComponent implements OnChanges, AfterViewInit {
276278
}
277279
}
278280

279-
isComputerUseResponse(message: {functionResponse?: FunctionResponse}):
280-
boolean {
281-
const response = message.functionResponse?.response as ComputerUsePayload;
282-
return !!(response?.image?.data && response?.url);
283-
}
284-
285-
getComputerUseScreenshot(message: {functionResponse?: FunctionResponse}):
286-
string {
287-
const response = message.functionResponse?.response as ComputerUsePayload;
288-
const imageInfo = response?.image;
289-
290-
if (!imageInfo?.data) return '';
291-
292-
const screenshot = imageInfo.data;
293-
if (screenshot.startsWith('data:')) return screenshot;
294-
295-
const mimeType = imageInfo.mimetype || 'image/png';
296-
return `data:${mimeType};base64,${screenshot}`;
281+
isComputerUseClick(message: {functionCall?: FunctionCall}): boolean {
282+
return isVisibleComputerUseClick(message);
297283
}
298284

299-
getComputerUseUrl(message: {functionResponse?: FunctionResponse}): string {
300-
const response = message.functionResponse?.response as ComputerUsePayload;
301-
return response?.url || '';
285+
isComputerUseResponse(message: {functionResponse?: FunctionResponse}): boolean {
286+
return isComputerUseResponse(message);
302287
}
303288
}

‎src/app/components/computer-action/computer-action.component.html‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
@if (message.functionCall && isComputerUseClick()) {
2+
@let screenshot = getPreviousComputerUseScreenshot();
3+
@if (screenshot) {
4+
<div class="computer-use-container click-visualization-container" (click)="clickEvent.emit(index)">
5+
<div class="computer-use-header">
6+
<span class="computer-use-tool-name">
7+
{{ message.functionCall.name }}
8+
@if (message.functionCall.args['action']) {
9+
({{ message.functionCall.args['action'] }})
10+
}
11+
@let actualCoords = getActualPixelCoordinates();
12+
@if (actualCoords) {
13+
<span class="actual-pixels" [matTooltip]="actualCoords.isVirtual ? 'Virtual (1000x1000)' : 'Hardware Mapping'">
14+
[&#64; {{ actualCoords.x }}{{ actualCoords.isVirtual ? 'v' : 'px' }}, {{ actualCoords.y }}{{
15+
actualCoords.isVirtual
16+
? 'v' : 'px' }}]
17+
</span>
18+
}
19+
</span>
20+
</div>
21+
<div class="image-wrapper">
22+
<img [src]="screenshot" class="computer-use-screenshot" alt="Computer Use Screenshot"
23+
(load)="onImageLoad($event)" />
24+
<div class="click-overlay-box" [ngStyle]="getClickBoxStyle()"></div>
25+
</div>
26+
</div>
27+
}
28+
} @else if (message.functionResponse && isComputerUseResponse()) {
29+
<div class="computer-use-container" (click)="clickEvent.emit(index)">
30+
<div class="computer-use-header">
31+
<span class="computer-use-tool-name">{{ message.functionResponse.name }}</span>
32+
</div>
33+
<img [src]="getComputerUseScreenshot()" class="computer-use-screenshot" alt="Computer Use Screenshot" />
34+
<div class="computer-use-footprint">
35+
<mat-icon class="computer-icon">computer</mat-icon>
36+
<span class="url-text">{{ getComputerUseUrl() }}</span>
37+
</div>
38+
</div>
39+
}

‎src/app/components/computer-action/computer-action.component.scss‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
:host {
2+
display: block;
3+
}
4+
5+
.computer-use-container {
6+
display: flex;
7+
flex-direction: column;
8+
max-width: 600px;
9+
border-radius: 12px;
10+
border: 1px solid var(--chat-panel-input-field-mat-mdc-text-field-wrapper-border-color);
11+
overflow: hidden;
12+
cursor: pointer;
13+
margin: 5px 5px 10px;
14+
background-color: var(--chat-panel-bot-message-message-card-background-color);
15+
transition: opacity 0.2s;
16+
&:hover {
17+
opacity: 0.9;
18+
}
19+
}
20+
21+
.computer-use-tool-name {
22+
font-size: 12px;
23+
font-family: monospace;
24+
font-weight: 600;
25+
color: var(--chat-panel-input-field-textarea-color);
26+
opacity: 0.9;
27+
padding: 12px;
28+
.actual-pixels {
29+
opacity: 0.6;
30+
margin-left: 8px;
31+
font-weight: 400;
32+
}
33+
}
34+
35+
.computer-use-screenshot {
36+
width: 100%;
37+
height: auto;
38+
display: block;
39+
border-bottom: 1px solid var(--chat-panel-input-field-mat-mdc-text-field-wrapper-border-color);
40+
}
41+
42+
.computer-use-footprint {
43+
display: flex;
44+
align-items: center;
45+
padding: 8px 12px;
46+
gap: 8px;
47+
background-color: var(--chat-panel-thought-chip-background-color);
48+
}
49+
50+
.computer-icon {
51+
font-size: 18px;
52+
width: 18px;
53+
height: 18px;
54+
flex-shrink: 0;
55+
}
56+
57+
.url-text {
58+
font-size: 11px;
59+
font-family: monospace;
60+
white-space: normal;
61+
word-break: break-all;
62+
color: var(--chat-panel-input-field-textarea-color);
63+
opacity: 0.8;
64+
min-width: 0;
65+
}
66+
67+
.image-wrapper {
68+
position: relative;
69+
width: 100%;
70+
}
71+
72+
.click-overlay-box {
73+
position: absolute;
74+
width: 24px;
75+
height: 24px;
76+
border: 1px solid rgba(255, 255, 255, 0.8);
77+
border-radius: 50%;
78+
transform: translate(-50%, -50%);
79+
background-color: rgba(255, 0, 0, 0.3);
80+
box-shadow: 0 0 4px rgba(0, 0, 0, 0.5);
81+
pointer-events: none;
82+
z-index: 10;
83+
display: flex;
84+
align-items: center;
85+
justify-content: center;
86+
&::before {
87+
content: '';
88+
width: 2px;
89+
height: 2px;
90+
background-color: #ff0000;
91+
border-radius: 50%;
92+
box-shadow: 0 0 2px white;
93+
z-index: 11;
94+
}
95+
&::after {
96+
content: '';
97+
position: absolute;
98+
width: 100%;
99+
height: 100%;
100+
background:
101+
linear-gradient(to right, transparent 48%, rgba(255, 255, 255, 0.6) 48%, rgba(255, 255, 255, 0.6) 52%, transparent 52%),
102+
linear-gradient(to bottom, transparent 48%, rgba(255, 255, 255, 0.6) 48%, rgba(255, 255, 255, 0.6) 52%, transparent 52%);
103+
border-radius: 50%;
104+
}
105+
}

0 commit comments

Comments
 (0)