|
1 | | -/* |
2 | | - * Copyright (c) 2025 Bytedance, Inc. and its affiliates. |
3 | | - * SPDX-License-Identifier: Apache-2.0 |
| 1 | +/** |
| 2 | + * Target path: apps/ui-tars/src/main/shared/setOfMarks.test.ts |
4 | 3 | */ |
5 | | -import { test, expect } from 'vitest'; |
| 4 | +import { describe, expect, it } from 'vitest'; |
6 | 5 | import { setOfMarksOverlays } from './setOfMarks'; |
7 | 6 |
|
8 | | -const testMakeScreenMarker = () => { |
9 | | - let xPos; |
10 | | - let yPos; |
11 | | - const actions = [ |
12 | | - { |
13 | | - action_type: 'double_click', |
14 | | - action_inputs: { |
15 | | - start_box: '[0.1171875,0.20833333,0.1171875,0.20833333]', |
16 | | - }, |
17 | | - reflection: 'reflection', |
18 | | - thought: 'thought', |
19 | | - }, |
20 | | - { |
21 | | - action_type: 'type', |
22 | | - action_inputs: { |
23 | | - content: 'Hello, world!', |
24 | | - }, |
25 | | - reflection: 'reflection', |
26 | | - thought: 'thought', |
27 | | - }, |
28 | | - { |
29 | | - action_type: 'drag', |
30 | | - action_inputs: { |
31 | | - start_box: '[0.1171875,0.20833333,0.1171875,0.20833333]', |
32 | | - end_box: '[0.175,0.647,0.175,0.647]', |
33 | | - }, |
34 | | - reflection: 'reflection', |
35 | | - thought: 'thought', |
36 | | - }, |
37 | | - { |
38 | | - reflection: '', |
39 | | - thought: |
40 | | - '我已经在搜索框中输入了"杭州天气"，但还需要按下回车键来执行搜索。现在需要按下回车键来提交搜索请求，这样就能看到杭州的天气信息。', |
41 | | - action_type: 'hotkey', |
42 | | - action_inputs: { key: 'ctrl enter' }, |
43 | | - }, |
44 | | - { |
45 | | - reflection: '', |
46 | | - thought: |
47 | | - 'To narrow down the search results to cat litters within the specified price range of $18 to $32, I need to adjust the price filter. The next logical step is to drag the left handle of the price slider to set the minimum price to $18, ensuring that only products within the desired range are displayed.\n' + |
48 | | - 'Drag the left handle of the price slider to set the minimum price to $18.', |
49 | | - action_type: 'drag', |
50 | | - action_inputs: { |
51 | | - start_box: '[0.072,0.646,0.072,0.646]', |
52 | | - end_box: '[0.175,0.647,0.175,0.647]', |
| 7 | +const screenshotContext = { |
| 8 | + size: { width: 1920, height: 1080 }, |
| 9 | +} as any; |
| 10 | + |
| 11 | +describe('setOfMarksOverlays SVG escaping', () => { |
| 12 | + it('escapes content for type action', () => { |
| 13 | + const predictions = [ |
| 14 | + { |
| 15 | + action_type: 'type', |
| 16 | + action_inputs: { |
| 17 | + content: '<img src=x onerror=alert(1)>', |
| 18 | + }, |
53 | 19 | }, |
54 | | - }, |
55 | | - { |
56 | | - reflection: null, |
57 | | - thought: |
58 | | - '我看到桌面上有Google Chrome的图标，要完成打开Chrome的任务，我需要双击该图标。在之前的操作中，我已经双击了Chrome图标，但是页面没有发生变化，我应该等待一段时间，等待页面加载完成。', |
59 | | - action_type: 'wait', |
60 | | - action_inputs: {}, |
61 | | - }, |
62 | | - ]; |
63 | | - for (const action of actions) { |
| 20 | + ] as any; |
| 21 | + |
64 | 22 | const { overlays } = setOfMarksOverlays({ |
65 | | - predictions: [action], |
66 | | - screenshotContext: { |
67 | | - size: { |
68 | | - width: 2560, |
69 | | - height: 1440, |
| 23 | + predictions, |
| 24 | + screenshotContext, |
| 25 | + xPos: 100, |
| 26 | + yPos: 100, |
| 27 | + }); |
| 28 | + |
| 29 | + expect(overlays[0].svg).toContain('<img'); |
| 30 | + expect(overlays[0].svg).not.toContain('<img'); |
| 31 | + }); |
| 32 | + |
| 33 | + it('escapes hotkey text', () => { |
| 34 | + const predictions = [ |
| 35 | + { |
| 36 | + action_type: 'hotkey', |
| 37 | + action_inputs: { |
| 38 | + key: 'ctrl <script>alert(1)</script>', |
70 | 39 | }, |
71 | | - scaleFactor: 1, |
72 | 40 | }, |
73 | | - xPos, |
74 | | - yPos, |
| 41 | + ] as any; |
| 42 | + |
| 43 | + const { overlays } = setOfMarksOverlays({ |
| 44 | + predictions, |
| 45 | + screenshotContext, |
| 46 | + xPos: 100, |
| 47 | + yPos: 100, |
75 | 48 | }); |
76 | | - console.log('overlays', overlays); |
77 | | - // for (let i = 0; i < overlays.length; i++) { |
78 | | - // const overlay = overlays[i]; |
79 | | - // const currentOverlay = new BrowserWindow({ |
80 | | - // width: overlay.boxWidth || 200, |
81 | | - // height: overlay.boxHeight || 200, |
82 | | - // transparent: true, |
83 | | - // frame: false, |
84 | | - // alwaysOnTop: true, |
85 | | - // skipTaskbar: true, |
86 | | - // focusable: false, |
87 | | - // hasShadow: false, |
88 | | - // thickFrame: false, |
89 | | - // paintWhenInitiallyHidden: true, |
90 | | - // type: 'panel', |
91 | | - // webPreferences: { |
92 | | - // nodeIntegration: true, |
93 | | - // contextIsolation: false, |
94 | | - // }, |
95 | | - // }); |
96 | | - // currentOverlay.webContents.openDevTools(); |
97 | | - // if (overlay.xPos && overlay.yPos && overlay.svg) { |
98 | | - // currentOverlay.setPosition( |
99 | | - // overlay.xPos + overlay.offsetX, |
100 | | - // overlay.yPos + overlay.offsetY, |
101 | | - // ); |
102 | | - // xPos = overlay.xPos; |
103 | | - // yPos = overlay.yPos; |
104 | | - // currentOverlay.loadURL(`data:text/html;charset=UTF-8, |
105 | | - // <html> |
106 | | - // <head> |
107 | | - // <style> |
108 | | - // html, body { |
109 | | - // background: transparent; |
110 | | - // margin: 0; |
111 | | - // padding: 0; |
112 | | - // overflow: hidden; |
113 | | - // width: 100%; |
114 | | - // height: 100%; |
115 | | - // } |
116 | | - // </style> |
117 | | - // </head> |
118 | | - // <body> |
119 | | - // ${overlay.svg} |
120 | | - // </body> |
121 | | - // </html> |
122 | | - // `); |
123 | | - // } |
124 | | - // await sleep(1000); |
125 | | - // currentOverlay.close(); |
126 | | - // } |
127 | | - } |
128 | | -}; |
129 | 49 |
|
130 | | -test('not throw error', () => { |
131 | | - expect(() => testMakeScreenMarker()).not.toThrow(); |
| 50 | + expect(overlays[0].svg).toContain('<script>'); |
| 51 | + expect(overlays[0].svg).not.toContain('<script>'); |
| 52 | + }); |
132 | 53 | }); |
0 commit comments