Skip to content

Commit 0cf9a71

Browse files
zhoushawzhoushaw
authored
feat(chrome-devtool): water flow animation on the target page (#271)
* feat(chrome-devtool): support water flow animation * chore: optimize debugger cause window resize * chore: optimize dynamic script inject * chore: optimize report error
1 parent 857c411 commit 0cf9a71

6 files changed

Lines changed: 195 additions & 28 deletions

File tree

‎packages/midscene/src/action/executor.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import type {
88
ExecutionTaskProgressOptions,
99
ExecutionTaskReturn,
1010
ExecutorContext,
11+
PlanningAction,
1112
} from '@/types';
1213
import { getVersion } from '@/utils';
1314

@@ -99,7 +100,7 @@ export class Executor {
99100
await this.onTaskStart(task);
100101
}
101102
} catch (e) {
102-
// console.error('error in onTaskStart', e);
103+
console.error('error in onTaskStart', e);
103104
}
104105
assert(
105106
['Insight', 'Action', 'Planning'].indexOf(task.type) >= 0,
@@ -114,6 +115,7 @@ export class Executor {
114115
task,
115116
element: previousFindOutput?.element,
116117
};
118+
117119
if (task.type === 'Insight') {
118120
assert(
119121
task.subType === 'Locate' ||

‎packages/visualizer/modern.config.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ export default defineConfig({
5151
format: 'iife',
5252
dts: false,
5353
input: {
54+
'water-flow': 'src/extension/scripts/water-flow.ts',
55+
'stop-water-flow': 'src/extension/scripts/stop-water-flow.ts',
5456
popup: 'src/extension/popup.tsx',
5557
worker: 'src/extension/worker.ts',
5658
'playground-entry': 'src/extension/playground-entry.tsx',

‎packages/visualizer/src/extension/scripts/stop-water-flow.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
const styleElements = document.querySelectorAll('[id="water-flow-animation"]');
2+
styleElements.forEach((element) => {
3+
document.head.removeChild(element);
4+
});

‎packages/visualizer/src/extension/scripts/water-flow.ts‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
const waterFlowAnimation = {
2+
styleElement: null as null | HTMLStyleElement,
3+
4+
enable() {
5+
if (this.styleElement) return;
6+
// Check if water flow animation style already exists
7+
const existingStyle = document.querySelector('#water-flow-animation');
8+
if (existingStyle) return;
9+
10+
this.styleElement = document.createElement('style');
11+
this.styleElement.id = 'water-flow-animation';
12+
this.styleElement.textContent = `
13+
html::before {
14+
content: "";
15+
position: fixed;
16+
top: 0; right: 0; bottom: 0; left: 0;
17+
pointer-events: none;
18+
z-index: 9999;
19+
background:
20+
linear-gradient(to right, rgba(30, 144, 255, 0.4), transparent 50%) left,
21+
linear-gradient(to left, rgba(30, 144, 255, 0.4), transparent 50%) right,
22+
linear-gradient(to bottom, rgba(30, 144, 255, 0.4), transparent 50%) top,
23+
linear-gradient(to top, rgba(30, 144, 255, 0.4), transparent 50%) bottom;
24+
background-repeat: no-repeat;
25+
background-size: 10% 100%, 10% 100%, 100% 10%, 100% 10%;
26+
animation: waterflow 5s cubic-bezier(0.4, 0, 0.6, 1) infinite;
27+
filter: blur(8px);
28+
}
29+
30+
@keyframes waterflow {
31+
0%, 100% {
32+
background-image:
33+
linear-gradient(to right, rgba(30, 144, 255, 0.4), transparent 50%),
34+
linear-gradient(to left, rgba(30, 144, 255, 0.4), transparent 50%),
35+
linear-gradient(to bottom, rgba(30, 144, 255, 0.4), transparent 50%),
36+
linear-gradient(to top, rgba(30, 144, 255, 0.4), transparent 50%);
37+
transform: scale(1);
38+
}
39+
25% {
40+
background-image:
41+
linear-gradient(to right, rgba(30, 144, 255, 0.39), transparent 52%),
42+
linear-gradient(to left, rgba(30, 144, 255, 0.39), transparent 52%),
43+
linear-gradient(to bottom, rgba(30, 144, 255, 0.39), transparent 52%),
44+
linear-gradient(to top, rgba(30, 144, 255, 0.39), transparent 52%);
45+
transform: scale(1.03);
46+
}
47+
50% {
48+
background-image:
49+
linear-gradient(to right, rgba(30, 144, 255, 0.38), transparent 55%),
50+
linear-gradient(to left, rgba(30, 144, 255, 0.38), transparent 55%),
51+
linear-gradient(to bottom, rgba(30, 144, 255, 0.38), transparent 55%),
52+
linear-gradient(to top, rgba(30, 144, 255, 0.38), transparent 55%);
53+
transform: scale(1.05);
54+
}
55+
75% {
56+
background-image:
57+
linear-gradient(to right, rgba(30, 144, 255, 0.39), transparent 52%),
58+
linear-gradient(to left, rgba(30, 144, 255, 0.39), transparent 52%),
59+
linear-gradient(to bottom, rgba(30, 144, 255, 0.39), transparent 52%),
60+
linear-gradient(to top, rgba(30, 144, 255, 0.39), transparent 52%);
61+
transform: scale(1.03);
62+
}
63+
}
64+
`;
65+
document.head.appendChild(this.styleElement);
66+
},
67+
68+
disable() {
69+
const styleElements = document.querySelectorAll(
70+
'[id="water-flow-animation"]',
71+
);
72+
styleElements.forEach((element) => {
73+
document.head.removeChild(element);
74+
});
75+
this.styleElement = null;
76+
},
77+
};
78+
79+
export {};
80+
declare global {
81+
interface Window {
82+
waterFlowAnimation: typeof waterFlowAnimation;
83+
}
84+
}
85+
(window as any).waterFlowAnimation = waterFlowAnimation;
86+
(window as any).waterFlowAnimation.enable();

‎packages/web-integration/src/chrome-extension/dynamic-scripts.ts‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
import fs from 'node:fs';
2+
import { ifInBrowser } from '@midscene/shared/utils';
3+
4+
// remember to include this file into extension's package
5+
// extract html element from page
6+
let scriptFileContentCache: string | null = null;
7+
export const getHtmlElementScript = async () => {
8+
const scriptFileToRetrieve = chrome.runtime.getURL('lib/htmlElement.js');
9+
if (scriptFileContentCache) return scriptFileContentCache;
10+
if (ifInBrowser) {
11+
const script = await fetch(scriptFileToRetrieve);
12+
scriptFileContentCache = await script.text();
13+
return scriptFileContentCache;
14+
}
15+
return fs.readFileSync(scriptFileToRetrieve, 'utf8');
16+
};
17+
18+
// inject water flow animation
19+
let waterFlowScriptFileContentCache: string | null = null;
20+
export const injectWaterFlowAnimation = async () => {
21+
const waterFlowScriptFileToRetrieve =
22+
chrome.runtime.getURL('lib/water-flow.js');
23+
if (waterFlowScriptFileContentCache) return waterFlowScriptFileContentCache;
24+
if (ifInBrowser) {
25+
const script = await fetch(waterFlowScriptFileToRetrieve);
26+
waterFlowScriptFileContentCache = await script.text();
27+
return waterFlowScriptFileContentCache;
28+
}
29+
return fs.readFileSync(waterFlowScriptFileToRetrieve, 'utf8');
30+
};
31+
32+
// inject stop water flow animation
33+
let stopWaterFlowScriptFileContentCache: string | null = null;
34+
export const injectStopWaterFlowAnimation = async () => {
35+
const stopWaterFlowScriptFileToRetrieve = chrome.runtime.getURL(
36+
'lib/stop-water-flow.js',
37+
);
38+
if (stopWaterFlowScriptFileContentCache)
39+
return stopWaterFlowScriptFileContentCache;
40+
if (ifInBrowser) {
41+
const script = await fetch(stopWaterFlowScriptFileToRetrieve);
42+
stopWaterFlowScriptFileContentCache = await script.text();
43+
return stopWaterFlowScriptFileContentCache;
44+
}
45+
return fs.readFileSync(stopWaterFlowScriptFileToRetrieve, 'utf8');
46+
};

‎packages/web-integration/src/chrome-extension/page.ts‎

Lines changed: 54 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -5,27 +5,22 @@
55
The page must be active when interacting with it.
66
*/
77

8-
import fs from 'node:fs';
98
import type { WebKeyInput } from '@/common/page';
109
import type { ElementInfo } from '@/extractor';
1110
import type { AbstractPage } from '@/page';
1211
import type { Point, Size } from '@midscene/core';
1312
import { ifInBrowser } from '@midscene/shared/utils';
1413
import type { Protocol as CDPTypes } from 'devtools-protocol';
1514
import { CdpKeyboard } from './cdpInput';
16-
17-
// remember to include this file into extension's package
18-
const scriptFileToRetrieve = './lib/htmlElement.js';
19-
let scriptFileContentCache: string | null = null;
20-
const scriptFileContent = async () => {
21-
if (scriptFileContentCache) return scriptFileContentCache;
22-
if (ifInBrowser) {
23-
const script = await fetch('/lib/htmlElement.js');
24-
scriptFileContentCache = await script.text();
25-
return scriptFileContentCache;
26-
}
27-
return fs.readFileSync(scriptFileToRetrieve, 'utf8');
28-
};
15+
import {
16+
getHtmlElementScript,
17+
injectStopWaterFlowAnimation,
18+
injectWaterFlowAnimation,
19+
} from './dynamic-scripts';
20+
21+
function sleep(ms: number) {
22+
return new Promise((resolve) => setTimeout(resolve, ms));
23+
}
2924

3025
export default class ChromeExtensionProxyPage implements AbstractPage {
3126
pageType = 'chrome-extension-proxy';
@@ -51,6 +46,13 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
5146

5247
// Create new attaching promise
5348
this.attachingDebugger = (async () => {
49+
const url = await this.url();
50+
if (url.startsWith('chrome://')) {
51+
throw new Error(
52+
'Cannot attach debugger to chrome:// pages, please use Midscene in a normal page with http://, https:// or file://',
53+
);
54+
}
55+
5456
try {
5557
const currentTabId = this.getTabId();
5658
// check if debugger is already attached to the tab
@@ -62,7 +64,8 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
6264
// await chrome.debugger.detach({ tabId: currentTabId });
6365
await chrome.debugger.attach({ tabId: currentTabId }, '1.3');
6466
this.debuggerAttached = true;
65-
67+
// Prevent AI logic from being influenced by changes in page width and height due to the debugger banner appearing on attach.
68+
await sleep(500);
6669
// listen to the debugger detach event
6770
chrome.debugger.onEvent.addListener((source, method, params) => {
6871
console.log('debugger event', source, method, params);
@@ -76,36 +79,60 @@ export default class ChromeExtensionProxyPage implements AbstractPage {
7679
}
7780
})();
7881

79-
const url = await this.url();
80-
if (url.startsWith('chrome://')) {
81-
throw new Error(
82-
'Cannot attach debugger to chrome:// pages, please use Midscene in a normal page with http://, https:// or file://',
83-
);
84-
}
85-
8682
await this.attachingDebugger;
8783
}
8884

85+
private async enableWaterFlowAnimation(tabId: number) {
86+
const script = await injectWaterFlowAnimation();
87+
88+
await chrome.debugger.sendCommand({ tabId }, 'Runtime.evaluate', {
89+
expression: script,
90+
});
91+
}
92+
93+
private async disableWaterFlowAnimation(tabId: number) {
94+
const script = await injectStopWaterFlowAnimation();
95+
96+
await chrome.debugger.sendCommand({ tabId }, 'Runtime.evaluate', {
97+
expression: script,
98+
});
99+
}
100+
89101
private async detachDebugger() {
90-
if (!this.debuggerAttached) return;
91-
await chrome.debugger.detach({ tabId: this.getTabId() });
92-
this.debuggerAttached = false;
102+
// check if debugger is already attached to the tab
103+
const targets = await chrome.debugger.getTargets();
104+
const attendTabs = targets.filter(
105+
(target) =>
106+
target.attached === true &&
107+
!target.url.startsWith('chrome-extension://'),
108+
);
109+
if (attendTabs.length > 0) {
110+
for (const tab of attendTabs) {
111+
if (tab.tabId) {
112+
await this.disableWaterFlowAnimation(tab.tabId);
113+
chrome.debugger.detach({ tabId: tab.tabId });
114+
}
115+
}
116+
this.debuggerAttached = false;
117+
}
93118
}
94119

95120
private async sendCommandToDebugger<ResponseType = any, RequestType = any>(
96121
command: string,
97122
params: RequestType,
98123
): Promise<ResponseType> {
99124
await this.attachDebugger();
125+
const tabId = this.getTabId();
126+
this.enableWaterFlowAnimation(tabId);
100127
return (await chrome.debugger.sendCommand(
101-
{ tabId: this.getTabId() },
128+
{ tabId },
102129
command,
103130
params as any,
104131
)) as ResponseType;
105132
}
106133

107134
private async getPageContentByCDP() {
108-
const script = await scriptFileContent();
135+
const script = await getHtmlElementScript();
109136

110137
// check tab url
111138
await this.sendCommandToDebugger<

0 commit comments

Comments
 (0)