Skip to content

Commit 6d45f9c

Browse files
quanruquanru
committed
ci: add real Midscene GTK end-to-end test
1 parent 6c1e9bb commit 6d45f9c

7 files changed

Lines changed: 5823 additions & 2 deletions

File tree

‎.github/workflows/midscene-e2e.yml‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,82 @@
1+
name: Midscene E2E
2+
3+
on:
4+
push:
5+
branches: [ci/midscene-e2e]
6+
paths:
7+
- ".github/workflows/midscene-e2e.yml"
8+
- "tests/midscene/package.json"
9+
- "tests/midscene/package-lock.json"
10+
- "tests/midscene/gtk_fixture.py"
11+
- "tests/midscene/e2e/**"
12+
- "src/doubao_input/ui/**"
13+
workflow_dispatch:
14+
15+
permissions:
16+
contents: read
17+
18+
concurrency:
19+
group: midscene-e2e-${{ github.ref }}
20+
cancel-in-progress: true
21+
22+
jobs:
23+
ubuntu-gtk:
24+
name: AI-driven GTK onboarding
25+
runs-on: ubuntu-22.04
26+
timeout-minutes: 15
27+
env:
28+
MIDSCENE_MODEL_API_KEY: ${{ secrets.MIDSCENE_MODEL_API_KEY }}
29+
MIDSCENE_MODEL_NAME: ${{ secrets.MIDSCENE_MODEL_NAME }}
30+
MIDSCENE_MODEL_BASE_URL: ${{ secrets.MIDSCENE_MODEL_BASE_URL }}
31+
MIDSCENE_MODEL_FAMILY: ${{ secrets.MIDSCENE_MODEL_FAMILY }}
32+
MIDSCENE_MODEL_REASONING_ENABLED: ${{ secrets.MIDSCENE_MODEL_REASONING_ENABLED }}
33+
MIDSCENE_COMPUTER_HEADLESS_LINUX: "true"
34+
MIDSCENE_REPLANNING_CYCLE_LIMIT: "40"
35+
steps:
36+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
37+
with:
38+
persist-credentials: false
39+
40+
- uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0
41+
with:
42+
node-version: "22"
43+
cache: npm
44+
cache-dependency-path: tests/midscene/package-lock.json
45+
46+
- name: Require Midscene model configuration
47+
run: |
48+
test -n "$MIDSCENE_MODEL_API_KEY"
49+
test -n "$MIDSCENE_MODEL_NAME"
50+
test -n "$MIDSCENE_MODEL_BASE_URL"
51+
test -n "$MIDSCENE_MODEL_FAMILY"
52+
53+
- name: Install desktop test environment
54+
run: |
55+
sudo apt-get update
56+
sudo apt-get install -y \
57+
adwaita-icon-theme dbus-x11 fluxbox fonts-noto-cjk \
58+
gir1.2-gtk-4.0 gnome-themes-extra imagemagick \
59+
libasound2 libatk-bridge2.0-0 libdrm2 libgbm1 libgtk-3-0 \
60+
libnotify4 libnss3 libpng16-16 libsecret-1-0 libx11-6 \
61+
libxinerama1 libxkbcommon-x11-0 libxss1 libxtst6 \
62+
python3-cairo python3-gi x11-xserver-utils xdg-utils xvfb
63+
64+
- name: Install pinned Midscene dependencies
65+
working-directory: tests/midscene
66+
# sharp's platform packages are pinned explicitly. Skipping lifecycle
67+
# scripts avoids an npm ordering bug that tries to build sharp before
68+
# those prebuilt packages are linked.
69+
run: npm ci --include=optional --ignore-scripts
70+
71+
- name: Run real Midscene desktop E2E
72+
working-directory: tests/midscene
73+
run: npm test
74+
75+
- name: Upload Midscene report
76+
if: always()
77+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
78+
with:
79+
name: midscene-gtk-e2e-report
80+
path: tests/midscene/midscene_run/
81+
if-no-files-found: warn
82+
retention-days: 7

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,9 @@ settings.json
9191
/screenshots/
9292
/backups/
9393
/.midscene-omarchy/
94+
/.midscene-config/
95+
/tests/midscene/midscene_run/
96+
/tests/midscene/node_modules/
9497
*.bak
9598
*.orig
9699
*.rej

‎tests/midscene/README.md‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,5 +29,13 @@ After that probe is stable, the next stages are:
2929
Ubuntu for every trusted change.
3030
4. Run a smaller Omarchy/Hyprland Midscene suite manually in the guest.
3131

32-
The AI stage needs a multimodal model credential. No local credential is
33-
copied into GitHub automatically, and tests must contain only synthetic data.
32+
The ordinary Ubuntu stage maps the real GTK onboarding window inside the
33+
headless Midscene desktop. Its synthetic fixture performs no login, recording,
34+
network request, paste, or user-settings read. Midscene visually navigates to
35+
the shortcut page, runs the fake endpoint check, verifies its visible feedback,
36+
continues to the voice page, checks that navigation reset the scroll position,
37+
and completes setup. The HTML replay is uploaded for every run.
38+
39+
The AI stage needs a multimodal model credential. Its `MIDSCENE_MODEL_*`
40+
configuration is stored only as GitHub Actions Secrets; tests contain only
41+
synthetic data.

‎tests/midscene/e2e/onboarding.test.ts‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
import { spawn, type ChildProcess } from 'node:child_process';
2+
import { resolve } from 'node:path';
3+
import { afterAll, beforeAll, describe, it, vi } from 'vitest';
4+
import {
5+
type ComputerAgent,
6+
agentFromComputer,
7+
} from '@midscene/computer';
8+
9+
vi.setConfig({ testTimeout: 8 * 60 * 1000, hookTimeout: 2 * 60 * 1000 });
10+
11+
const sleep = (milliseconds: number) =>
12+
new Promise((resolvePromise) => setTimeout(resolvePromise, milliseconds));
13+
14+
const stopProcessGroup = (child?: ChildProcess) => {
15+
if (!child?.pid) return;
16+
try {
17+
process.kill(-child.pid, 'SIGTERM');
18+
} catch {
19+
// The fixture may already have exited after a test failure.
20+
}
21+
};
22+
23+
describe('Doubao Say onboarding', () => {
24+
let agent: ComputerAgent;
25+
let fluxbox: ChildProcess;
26+
let fixture: ChildProcess;
27+
28+
beforeAll(async () => {
29+
agent = await agentFromComputer({
30+
aiActionContext:
31+
'You are testing the English Doubao Say GTK onboarding window. ' +
32+
'Interact only with the Doubao Say window and use visible labels.',
33+
xvfbResolution: '1280x960x24',
34+
});
35+
36+
fluxbox = spawn('fluxbox', [], {
37+
detached: true,
38+
stdio: 'ignore',
39+
env: process.env,
40+
});
41+
await sleep(1000);
42+
43+
const repositoryRoot = resolve(import.meta.dirname, '../../..');
44+
fixture = spawn('/usr/bin/python3', ['tests/midscene/gtk_fixture.py'], {
45+
cwd: repositoryRoot,
46+
detached: true,
47+
stdio: ['ignore', 'inherit', 'inherit'],
48+
env: {
49+
...process.env,
50+
GTK_A11Y: 'none',
51+
PYTHONPATH: resolve(repositoryRoot, 'src'),
52+
XDG_CONFIG_HOME: resolve(repositoryRoot, '.midscene-config'),
53+
},
54+
});
55+
await sleep(2500);
56+
});
57+
58+
afterAll(() => {
59+
stopProcessGroup(fixture);
60+
stopProcessGroup(fluxbox);
61+
});
62+
63+
it('navigates, reports endpoint feedback, resets scroll and completes setup', async () => {
64+
await agent.aiAct(
65+
'Verify the visible Doubao Say window is on the Microphone step and has a Next button in the fixed top navigation.',
66+
);
67+
68+
await agent.aiAct(
69+
'Click the Next button in the fixed top navigation and wait until the current step is Trigger key.',
70+
);
71+
await agent.aiAct(
72+
'In the Voice polishing section, scroll within the window if needed, click Test endpoint, and wait for the endpoint result.',
73+
);
74+
await agent.aiAct(
75+
'Verify a visible message says the endpoint works and includes Synthetic endpoint response.',
76+
);
77+
78+
await agent.aiAct(
79+
'Click the Next button in the fixed top navigation and wait for the Voice test step.',
80+
);
81+
await agent.aiAct(
82+
'Verify the Voice test heading and its introductory text are visible near the top of the content, showing that the page did not remain scrolled to the bottom.',
83+
);
84+
await agent.aiAct(
85+
'Scroll within the content if needed and click the Finish setup button.',
86+
);
87+
await agent.aiAct(
88+
'Verify the visible feedback says Setup completed by the synthetic E2E fixture.',
89+
);
90+
});
91+
});

‎tests/midscene/gtk_fixture.py‎

Lines changed: 87 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
"""Mapped GTK onboarding fixture for Midscene CI; uses synthetic data only."""
2+
3+
import signal
4+
5+
import gi
6+
7+
gi.require_version("Gtk", "4.0")
8+
from gi.repository import GLib, Gtk
9+
10+
from doubao_input.doubao.app_state import AppState, LoginStatus
11+
from doubao_input.i18n import set_language
12+
from doubao_input.settings import Settings
13+
from doubao_input.ui.control_window import ControlWindow
14+
from doubao_input.ui.setup_actions import SetupActions
15+
16+
17+
def main():
18+
Gtk.init()
19+
set_language("en")
20+
state = AppState()
21+
state.login_status = LoginStatus.LOGGED_IN
22+
settings = Settings(
23+
polish_enabled=True,
24+
polish_base_url="https://example.invalid/v1",
25+
polish_model="synthetic-model",
26+
)
27+
summary = {
28+
"key": "Fn",
29+
"key_code": 464,
30+
"key_modifiers": (),
31+
"microphone": "Synthetic microphone",
32+
"microphone_id": "",
33+
"microphone_ok": True,
34+
"voice_test_ok": True,
35+
"onboarding_complete": False,
36+
}
37+
holder = {}
38+
39+
def test_endpoint(_settings, _key, done):
40+
def finish():
41+
done("Synthetic endpoint response", None)
42+
return GLib.SOURCE_REMOVE
43+
44+
GLib.timeout_add(350, finish)
45+
46+
def complete_setup():
47+
holder["control"].set_feedback("Setup completed by the synthetic E2E fixture.")
48+
49+
actions = SetupActions(
50+
test_voice=lambda: None,
51+
cancel_preview=lambda: None,
52+
open_settings=lambda: None,
53+
is_preview_testing=lambda: False,
54+
summary=lambda: summary,
55+
complete_setup=complete_setup,
56+
apply_key=lambda _key, _modifiers=(): None,
57+
polish_settings=lambda: settings,
58+
polish_has_key=lambda: True,
59+
save_polish=lambda _settings, _key: None,
60+
test_polish=test_endpoint,
61+
apply_microphone=lambda _microphone: None,
62+
)
63+
control = ControlWindow(
64+
state,
65+
on_login_clicked=lambda: None,
66+
on_quit_clicked=lambda: None,
67+
on_check_mic_clicked=lambda: None,
68+
actions=actions,
69+
)
70+
holder["control"] = control
71+
control.show()
72+
73+
loop = GLib.MainLoop()
74+
75+
def stop(*_args):
76+
loop.quit()
77+
return GLib.SOURCE_REMOVE
78+
79+
signal.signal(signal.SIGTERM, lambda *_args: GLib.idle_add(stop))
80+
signal.signal(signal.SIGINT, lambda *_args: GLib.idle_add(stop))
81+
print("READY: synthetic Doubao Say GTK fixture", flush=True)
82+
loop.run()
83+
control.destroy()
84+
85+
86+
if __name__ == "__main__":
87+
main()

0 commit comments

Comments
 (0)