From d830ca78d501ad1c17d352c652d7612d31ab7e0a Mon Sep 17 00:00:00 2001 From: Perry Zhu Date: Fri, 12 Jun 2026 20:43:07 -0700 Subject: [PATCH 1/2] refactor: move action recording/screenshot to CDP server #125 --- CHANGELOG.md | 4 + README.md | 23 +- docs/README.zh-CN.md | 25 +- src/README.md | 4 +- .../runtime/chrome-extension/README.md | 17 +- .../runtime/chrome-extension/background.js | 56 ----- .../runtime/chrome-extension/content.js | 70 ------ .../runtime/chrome-extension/manifest.json | 13 +- .../runtime/extension-server/README.md | 16 +- .../runtime/extension-server/server.py | 217 ++++++++++++++++-- 10 files changed, 245 insertions(+), 200 deletions(-) delete mode 100644 src/clawbench/runtime/chrome-extension/background.js delete mode 100644 src/clawbench/runtime/chrome-extension/content.js diff --git a/CHANGELOG.md b/CHANGELOG.md index 916d3ce1..0563b767 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/) and this project adheres to [Semantic Versioning](https://semver.org/). +## [Unreleased] +### Changed +- Refactored the container runtime to move the action recording and screenshot capturing logic into the CDP server rather than the Chrome extension, making it possible to integrate remote browsers in the future. + ## [0.6.0] - 2026-06-04 ### Added - Added support for the [Harbor](https://github.com/harbor-framework/harbor) framework. diff --git a/README.md b/README.md index 6d821b33..f545bd43 100644 --- a/README.md +++ b/README.md @@ -583,23 +583,20 @@ ClawBench's niche: **live consumer websites, everyday tasks, end-to-end recordin ┌─────────────────────────────────────────────────┐ │ Container (Docker / Podman) │ │ │ -│ ┌───────────┐ DOM events ┌──────────────┐ │ -│ │ content.js├──────────────►│ background.js│ │ -│ │ (per tab) │ │ (service │ │ -│ └───────────┘ │ worker) │ │ -│ └──┬──────┬────┘ │ -│ │ │ │ -│ actions │ │ screenshots -│ │ │ │ -│ ┌──────────┐ ┌──────▼──────▼────┐ │ +│ ┌──────────┐ CDP Fetch/Runtime/Page events │ +│ │ Chromium ├─────────────────────────────┐ │ +│ │ :9222 CDP│ │ │ +│ └──────────┘ │ │ +│ │ │ +│ ┌──────────┐ ┌────────────────▼─┐ │ │ │ Xvfb │◄──ffmpeg──►│ FastAPI Server │ │ │ │ :99 │ x11grab │ :7878 │ │ │ └──────────┘ └──────────────────┘ │ │ │ │ -│ ┌──────────┐ ┌───────▼─────────┐ │ -│ │ Chromium │ │ /data │ │ -│ │ :9222 CDP│ │ actions.jsonl │ │ -│ └──────────┘ │ requests.jsonl │ │ +│ ┌───────▼─────────┐ │ +│ │ /data │ │ +│ │ actions.jsonl │ │ +│ │ requests.jsonl │ │ │ │ screenshots/ │ │ │ │ recording.mp4 │ │ │ └─────────────────┘ │ diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 0f36fc2e..3c1a82a0 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -484,25 +484,22 @@ ClawBench 定位:**真实消费级网站、日常任务、端到端录制**。 ``` ┌─────────────────────────────────────────────────┐ -│ 容器 (Docker / Podman) │ +│ 容器 (Docker / Podman) │ │ │ -│ ┌───────────┐ DOM 事件 ┌──────────────┐ │ -│ │ content.js├──────────────►│ background.js│ │ -│ │ (每个标签)│ │ (service │ │ -│ └───────────┘ │ worker) │ │ -│ └──┬──────┬────┘ │ -│ │ │ │ -│ 动作 │ │ 截图 │ -│ │ │ │ -│ ┌──────────┐ ┌──────▼──────▼────┐ │ +│ ┌──────────┐ CDP Fetch/Runtime/Page 事件 │ +│ │ Chromium ├─────────────────────────────┐ │ +│ │ :9222 CDP│ │ │ +│ └──────────┘ │ │ +│ │ │ +│ ┌──────────┐ ┌────────────────▼─┐ │ │ │ Xvfb │◄──ffmpeg──►│ FastAPI Server │ │ │ │ :99 │ x11grab │ :7878 │ │ │ └──────────┘ └──────────────────┘ │ │ │ │ -│ ┌──────────┐ ┌───────▼─────────┐ │ -│ │ Chromium │ │ /data │ │ -│ │ :9222 CDP│ │ actions.jsonl │ │ -│ └──────────┘ │ requests.jsonl │ │ +│ ┌───────▼─────────┐ │ +│ │ /data │ │ +│ │ actions.jsonl │ │ +│ │ requests.jsonl │ │ │ │ screenshots/ │ │ │ │ recording.mp4 │ │ │ └─────────────────┘ │ diff --git a/src/README.md b/src/README.md index 780cb476..bdacab2a 100644 --- a/src/README.md +++ b/src/README.md @@ -46,14 +46,12 @@ src/ paths.py # Shared PROJECT_ROOT / HARNESS_ROOT discovery hf_upload.py # Optional HuggingFace upload helpers extension-server/ - server.py # FastAPI server for actions, screenshots, recording, request interception + server.py # CDP instrumentation server for actions, screenshots, requests, recording pyproject.toml # Container-only uv project uv.lock README.md chrome-extension/ manifest.json - content.js # DOM action capture - background.js # Screenshot capture + server relay stealth.js # Browser fingerprint hardening patches setup.sh # Local extension launch helper README.md diff --git a/src/clawbench/runtime/chrome-extension/README.md b/src/clawbench/runtime/chrome-extension/README.md index 6839a35a..4506434b 100644 --- a/src/clawbench/runtime/chrome-extension/README.md +++ b/src/clawbench/runtime/chrome-extension/README.md @@ -1,14 +1,13 @@ # ClawBench Chrome Extension -This is the source code for the ClawBench Chrome Extension, which acts as the client for the ClawBench benchmarking framework. +This is the source code for the ClawBench Chrome Extension, which provides browser fingerprint hardening for benchmark runs. The extension is responsible for the following tasks: -- Collecting every browser action performed by the user or agent and sending it to the ClawBench server. -- Taking screenshots after browser actions, with high-frequency events throttled. -- Keeping the active tab aligned with the agent's work so server-side screen recording and screenshots capture the right browser state. +- Injecting `stealth.js` at `document_start` in the page's `MAIN` world. +- Applying anti-bot-detection patches for webdriver, plugins, WebGL, permissions, and related browser fingerprints. -The extension should auto start when any non-built-in page is loaded, and should stop when the browser is closed. No UI or configuration is needed for the extension, as all configuration is done on the server side. +Action capture, screenshot capture, request logging, request interception, and tab activation are handled by the runtime server over CDP. The extension does not send data to the server. A `setup.sh` script is provided to load the extension into Chrome. Linux and macOS are supported. @@ -16,15 +15,13 @@ A `setup.sh` script is provided to load the extension into Chrome. Linux and mac | File | Description | |------|-------------| -| `manifest.json` | Manifest V3 extension definition. Permissions: `activeTab`, `tabs`. Content scripts injected on all URLs. | +| `manifest.json` | Manifest V3 extension definition. Loads `stealth.js` on all URLs. | | `stealth.js` | Anti-bot-detection patches. Runs at `document_start` in `MAIN` world. Overrides `navigator.webdriver`, plugins, WebGL, permissions, etc. | -| `content.js` | Injected into every non-chrome:// page. Listens for DOM events, extracts metadata, sends to background. Runs at `document_idle` in `ISOLATED` world. | -| `background.js` | Service worker. Relays actions to server via HTTP POST. Captures screenshots with `chrome.tabs.captureVisibleTab`. | | `setup.sh` | Detects Chrome/Chromium binary on macOS or Linux and launches with `--load-extension` and remote debugging enabled. | ## Event Capture -### Captured Events +Browser event capture now lives in `src/clawbench/runtime/extension-server/server.py`. The server injects capture listeners through CDP and preserves the historical action schema: `click`, `keydown`, `keyup`, `input`, `scroll`, `change`, `submit`, plus a synthetic `pageLoad` on each navigation. @@ -123,5 +120,3 @@ Run Chrome with the extension loaded: ```bash ./setup.sh https://example.com ``` - -The server must be running on `http://localhost:7878` for the extension to send data. diff --git a/src/clawbench/runtime/chrome-extension/background.js b/src/clawbench/runtime/chrome-extension/background.js deleted file mode 100644 index 5771c7d1..00000000 --- a/src/clawbench/runtime/chrome-extension/background.js +++ /dev/null @@ -1,56 +0,0 @@ -const SERVER = "http://localhost:7878"; -const SCREENSHOT_THROTTLE_MS = 500; // debouncing; otherwise one field input (multi input event) can trigger many screenshots - -let lastScreenshot = 0; - -// Auto-focus newly created tabs so the agent's working tab is always visible -chrome.tabs.onCreated.addListener((tab) => { - if (tab.id) { - chrome.tabs.update(tab.id, { active: true }); - } -}); - -// Receive events from content script -chrome.runtime.onMessage.addListener((msg, sender) => { - if (msg.type === "action") { - // Bring the tab where the action occurred to front so the screen recording - // and captureVisibleTab always show the tab the agent is working on. - if (sender.tab && sender.tab.id) { - chrome.tabs.update(sender.tab.id, { active: true }); - } - postAction(msg.data); - captureScreenshot(); - } -}); - -async function postAction(data) { - try { - await fetch(`${SERVER}/api/action`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify(data), - }); - } catch (e) { - console.error("[clawbench] postAction failed:", e); - } -} - -async function captureScreenshot() { - const now = Date.now(); - if (now - lastScreenshot < SCREENSHOT_THROTTLE_MS) return; - lastScreenshot = now; - - try { - const dataUrl = await chrome.tabs.captureVisibleTab(null, { - format: "png", - }); - const base64 = dataUrl.replace(/^data:image\/png;base64,/, ""); - await fetch(`${SERVER}/api/screenshot`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ timestamp: now, data: base64 }), - }); - } catch (e) { - console.error("[clawbench] captureScreenshot failed:", e); - } -} diff --git a/src/clawbench/runtime/chrome-extension/content.js b/src/clawbench/runtime/chrome-extension/content.js deleted file mode 100644 index 5048289e..00000000 --- a/src/clawbench/runtime/chrome-extension/content.js +++ /dev/null @@ -1,70 +0,0 @@ -const THROTTLE_MS = 500; -const lastSent = {}; - -function getXPath(el) { - if (!el || el.nodeType !== 1) return ""; - const parts = []; - while (el && el.nodeType === 1) { - let idx = 1; - for (let sib = el.previousElementSibling; sib; sib = sib.previousElementSibling) { - if (sib.tagName === el.tagName) idx++; - } - parts.unshift(`${el.tagName.toLowerCase()}[${idx}]`); - el = el.parentElement; - } - return "/" + parts.join("/"); -} - -function buildPayload(type, e) { - const target = e.target || {}; - const payload = { - type, - timestamp: Date.now(), - url: location.href, - target: { - tagName: target.tagName || "", - id: target.id || "", - className: target.className || "", - textContent: (target.textContent || "").slice(0, 100), - xpath: getXPath(target), - }, - }; - if (e.clientX !== undefined) { - payload.x = e.clientX; - payload.y = e.clientY; - } - if (e.key) payload.key = e.key; - if (target.value !== undefined) payload.value = String(target.value).slice(0, 200); - if (type === "scroll") { - payload.scrollX = window.scrollX; - payload.scrollY = window.scrollY; - } - return payload; -} - -function throttled(type) { - return type === "scroll" || type === "input"; -} - -function send(type, e) { - if (throttled(type)) { - const now = Date.now(); - if (lastSent[type] && now - lastSent[type] < THROTTLE_MS) return; - lastSent[type] = now; - } - chrome.runtime.sendMessage({ type: "action", data: buildPayload(type, e) }); -} - -["click", "keydown", "keyup", "input", "scroll", "change", "submit"].forEach((evt) => { - document.addEventListener(evt, (e) => send(evt, e), true); -}); - -chrome.runtime.sendMessage({ - type: "action", - data: { - type: "pageLoad", - timestamp: Date.now(), - url: location.href, - title: document.title, - }, -}); diff --git a/src/clawbench/runtime/chrome-extension/manifest.json b/src/clawbench/runtime/chrome-extension/manifest.json index 7ee24e88..3feba6e5 100644 --- a/src/clawbench/runtime/chrome-extension/manifest.json +++ b/src/clawbench/runtime/chrome-extension/manifest.json @@ -2,24 +2,15 @@ "manifest_version": 3, "name": "ClawBench", "version": "1.0", - "description": "Browser action recording for benchmarking", - "permissions": ["activeTab", "tabs"], + "description": "Anti-bot-detection browser configuration for benchmarking", + "permissions": [], "host_permissions": [""], - "background": { - "service_worker": "background.js" - }, "content_scripts": [ { "matches": [""], "js": ["stealth.js"], "run_at": "document_start", "world": "MAIN" - }, - { - "matches": [""], - "js": ["content.js"], - "run_at": "document_idle", - "all_frames": true } ] } diff --git a/src/clawbench/runtime/extension-server/README.md b/src/clawbench/runtime/extension-server/README.md index 1cdde551..dfebc747 100644 --- a/src/clawbench/runtime/extension-server/README.md +++ b/src/clawbench/runtime/extension-server/README.md @@ -1,11 +1,11 @@ -# ClawBench Extension Server +# ClawBench Runtime Server -The ClawBench Extension Server is a Python backend instrumentation server that receives data from the ClawBench Chrome Extension and processes it for benchmarking purposes. It is responsible for: +The ClawBench Runtime Server is a Python backend instrumentation server that connects to Chromium over CDP and writes benchmark artifacts. It is responsible for: -- Organizing and assembling the data received from the browser extension in a structured format. -- Receiving user actions and storing them in a jsonl format. -- Receiving screenshots and storing them in a dedicated folder. -- Receiving and converting session recording chunks into .mp4 files when the session is complete. +- Capturing browser DOM actions through CDP-injected listeners and storing them in jsonl format. +- Capturing screenshots through CDP after browser actions. +- Logging HTTP requests and blocking matching eval-schema requests through CDP's `Fetch` domain. +- Recording the Xvfb display to `recording.mp4`. ## Implementation @@ -16,8 +16,8 @@ Single `server.py` — a FastAPI application run with uvicorn. | Method | Path | Content-Type | Description | | ------ | --------------------- | ---------------- | --------------------------------------------------------------------------------------- | | GET | `/api/status` | — | Returns `{"status": "ok"}` | -| POST | `/api/action` | application/json | Appends action JSON to `actions.jsonl` | -| POST | `/api/screenshot` | application/json | Decodes base64 PNG from `{"timestamp", "data"}`, saves to `screenshots/{timestamp}.png` | +| POST | `/api/action` | application/json | Compatibility endpoint; CDP capture writes actions directly | +| POST | `/api/screenshot` | application/json | Compatibility endpoint; CDP capture writes screenshots directly | | POST | `/api/stop` | — | Signals session stop, returns session summary | | POST | `/api/stop-recording` | — | Stops ffmpeg recording, finalizes MP4 | diff --git a/src/clawbench/runtime/extension-server/server.py b/src/clawbench/runtime/extension-server/server.py index 07e452d6..9776d661 100644 --- a/src/clawbench/runtime/extension-server/server.py +++ b/src/clawbench/runtime/extension-server/server.py @@ -24,12 +24,117 @@ INTERCEPTION_FILE = DATA_DIR / "interception.json" CDP_URL = "http://127.0.0.1:9222" +ACTION_BINDING = "__clawbenchAction" +SCREENSHOT_THROTTLE_MS = 500 ffmpeg_proc = None eval_schema = None eval_interceptor_ready = False +ACTION_CAPTURE_SCRIPT = r""" +(function () { + "use strict"; + + if (window.__clawbenchActionCaptureInstalled) return; + window.__clawbenchActionCaptureInstalled = true; + + const THROTTLE_MS = 500; + const lastSent = {}; + + function getXPath(el) { + if (!el || el.nodeType !== 1) return ""; + const parts = []; + while (el && el.nodeType === 1) { + let idx = 1; + for (let sib = el.previousElementSibling; sib; sib = sib.previousElementSibling) { + if (sib.tagName === el.tagName) idx++; + } + parts.unshift(`${el.tagName.toLowerCase()}[${idx}]`); + el = el.parentElement; + } + return "/" + parts.join("/"); + } + + function classNameFor(target) { + if (!target || target.className === undefined) return ""; + if (typeof target.className === "string") return target.className; + if (target.className && typeof target.className.baseVal === "string") { + return target.className.baseVal; + } + return String(target.className || ""); + } + + function emit(payload) { + try { + if (typeof window.__clawbenchAction === "function") { + window.__clawbenchAction(JSON.stringify(payload)); + } + } catch (_) {} + } + + function buildPayload(type, e) { + const target = e.target || {}; + const payload = { + type, + timestamp: Date.now(), + url: location.href, + target: { + tagName: target.tagName || "", + id: target.id || "", + className: classNameFor(target), + textContent: (target.textContent || "").slice(0, 100), + xpath: getXPath(target), + }, + }; + if (e.clientX !== undefined) { + payload.x = e.clientX; + payload.y = e.clientY; + } + if (e.key) payload.key = e.key; + if (target.value !== undefined) payload.value = String(target.value).slice(0, 200); + if (type === "scroll") { + payload.scrollX = window.scrollX; + payload.scrollY = window.scrollY; + } + return payload; + } + + function throttled(type) { + return type === "scroll" || type === "input"; + } + + function send(type, e) { + if (throttled(type)) { + const now = Date.now(); + if (lastSent[type] && now - lastSent[type] < THROTTLE_MS) return; + lastSent[type] = now; + } + emit(buildPayload(type, e)); + } + + ["click", "keydown", "keyup", "input", "scroll", "change", "submit"].forEach((evt) => { + document.addEventListener(evt, (e) => send(evt, e), true); + }); + + function sendPageLoad() { + emit({ + type: "pageLoad", + timestamp: Date.now(), + url: location.href, + title: document.title, + }); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", sendPageLoad, { once: true }); + } else { + setTimeout(sendPageLoad, 0); + } +})(); +""" + + def _const_fields_match(expected, actual): """Check that all key-value pairs in expected match in actual data. For list bodies (batched GraphQL), returns True if any item matches. @@ -96,6 +201,12 @@ def _log_request(log_file, params): log_file.flush() +def _log_action(log_file, payload): + """Append a browser action payload to actions.jsonl.""" + log_file.write(json.dumps(payload) + "\n") + log_file.flush() + + def start_cdp_handler( url_pattern=None, required_method=None, match_body=None, match_params=None ): @@ -122,11 +233,13 @@ def start_cdp_handler( msg_id = [1] def send(method, params=None, session_id=None): - msg = {"id": msg_id[0], "method": method, "params": params or {}} + current_id = msg_id[0] + msg = {"id": current_id, "method": method, "params": params or {}} if session_id: msg["sessionId"] = session_id ws.send(json.dumps(msg)) msg_id[0] += 1 + return current_id # Auto-attach to all targets with flatten so events come on this connection. # waitForDebuggerOnStart=True pauses new targets until we explicitly resume @@ -147,12 +260,41 @@ def send(method, params=None, session_id=None): else: print("[cdp] Request logger connected (no intercept pattern)", flush=True) - # Track sessions where Fetch is enabled, and map sessions to target IDs - # so we can bring the correct tab to front when it receives activity. + # Track page sessions and async screenshot requests. Fetch handles network + # logging/interception; Runtime bindings carry DOM action payloads back to + # the server; Page.captureScreenshot replaces extension screenshots. fetch_sessions = set() + instrumented_sessions = set() session_to_target = {} # sessionId -> targetId active_target = [None] # mutable ref: currently active targetId - log_file = open(REQUESTS_FILE, "a") + pending_screenshots = {} # CDP command id -> timestamp + last_screenshot = [0.0] + requests_log_file = open(REQUESTS_FILE, "a") + actions_log_file = open(ACTIONS_FILE, "a") + + def request_screenshot(session_id, timestamp): + if not session_id: + return + now = time.time() * 1000 + if now - last_screenshot[0] < SCREENSHOT_THROTTLE_MS: + return + last_screenshot[0] = now + screenshot_id = send( + "Page.captureScreenshot", + {"format": "png", "captureBeyondViewport": False}, + session_id, + ) + pending_screenshots[screenshot_id] = timestamp + + def activate_session_target(session_id, reason): + target_id = session_to_target.get(session_id) + if target_id and target_id != active_target[0]: + send("Target.activateTarget", {"targetId": target_id}) + active_target[0] = target_id + print( + f"[cdp] Auto-focused tab {target_id[:12]}... ({reason})", + flush=True, + ) try: while True: @@ -163,6 +305,20 @@ def send(method, params=None, session_id=None): msg = json.loads(raw) session_id = msg.get("sessionId") + if msg.get("id") in pending_screenshots: + ts = pending_screenshots.pop(msg["id"]) + data = msg.get("result", {}).get("data") + if data: + try: + (SCREENSHOTS_DIR / f"{ts}.png").write_bytes( + base64.b64decode(data) + ) + except Exception as e: + print(f"[cdp] Screenshot write failed: {e}", flush=True) + elif "error" in msg: + print(f"[cdp] Screenshot failed: {msg['error']}", flush=True) + continue + # When a new target attaches, enable Fetch then resume execution. # Because waitForDebuggerOnStart=True, the target is paused until # we call Runtime.runIfWaitingForDebugger — this avoids the @@ -174,6 +330,29 @@ def send(method, params=None, session_id=None): target_id = msg["params"]["targetInfo"]["targetId"] if target_type == "page": session_to_target[child_session] = target_id + if child_session not in instrumented_sessions: + send("Runtime.enable", {}, child_session) + send("Page.enable", {}, child_session) + send( + "Runtime.addBinding", + {"name": ACTION_BINDING}, + child_session, + ) + send( + "Page.addScriptToEvaluateOnNewDocument", + {"source": ACTION_CAPTURE_SCRIPT}, + child_session, + ) + send( + "Runtime.evaluate", + {"expression": ACTION_CAPTURE_SCRIPT}, + child_session, + ) + instrumented_sessions.add(child_session) + print( + f"[cdp] Action capture enabled on session {child_session[:12]}...", + flush=True, + ) if child_session not in fetch_sessions: send( "Fetch.enable", @@ -193,6 +372,22 @@ def send(method, params=None, session_id=None): send("Runtime.runIfWaitingForDebugger", {}, child_session) continue + if msg.get("method") == "Runtime.bindingCalled": + params = msg.get("params", {}) + if params.get("name") != ACTION_BINDING: + continue + try: + payload = json.loads(params.get("payload", "{}")) + except json.JSONDecodeError: + print("[cdp] Ignoring malformed action payload", flush=True) + continue + activate_session_target(session_id, "action") + _log_action(actions_log_file, payload) + request_screenshot( + session_id, payload.get("timestamp", int(time.time() * 1000)) + ) + continue + if msg.get("method") != "Fetch.requestPaused": if "error" in msg and msg.get("id"): print(f"[cdp] CDP error: {msg['error']}", flush=True) @@ -207,17 +402,10 @@ def send(method, params=None, session_id=None): # screenshots always show the tab the agent is working on. resource_type = params.get("resourceType", "") if resource_type == "Document" and session_id: - target_id = session_to_target.get(session_id) - if target_id and target_id != active_target[0]: - send("Target.activateTarget", {"targetId": target_id}) - active_target[0] = target_id - print( - f"[cdp] Auto-focused tab {target_id[:12]}... (Document request)", - flush=True, - ) + activate_session_target(session_id, "Document request") # Log every non-internal request - _log_request(log_file, params) + _log_request(requests_log_file, params) # If no intercept pattern, just continue the request if not url_pattern: @@ -280,7 +468,8 @@ def send(method, params=None, session_id=None): except Exception: pass finally: - log_file.close() + requests_log_file.close() + actions_log_file.close() ws.close() From 1d483351e8e9e22ae6e5ff2267204b4c8e2bae5c Mon Sep 17 00:00:00 2001 From: Perry Zhu Date: Fri, 12 Jun 2026 20:47:58 -0700 Subject: [PATCH 2/2] refactor: rename extension-server to runtime-server --- AGENTS.md | 4 ++-- CONTRIBUTING.md | 4 ++-- src/README.md | 2 +- src/clawbench/runtime/chrome-extension/README.md | 2 +- src/clawbench/runtime/harnesses/base/Dockerfile.base | 6 +++--- src/clawbench/runtime/harnesses/base/entrypoint.sh | 2 +- .../{extension-server => runtime-server}/README.md | 8 ++++---- .../{extension-server => runtime-server}/pyproject.toml | 4 ++-- .../{extension-server => runtime-server}/server.py | 0 .../runtime/{extension-server => runtime-server}/uv.lock | 2 +- 10 files changed, 17 insertions(+), 17 deletions(-) rename src/clawbench/runtime/{extension-server => runtime-server}/README.md (93%) rename src/clawbench/runtime/{extension-server => runtime-server}/pyproject.toml (61%) rename src/clawbench/runtime/{extension-server => runtime-server}/server.py (100%) rename src/clawbench/runtime/{extension-server => runtime-server}/uv.lock (99%) diff --git a/AGENTS.md b/AGENTS.md index f6555f25..5e0e6ae2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,7 +25,7 @@ ClawBench/ runtime/ shared/ alex_green_personal_info.json - extension-server/ # Container-only uv project for server deps + runtime-server/ # Container-only uv project for server deps chrome-extension/ # Recording extension harnesses/ # Dockerfiles + setup/run scripts models/ @@ -139,7 +139,7 @@ test-output//---/ ## Key Documentation - [README.md#-cli](README.md#-cli) -- CLI usage, batch runner flags, output format -- [src/clawbench/runtime/extension-server/README.md](src/clawbench/runtime/extension-server/README.md) -- FastAPI server, endpoints, screen recording +- [src/clawbench/runtime/runtime-server/README.md](src/clawbench/runtime/runtime-server/README.md) -- FastAPI server, endpoints, screen recording - [CONTRIBUTING.md](CONTRIBUTING.md) -- how to add new test cases - [eval/README.md](eval/README.md) -- evaluation guide and Claude Code prompt template - [eval/agentic_eval.md](eval/agentic_eval.md) -- evaluator rubric for PASS/FAIL judgment diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6f9b693e..560083ee 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -108,11 +108,11 @@ If the task requires additional context (e.g., a pre-filled profile, a specific ## Code changes -For changes to the framework itself (test driver, extension server, Chrome extension, container): +For changes to the framework itself (test driver, runtime server, Chrome extension, container): 1. Read the relevant sub-README for component-specific documentation: - [README.md#-cli](README.md#-cli) - - [src/clawbench/runtime/extension-server/README.md](src/clawbench/runtime/extension-server/README.md) + - [src/clawbench/runtime/runtime-server/README.md](src/clawbench/runtime/runtime-server/README.md) - [src/clawbench/runtime/chrome-extension/README.md](src/clawbench/runtime/chrome-extension/README.md) 2. Open an issue first for anything beyond a small bug fix so we can align on approach before you spend time. 3. Open a PR with a clear description of the change and how you tested it. diff --git a/src/README.md b/src/README.md index bdacab2a..f3642a40 100644 --- a/src/README.md +++ b/src/README.md @@ -45,7 +45,7 @@ src/ utils/ paths.py # Shared PROJECT_ROOT / HARNESS_ROOT discovery hf_upload.py # Optional HuggingFace upload helpers - extension-server/ + runtime-server/ server.py # CDP instrumentation server for actions, screenshots, requests, recording pyproject.toml # Container-only uv project uv.lock diff --git a/src/clawbench/runtime/chrome-extension/README.md b/src/clawbench/runtime/chrome-extension/README.md index 4506434b..9d394dbb 100644 --- a/src/clawbench/runtime/chrome-extension/README.md +++ b/src/clawbench/runtime/chrome-extension/README.md @@ -21,7 +21,7 @@ A `setup.sh` script is provided to load the extension into Chrome. Linux and mac ## Event Capture -Browser event capture now lives in `src/clawbench/runtime/extension-server/server.py`. The server injects capture listeners through CDP and preserves the historical action schema: +Browser event capture now lives in `src/clawbench/runtime/runtime-server/server.py`. The server injects capture listeners through CDP and preserves the historical action schema: `click`, `keydown`, `keyup`, `input`, `scroll`, `change`, `submit`, plus a synthetic `pageLoad` on each navigation. diff --git a/src/clawbench/runtime/harnesses/base/Dockerfile.base b/src/clawbench/runtime/harnesses/base/Dockerfile.base index bfb023c8..801149b3 100644 --- a/src/clawbench/runtime/harnesses/base/Dockerfile.base +++ b/src/clawbench/runtime/harnesses/base/Dockerfile.base @@ -23,12 +23,12 @@ RUN git clone --depth 1 --branch v1.6.0 https://github.com/novnc/noVNC.git /opt/ COPY --from=ghcr.io/astral-sh/uv:0.11.6 /uv /usr/local/bin/uv WORKDIR /app -COPY extension-server/pyproject.toml extension-server/uv.lock ./src/extension-server/ -WORKDIR /app/src/extension-server +COPY runtime-server/pyproject.toml runtime-server/uv.lock ./src/runtime-server/ +WORKDIR /app/src/runtime-server RUN UV_PYTHON_PREFERENCE=only-system uv sync --frozen WORKDIR /app -COPY extension-server/server.py ./src/extension-server/server.py +COPY runtime-server/server.py ./src/runtime-server/server.py COPY chrome-extension/ ./src/chrome-extension/ diff --git a/src/clawbench/runtime/harnesses/base/entrypoint.sh b/src/clawbench/runtime/harnesses/base/entrypoint.sh index 07852bbb..7166e92e 100644 --- a/src/clawbench/runtime/harnesses/base/entrypoint.sh +++ b/src/clawbench/runtime/harnesses/base/entrypoint.sh @@ -10,7 +10,7 @@ export DISPLAY=:99 sleep 1 # Start the server -cd /app/src/extension-server +cd /app/src/runtime-server uv run --no-sync uvicorn server:app --host 0.0.0.0 --port 7878 & sleep 1 diff --git a/src/clawbench/runtime/extension-server/README.md b/src/clawbench/runtime/runtime-server/README.md similarity index 93% rename from src/clawbench/runtime/extension-server/README.md rename to src/clawbench/runtime/runtime-server/README.md index dfebc747..2cbb4f7f 100644 --- a/src/clawbench/runtime/extension-server/README.md +++ b/src/clawbench/runtime/runtime-server/README.md @@ -42,18 +42,18 @@ All data is written to the directory specified by `CLAWBENCH_DATA_DIR` (default: ### Running Locally -The extension server normally runs only inside the benchmark container. For +The runtime server normally runs only inside the benchmark container. For debugging the server by itself, use its local uv project: ```bash -cd src/clawbench/runtime/extension-server +cd src/clawbench/runtime/runtime-server CLAWBENCH_DATA_DIR=./data DISPLAY=:99 uv run --frozen uvicorn server:app --host 0.0.0.0 --port 7878 ``` ### Dependencies -The extension server is container-only and has its own uv project in -`src/clawbench/runtime/extension-server/`: +The runtime server is container-only and has its own uv project in +`src/clawbench/runtime/runtime-server/`: - `fastapi` — web framework - `uvicorn` — ASGI server - `websocket-client` — WebSocket client for CDP communication diff --git a/src/clawbench/runtime/extension-server/pyproject.toml b/src/clawbench/runtime/runtime-server/pyproject.toml similarity index 61% rename from src/clawbench/runtime/extension-server/pyproject.toml rename to src/clawbench/runtime/runtime-server/pyproject.toml index 2781d209..c8ddcdee 100644 --- a/src/clawbench/runtime/extension-server/pyproject.toml +++ b/src/clawbench/runtime/runtime-server/pyproject.toml @@ -1,7 +1,7 @@ [project] -name = "clawbench-extension-server" +name = "clawbench-runtime-server" version = "0.1.0" -description = "In-container ClawBench extension data collection server" +description = "In-container ClawBench CDP instrumentation server" requires-python = ">=3.11" dependencies = [ "fastapi>=0.115", diff --git a/src/clawbench/runtime/extension-server/server.py b/src/clawbench/runtime/runtime-server/server.py similarity index 100% rename from src/clawbench/runtime/extension-server/server.py rename to src/clawbench/runtime/runtime-server/server.py diff --git a/src/clawbench/runtime/extension-server/uv.lock b/src/clawbench/runtime/runtime-server/uv.lock similarity index 99% rename from src/clawbench/runtime/extension-server/uv.lock rename to src/clawbench/runtime/runtime-server/uv.lock index 68c5dc39..5d005f4d 100644 --- a/src/clawbench/runtime/extension-server/uv.lock +++ b/src/clawbench/runtime/runtime-server/uv.lock @@ -39,7 +39,7 @@ wheels = [ ] [[package]] -name = "clawbench-extension-server" +name = "clawbench-runtime-server" version = "0.1.0" source = { virtual = "." } dependencies = [