Merge pull request #55 from reacher-z/feat/12-support-opencode-harness

feat: support opencode harness

Author: Perry2004

AGENTS.md

@@ -4,15 +4,21 @@ This file is for coding agents (Claude Code, Cursor, Copilot, etc.) to understan
 
 ## What This Is
 
-ClawBench is a benchmarking framework for evaluating AI web agents on 153 real-world online tasks spanning 144 live websites and 15 life categories. Each task runs in an isolated Docker container with Chromium, a recording Chrome extension, and an AI agent (OpenClaw). The framework captures five layers of data: session replay (MP4), action screenshots, HTTP traffic, browser actions, and agent messages.
+ClawBench is a benchmarking framework for evaluating AI web agents on 153 real-world online tasks spanning 144 live websites and 15 life categories. Each task runs in an isolated Docker container with Chromium, a recording Chrome extension, and an AI agent harness (`openclaw` or `opencode`, selectable via `--harness`). The framework captures five layers of data: session replay (MP4), action screenshots, HTTP traffic, browser actions, and agent messages.
 
 ## Project Structure
 
 ```
 ClawBench/
   run.sh                          # Entry point -- launches interactive TUI
-  Dockerfile                      # Container image (Chromium + extension + agent)
-  entrypoint.sh                   # Container entrypoint (orchestrates all services)
+  Dockerfile.base                 # Base image (Chromium + extension + Node + uv, harness-agnostic)
+  Dockerfile.openclaw             # Layer that adds the openclaw CLI on top of base
+  Dockerfile.opencode             # Layer that adds opencode + @playwright/mcp on top of base
+  entrypoint.sh                   # Shared infra (Xvfb, Chrome, noVNC, human mode); execs /run-harness.sh in agent mode
+  setup-openclaw.sh               # Generates ~/.openclaw/openclaw.json from env vars (called from run-openclaw.sh)
+  setup-opencode.sh               # Generates ~/.config/opencode/opencode.json from env vars (called from run-opencode.sh)
+  run-openclaw.sh                 # Per-harness agent runner; copied into clawbench-openclaw image as /run-harness.sh
+  run-opencode.sh                 # Per-harness agent runner; copied into clawbench-opencode image as /run-harness.sh
   .env.example                    # Template for PurelyMail credentials
   models/
     models.yaml                   # Model API configs (gitignored -- copy from example)
@@ -64,13 +70,13 @@ cp models/models.example.yaml models/models.yaml
 ./run.sh
 
 # Single run:
-uv run --project test-driver test-driver/run.py test-cases/<case-dir> <model-name>
+uv run --project test-driver test-driver/run.py test-cases/<case-dir> <model-name> --harness openclaw
 
 # Batch run (model x case cross-product):
 uv run --project test-driver test-driver/batch.py \
-  --models <model-name> --all-cases --max-concurrent 3
+  --models <model-name> --all-cases --max-concurrent 3 --harness openclaw
 
-# Human mode (manual browser control via noVNC):
+# Human mode (manual browser control via noVNC; no harness needed):
 uv run --project test-driver test-driver/run.py test-cases/<case-dir> --human
 ```
 
@@ -97,13 +103,15 @@ See `test-cases/task.schema.json` for the full schema.
 
 Each run produces:
 ```
-test-output/<model>/<case>-<model>-<timestamp>/
-  run-meta.json             # Run metadata (model, duration, intercepted)
+test-output/<model>/<harness>-<case>-<model>-<timestamp>/
+  run-meta.json             # Run metadata (harness, model, duration, intercepted)
   eval-schema.json          # Schema used for this run
   data/
     actions.jsonl           # Browser action log
     requests.jsonl          # HTTP request log
-    agent-messages.jsonl    # Agent conversation transcript
+    agent-messages.jsonl    # Agent conversation transcript (shape varies by harness:
+                            #   openclaw → session-schema events; opencode → AI-SDK
+                            #   `step_start`/`tool_use`/`text`/`reasoning`/`step_finish` events)
     screenshots/            # Timestamped PNGs
     recording.mp4           # Full session video
     interception.json       # Interception result

Dockerfile Dockerfile.baseDockerfile renamed to Dockerfile.base

@@ -24,14 +24,6 @@ COPY --from=node:24-slim /usr/local/lib/node_modules /usr/local/lib/node_modules
 RUN ln -s /usr/local/lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm \
     && ln -s /usr/local/lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx
 
-# Pin openclaw version — patched below for #47879 (--autoConnect → --browserUrl)
-RUN PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install -g openclaw@2026.3.13
-
-# Patch: replace --autoConnect with --browserUrl http://127.0.0.1:9222 in all dist files
-# See https://github.com/openclaw/openclaw/issues/47879
-RUN find /usr/local/lib/node_modules/openclaw/dist -name '*.js' -exec \
-    sed -i 's/"--autoConnect"/"--browserUrl","http:\/\/127.0.0.1:9222"/g' {} +
-
 COPY --from=ghcr.io/astral-sh/uv:0.11.6 /uv /usr/local/bin/uv
 
 WORKDIR /app
@@ -40,9 +32,6 @@ RUN cd extension-server && UV_PYTHON_PREFERENCE=only-system uv sync
 
 COPY chrome-extension/ ./chrome-extension/
 
-COPY setup-openclaw.sh /setup-openclaw.sh
-RUN chmod +x /setup-openclaw.sh
-
 COPY entrypoint.sh /entrypoint.sh
 RUN chmod +x /entrypoint.sh
 

Dockerfile.openclaw

@@ -0,0 +1,13 @@
+FROM clawbench-base
+
+# Pin openclaw version — patched below for #47879 (--autoConnect → --browserUrl)
+RUN PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install -g openclaw@2026.3.13
+
+# Patch: replace --autoConnect with --browserUrl http://127.0.0.1:9222 in all dist files
+# See https://github.com/openclaw/openclaw/issues/47879
+RUN find /usr/local/lib/node_modules/openclaw/dist -name '*.js' -exec \
+    sed -i 's/"--autoConnect"/"--browserUrl","http:\/\/127.0.0.1:9222"/g' {} +
+
+COPY setup-openclaw.sh /setup-openclaw.sh
+COPY run-openclaw.sh /run-harness.sh
+RUN chmod +x /setup-openclaw.sh /run-harness.sh

Dockerfile.opencode

@@ -0,0 +1,11 @@
+FROM clawbench-base
+
+# Pin opencode and Playwright MCP versions for reproducible builds —
+# matches the openclaw-side `openclaw@2026.3.13` pin pattern.
+RUN PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 npm install -g \
+    opencode-ai@1.4.4 \
+    @playwright/mcp@0.0.70
+
+COPY setup-opencode.sh /setup-opencode.sh
+COPY run-opencode.sh /run-harness.sh
+RUN chmod +x /setup-opencode.sh /run-harness.sh

README.md

@@ -155,7 +155,7 @@ clawbench run 001-daily-life-food-uber-eats claude-sonnet-4-6
 ```
 Once the container starts, the script prints a **noVNC URL** (e.g. `http://localhost:6080/vnc.html`) — open it in your browser to watch the agent operate in real-time. If port 6080 is already in use, an alternative port is chosen automatically.
 
-Results land in `./claw-output/<model>/<timestamp>-001-.../` with the full five-layer recording.
+Results land in `./claw-output/<model>/<harness>-<case>-<model>-<timestamp>/` with the full five-layer recording. The default harness is `openclaw`; pass `--harness opencode` to use [opencode](https://opencode.ai) (driven via the [Playwright MCP server](https://github.com/microsoft/playwright-mcp)) instead.
 
 **(c) Drive the browser yourself via noVNC** — produces a human reference run:
 ```bash
@@ -445,7 +445,7 @@ Yes. Set `export CONTAINER_ENGINE=podman`. The framework auto-detects whichever
 <details>
 <summary><b>What tools can the agent use?</b></summary>
 
-The OpenClaw agent can only use the browser tool and a restricted set of read-only shell commands (`ls`, `cat`, `find`, `grep`, `head`, `tail`, `jq`, `wc`, etc.). Commands that could bypass the browser (`curl`, `python`, `node`, `wget`) are blocked. The agent instruction also explicitly requires browser-only task completion.
+All supported harnesses are sandboxed identically: the agent can only use the browser tool and a restricted set of read-only shell commands (`ls`, `cat`, `find`, `grep`, `head`, `tail`, `jq`, `wc`, etc.). Commands that could bypass the browser (`curl`, `python`, `node`, `wget`) are blocked, as are file `edit`/`write` and `webfetch`. The agent instruction also explicitly requires browser-only task completion.
 
 </details>
 
@@ -560,4 +560,4 @@ If you use ClawBench in your research, please cite:
 
 Apache 2.0 -- see [LICENSE](LICENSE).
 
-Built with [OpenClaw](https://github.com/openclaw/openclaw), [noVNC](https://github.com/novnc/noVNC) (MPL 2.0), and [websockify](https://github.com/novnc/websockify) (LGPL 3.0).
+Built with [OpenClaw](https://github.com/openclaw/openclaw) and [opencode](https://opencode.ai) (selectable harnesses), [Microsoft Playwright MCP](https://github.com/microsoft/playwright-mcp) (browser control bridge for the opencode harness), [noVNC](https://github.com/novnc/noVNC) (MPL 2.0), and [websockify](https://github.com/novnc/websockify) (LGPL 3.0).

README.zh-CN.md

@@ -155,7 +155,7 @@ clawbench run 001-daily-life-food-uber-eats claude-sonnet-4-6
 ```
 容器启动后,脚本会打印一个 **noVNC URL**（如 `http://localhost:6080/vnc.html`）—— 在浏览器中打开即可实时观看 agent 操作。如果 6080 端口被占用,会自动选一个空闲端口。
 
-结果落在 `./claw-output/<model>/<timestamp>-001-.../`,包含完整的五层录制。
+结果落在 `./claw-output/<model>/<harness>-<case>-<model>-<timestamp>/`,包含完整的五层录制。默认 harness 是 `openclaw`；想用 [opencode](https://opencode.ai)（通过 [Playwright MCP](https://github.com/microsoft/playwright-mcp) 驱动浏览器）就加 `--harness opencode`。
 
 **(c) 通过 noVNC 手动控制浏览器** —— 产出人工参考轨迹:
 ```bash
@@ -445,7 +445,7 @@ uv run --project test-driver test-driver/batch.py --all-models --case-range 1-50
 <details>
 <summary><b>智能体可以使用哪些工具?</b></summary>
 
-OpenClaw 智能体只能使用浏览器工具和一组受限的只读 shell 命令 (`ls`、`cat`、`find`、`grep`、`head`、`tail`、`jq`、`wc` 等)。可能绕过浏览器的命令 (`curl`、`python`、`node`、`wget`) 会被阻止。智能体指令也明确要求只通过浏览器完成任务。
+所有支持的 harness 沙箱限制完全一致：智能体只能使用浏览器工具和一组受限的只读 shell 命令 (`ls`、`cat`、`find`、`grep`、`head`、`tail`、`jq`、`wc` 等)。可能绕过浏览器的命令 (`curl`、`python`、`node`、`wget`) 会被阻止，文件 `edit`/`write` 和 `webfetch` 同样被禁。智能体指令也明确要求只通过浏览器完成任务。
 
 </details>
 
@@ -560,4 +560,4 @@ OpenClaw 智能体只能使用浏览器工具和一组受限的只读 shell 命
 
 Apache 2.0 -- 详见 [LICENSE](LICENSE)。
 
-基于以下开源项目构建: [OpenClaw](https://github.com/openclaw/openclaw), [noVNC](https://github.com/novnc/noVNC) (MPL 2.0), [websockify](https://github.com/novnc/websockify) (LGPL 3.0)。
+基于以下开源项目构建: [OpenClaw](https://github.com/openclaw/openclaw) 和 [opencode](https://opencode.ai)（可选的 harness）, [Microsoft Playwright MCP](https://github.com/microsoft/playwright-mcp)（opencode harness 的浏览器控制桥）, [noVNC](https://github.com/novnc/noVNC) (MPL 2.0), [websockify](https://github.com/novnc/websockify) (LGPL 3.0)。

entrypoint.sh

@@ -256,139 +256,18 @@ if [ "$HUMAN_MODE" = "1" ]; then
   exit 0
 fi
 
-# If no INSTRUCTION provided, skip OpenClaw and keep container alive for external use
+# If no INSTRUCTION provided, skip the harness and keep container alive for external use
 if [ -z "$INSTRUCTION" ]; then
-  echo "No INSTRUCTION set, running in manual mode (no OpenClaw agent)."
+  echo "No INSTRUCTION set, running in manual mode (no agent harness)."
   wait -n
   exit 0
 fi
 
-# Generate a shared gateway token so gateway and agent can authenticate
-OPENCLAW_GATEWAY_TOKEN="$(head -c 32 /dev/urandom | od -A n -t x1 | tr -d ' \n')"
-export OPENCLAW_GATEWAY_TOKEN
-
-# Generate OpenClaw config from env vars
-/setup-openclaw.sh
-
-# Copy /my-info/ into the OpenClaw workspace so the agent can access it via ./my-info/
-WORKSPACE=/root/workspace
-mkdir -p "$WORKSPACE"
-if [ -d /my-info ]; then
-  cp -r /my-info "$WORKSPACE/my-info"
-  echo "Copied /my-info to $WORKSPACE/my-info"
-fi
-
-# Wait for Chrome CDP to be ready
-echo "Waiting for Chrome CDP..."
-for i in $(seq 1 30); do
-  if curl -sf http://127.0.0.1:9222/json/version > /dev/null 2>&1; then
-    echo "Chrome CDP ready"
-    break
-  fi
-  if [ "$i" -eq 30 ]; then
-    echo "Chrome CDP not ready after 30s, aborting"
-    echo "chrome_cdp_timeout" > /data/.stop-reason
-    exit 1
-  fi
-  sleep 1
-done
-
-# Start OpenClaw gateway (log to /data for post-mortem debugging)
-openclaw gateway run > /data/gateway.log 2>&1 &
-GATEWAY_PID=$!
-sleep 3
-
-# Check gateway is alive
-if ! kill -0 $GATEWAY_PID 2>/dev/null; then
-  echo "ERROR: OpenClaw gateway died on startup. Log:"
-  cat /data/gateway.log
-  echo "gateway_failed" > /data/.stop-reason
+# Delegate agent mode to the harness-specific runner, installed by the
+# per-harness Dockerfile layer (Dockerfile.openclaw, Dockerfile.opencode, …).
+if [ ! -x /run-harness.sh ]; then
+  echo "ERROR: /run-harness.sh missing — image was built without a harness layer"
+  echo "missing_harness" > /data/.stop-reason
   exit 1
 fi
-echo "OpenClaw gateway running (pid=$GATEWAY_PID)"
-
-# Run the agent from workspace directory (where my-info/ lives)
-echo "Starting OpenClaw agent from $WORKSPACE..."
-TIMEOUT_MS=$(( ${TIME_LIMIT_S:-1800} * 1000 ))
-AGENT_CMD=(openclaw agent --session-id clawbench --message "$INSTRUCTION" --thinking "${THINKING_LEVEL:-medium}" --timeout "$TIMEOUT_MS" --local)
-echo "Agent command: ${AGENT_CMD[*]}"
-if [ -n "$TEMPERATURE" ]; then
-  AGENT_CMD+=(--temperature "$TEMPERATURE")
-fi
-if [ -n "$MAX_TOKENS" ]; then
-  AGENT_CMD+=(--max-tokens "$MAX_TOKENS")
-fi
-cd "$WORKSPACE"
-"${AGENT_CMD[@]}" > /data/agent.log 2>&1 &
-AGENT_PID=$!
-
-# Watchdog: wait for agent activity, then detect idle (no new actions for 300s)
-IDLE_THRESHOLD=300
-MAX_WAIT=${TIME_LIMIT_S:-1800}
-ELAPSED=0
-LAST_SIZE=0
-IDLE=0
-STOP_REASON=""
-
-while kill -0 $AGENT_PID 2>/dev/null && [ "$ELAPSED" -lt "$MAX_WAIT" ]; do
-  sleep 5
-  ELAPSED=$((ELAPSED + 5))
-
-  # Check if server requested stop (eval interceptor matched)
-  if [ -f /data/.stop-requested ]; then
-    echo "Stop requested by server (eval matched), killing agent."
-    STOP_REASON="eval_matched"
-    break
-  fi
-
-  CURRENT_SIZE=$(wc -c < /data/actions.jsonl 2>/dev/null || echo 0)
-
-  if [ "$CURRENT_SIZE" -gt 0 ] && [ "$CURRENT_SIZE" -eq "$LAST_SIZE" ]; then
-    IDLE=$((IDLE + 5))
-    if [ "$IDLE" -ge "$IDLE_THRESHOLD" ]; then
-      echo "Agent idle for ${IDLE_THRESHOLD}s, assuming done."
-      STOP_REASON="agent_idle"
-      break
-    fi
-  else
-    IDLE=0
-  fi
-  LAST_SIZE=$CURRENT_SIZE
-done
-
-# Determine stop reason if not set (loop exited without breaking)
-if [ -z "$STOP_REASON" ]; then
-  if ! kill -0 $AGENT_PID 2>/dev/null; then
-    STOP_REASON="agent_exited"
-  else
-    echo "Time limit (${MAX_WAIT}s) exceeded, killing agent."
-    STOP_REASON="time_limit_exceeded"
-  fi
-fi
-
-echo "$STOP_REASON" > /data/.stop-reason
-
-# Kill all openclaw processes
-kill $AGENT_PID 2>/dev/null || true
-kill $GATEWAY_PID 2>/dev/null || true
-pkill -f "openclaw" 2>/dev/null || true
-sleep 2
-
-# Copy OpenClaw session transcript to /data
-cp /root/.openclaw/agents/main/sessions/clawbench.jsonl /data/agent-messages.jsonl 2>/dev/null || true
-
-# Finalize bookkeeping (eval promotion, etc.) — recording keeps running
-curl -sf -X POST http://localhost:7878/api/stop || true
-
-# Clean up internal marker (created by /api/stop)
-rm -f /data/.stop-requested
-
-# Grace period: keep recording for 15s after agent is killed to capture end result
-echo "Agent finished, recording grace period (15s)..."
-sleep 15
-
-# Stop recording
-echo "Stopping recording..."
-curl -sf -X POST http://localhost:7878/api/stop-recording || true
-sleep 2
-echo "Done."
+exec /run-harness.sh