feat: v1.3.2 DOM-state capture (Tier 2) — serialize iframe DOM to compact JSON, context injection callback, DOM State panel

Author: unbug

README.md

@@ -626,7 +626,7 @@ in-browser computer-use system.
       `sandbox="allow-scripts"` iframe → `html2canvas` screenshot → attach as
       image to next LLM call → iterate; debounced auto-refresh + manual ↻ Run;
       reuses the iframe infra planned for v0.31 Code Arena
-- [ ] **DOM-state capture (Tier 2)** — serialize active iframe's accessibility
+- [x] **DOM-state capture (Tier 2)** — serialize active iframe's accessibility
       tree (ARIA roles, labels, input states) to compact JSON injected into context
       before each model turn; depth + node-count budget to stay token-safe
 - [ ] **Vision mode (Tier 1/2)** — `OffscreenCanvas` / `html2canvas` screenshot

src/App.css

@@ -17629,6 +17629,66 @@ textarea:focus:not(:focus-visible),
   font-style: italic;
 }
 
+/* DOM State capture (Tier 2) */
+.agent-loop-domstate-section {
+  border-top: 1px solid var(--border-color);
+}
+
+.agent-loop-domstate-header {
+  padding: 8px 20px;
+  background: var(--surface-secondary);
+}
+
+.agent-loop-domstate-toggle {
+  display: flex;
+  align-items: center;
+  gap: 6px;
+  width: 100%;
+  background: none;
+  border: none;
+  color: var(--text-secondary);
+  font-size: 13px;
+  font-weight: 600;
+  cursor: pointer;
+  padding: 0;
+  text-align: left;
+}
+
+.agent-loop-domstate-toggle:hover {
+  color: var(--text-primary);
+}
+
+.agent-loop-domstate-toggle.active {
+  color: var(--text-primary);
+}
+
+.agent-loop-domstate-badge {
+  font-size: 11px;
+  font-weight: 400;
+  color: var(--text-muted);
+  background: var(--surface-tertiary);
+  padding: 2px 8px;
+  border-radius: 10px;
+  margin-left: auto;
+}
+
+.agent-loop-domstate-source {
+  padding: 12px 20px;
+  margin: 0;
+  font-size: 11px;
+  color: var(--text-secondary);
+  background: var(--surface-tertiary);
+  overflow-x: auto;
+  max-height: 300px;
+  overflow-y: auto;
+  white-space: pre;
+  line-height: 1.5;
+}
+
+.agent-loop-domstate-source code {
+  font-family: 'SF Mono', 'Fira Code', 'Cascadia Code', monospace;
+}
+
 .agent-loop-screenshot-section {
   border-top: 1px solid var(--border-color);
 }
@@ -17873,6 +17933,16 @@ textarea:focus:not(:focus-visible),
   color: #374151;
 }
 
+[data-theme='light'] .agent-loop-domstate-source {
+  background: #f9fafb;
+  color: #374151;
+}
+
+[data-theme='light'] .agent-loop-domstate-badge {
+  background: #e5e7eb;
+  color: #6b7280;
+}
+
 [data-theme='light'] .agent-loop-step-tool-args {
   background: #f3f4f6;
 }

src/components/AgentLoopPanel.tsx

@@ -20,6 +20,8 @@ interface Props {
   onScreenshotReady?: (dataUrl: string) => void
   /** When the model generates HTML, this callback receives it for rendering */
   onHtmlGenerated?: (html: string) => void
+  /** When DOM state is captured, this callback receives the serialized DOM for context injection */
+  onDomStateReady?: (domState: string) => void
   /** Current task goal from the agent engine */
   taskGoal?: string
   /** Steps from the agent engine */
@@ -65,6 +67,7 @@ export function AgentLoopPanel({
   onBack,
   onScreenshotReady,
   onHtmlGenerated,
+  onDomStateReady,
   taskGoal,
   steps,
   agentRunning,
@@ -75,10 +78,12 @@ export function AgentLoopPanel({
   const [activeIteration, setActiveIteration] = useState<number | null>(null)
   const [iframeVisible, setIframeVisible] = useState(true)
   const [showHtml, setShowHtml] = useState(false)
+  const [showDomState, setShowDomState] = useState(false)
   const [autoRefresh, setAutoRefresh] = useState(true)
+  const [domStateNodeCount, setDomStateNodeCount] = useState(0)
   const iframeContainerRef = useRef<HTMLDivElement>(null)
 
-  const { state: loopState, actions: loopActions, iframeEl, screenshotRef } = useAgentLoop({
+  const { state: loopState, actions: loopActions, iframeEl, currentDomState, screenshotRef } = useAgentLoop({
     onScreenshot: (dataUrl, iteration) => {
       setIterations((prev) => {
         const entry = {
@@ -103,6 +108,11 @@ export function AgentLoopPanel({
         })
       }
     },
+    onDomState: (domState, iteration) => {
+      const match = domState.match(/(\d+) nodes/)
+      setDomStateNodeCount(match ? parseInt(match[1], 10) : 0)
+      onDomStateReady?.(domState)
+    },
     autoRefreshDelay: autoRefresh ? 800 : Infinity,
   })
 
@@ -307,6 +317,26 @@ export function AgentLoopPanel({
         </div>
       )}
 
+      {/* DOM state capture (Tier 2) */}
+      {currentDomState && (
+        <div className="agent-loop-domstate-section">
+          <div className="agent-loop-domstate-header">
+            <button
+              className={`agent-loop-domstate-toggle ${showDomState ? 'active' : ''}`}
+              onClick={() => setShowDomState(!showDomState)}
+            >
+              {showDomState ? '▼' : '▶'} {t('agent.loopDomState') || 'DOM State'}
+              <span className="agent-loop-domstate-badge">{domStateNodeCount} nodes</span>
+            </button>
+          </div>
+          {showDomState && (
+            <pre className="agent-loop-domstate-source">
+              <code>{currentDomState}</code>
+            </pre>
+          )}
+        </div>
+      )}
+
       {/* Screenshot preview */}
       {activeEntry?.screenshotDataUrl && (
         <div className="agent-loop-screenshot-section">

src/hooks/useAgentLoop.ts

@@ -10,6 +10,7 @@
 import { useState, useCallback, useRef, useEffect } from 'react'
 import { getOrCreateIframe, getSandboxIframe, removeSandboxIframe } from '../lib/browserUseTools'
 import { extractHTMLCode } from '../lib/htmlExtract'
+import { serializeIframeDomState, type SerializeOptions } from '../lib/domState'
 
 export type AgentLoopStatus = 'idle' | 'running' | 'paused' | 'error'
 
@@ -36,6 +37,10 @@ export interface UseAgentLoopOptions {
   onScreenshot?: (dataUrl: string, iteration: number) => void
   /** Called when iteration completes — receives iteration count and HTML */
   onIterationComplete?: (html: string | null, iteration: number) => void
+  /** Called before each model turn — receives serialized DOM-state JSON for context injection */
+  onDomState?: (domState: string, iteration: number) => void
+  /** DOM-state serialization options (depth + node-count budget) */
+  domStateOptions?: SerializeOptions
   /** Debounce delay in ms for auto-refresh (default: 500) */
   autoRefreshDelay?: number
   /** Iframe target ID (default: 'agent-loop') */
@@ -134,11 +139,14 @@ export function useAgentLoop(options: UseAgentLoopOptions = {}): {
   state: AgentLoopState
   actions: AgentLoopActions
   iframeEl: HTMLIFrameElement | null
+  currentDomState: string | null
   screenshotRef: React.RefCallback<HTMLIFrameElement>
 } {
   const {
     onScreenshot,
     onIterationComplete,
+    onDomState,
+    domStateOptions,
     autoRefreshDelay = 500,
     iframeId = 'agent-loop',
   } = options
@@ -153,12 +161,15 @@ export function useAgentLoop(options: UseAgentLoopOptions = {}): {
     lastScreenshotAt: null,
   })
 
+  const [currentDomState, setCurrentDomState] = useState<string | null>(null)
+
   const statusRef = useRef<AgentLoopStatus>('idle')
   const htmlRef = useRef<string | null>(null)
   const iframeRef = useRef<HTMLIFrameElement | null>(null)
   const abortRef = useRef(false)
   const autoRefreshTimerRef = useRef<ReturnType<typeof setTimeout> | null>(null)
   const iterationRef = useRef(0)
+  const domStateOptionsRef = useRef(domStateOptions)
 
   // Create iframe on mount
   useEffect(() => {
@@ -180,6 +191,11 @@ export function useAgentLoop(options: UseAgentLoopOptions = {}): {
     }
   }, [])
 
+  // Update DOM-state options ref when they change
+  useEffect(() => {
+    domStateOptionsRef.current = domStateOptions
+  }, [domStateOptions])
+
   const start = useCallback(
     (goal: string) => {
       abortRef.current = false
@@ -247,14 +263,47 @@ export function useAgentLoop(options: UseAgentLoopOptions = {}): {
         }))
       }
 
+      // Capture DOM-state for context injection (Tier 2)
+      const iframe = iframeRef.current
+      if (iframe && iframe.contentDocument && onDomState) {
+        const domJson = serializeIframeDomState(iframe, domStateOptionsRef.current)
+        const lines: string[] = []
+        lines.push('// DOM State Context')
+        lines.push(`// ${domJson.nodeCount} nodes, depth ${domJson.budget.depthReached}/${domJson.budget.maxDepth}`)
+        if (domJson.budget.nodesTruncated) lines.push('// ⚠️ Truncated — node budget exceeded')
+        lines.push('')
+        for (const node of domJson.tree) {
+          const indent = '  '.repeat(node.depth)
+          const attrs: string[] = []
+          if (node.id) attrs.push(`id="${node.id}"`)
+          if (node.classes) attrs.push(`class="${node.classes}"`)
+          if (node.type) attrs.push(`type="${node.type}"`)
+          if (node.name) attrs.push(`name="${node.name}"`)
+          if (node.value !== undefined) attrs.push(`value="${node.value}"`)
+          if (node.checked !== undefined) attrs.push(`checked=${node.checked}`)
+          if (node.disabled !== undefined) attrs.push(`disabled=${node.disabled}`)
+          if (node.selected !== undefined) attrs.push(`selected=${node.selected}`)
+          if (node.focused) attrs.push('focused')
+          if (node.interactive) attrs.push('interactive')
+          if (node.ariaRole) attrs.push(`role="${node.ariaRole}"`)
+          if (node.ariaLabel) attrs.push(`aria-label="${node.ariaLabel}"`)
+          const attrStr = attrs.length > 0 ? ` [${attrs.join(', ')}]` : ''
+          const text = node.text ? ` "${node.text.slice(0, 80)}"` : ''
+          lines.push(`${indent}<${node.tag}${attrStr}>${text}</${node.tag}>`)
+        }
+        const domStateStr = lines.join('\n')
+        setCurrentDomState(domStateStr)
+        onDomState(domStateStr, iteration)
+      }
+
       onIterationComplete?.(htmlRef.current, iteration)
     } catch (err) {
       setState((prev) => ({
         ...prev,
         error: err instanceof Error ? err.message : 'Screenshot failed',
       }))
     }
-  }, [iframeId, onScreenshot, onIterationComplete])
+  }, [iframeId, onScreenshot, onIterationComplete, onDomState])
 
   const setHtml = useCallback(
     (html: string | null) => {
@@ -323,6 +372,7 @@ export function useAgentLoop(options: UseAgentLoopOptions = {}): {
     state,
     actions: { start, stop, refresh, setHtml, clear },
     iframeEl: iframeRef.current,
+    currentDomState,
     screenshotRef,
   }
 }

src/lib/changelog.ts

@@ -1414,6 +1414,21 @@ export const CHANGELOG: ChangelogEntry[] = [
       { type: 'changed', text: 'Updated version to v1.3.0' },
     ],
   },
+  {
+    version: '1.3.2',
+    date: '2026-05-11',
+    title: 'Browser-Use — DOM-State Capture (Tier 2)',
+    description:
+      'Serialize the active sandboxed iframe\'s DOM to a compact accessibility-tree JSON and inject it as context before each model turn. Includes depth + node-count budget limits to stay token-safe, plus a DOM State panel in the AgentLoop UI for inspection.',
+    changes: [
+      { type: 'added', text: 'domState.ts — DOM-state serialization utility: serializeIframeDomState() with ARIA roles, labels, input states, interactive detection; serializeDomStateCompact() for LLM context injection; configurable maxDepth/maxNodes budget' },
+      { type: 'added', text: 'useAgentLoop onDomState callback — captures serialized DOM state before each model turn via the refresh cycle; passes compact JSON to parent for context injection' },
+      { type: 'added', text: 'AgentLoopPanel DOM State panel — collapsible DOM state viewer with node-count badge, syntax-highlighted tree output, light theme support' },
+      { type: 'added', text: 'i18n — agent.loopDomState en/zh translation' },
+      { type: 'added', text: 'CSS — dark terminal-style DOM State panel with node-count badge, light theme overrides' },
+      { type: 'changed', text: 'Updated version to v1.3.2' },
+    ],
+  },
 ]
 
 // Sort descending by version (latest first)