Migrate browser automation from Playwright to agent-browser

Replace raw Playwright with agent-browser (Vercel Labs wrapper) for more
stable element selection via accessibility tree snapshots instead of
CSS selectors.

Key changes:
- Add AgentBrowserClient wrapper around BrowserManager
- Replace CSS selectors with ElementMatchers (role, name, text)
- Add snapshot-helpers.ts with findElementByMatcher/clickByMatcher
- Implement fallback strategy: snapshot refs → CSS selectors → keyboard
- Fix Drive API calls to include required 'fields' parameter
- Add comprehensive tests for matchers and snapshot helpers

Files added:
- src/browser/agent-browser-client.ts
- src/browser/matchers.ts
- src/browser/snapshot-helpers.ts
- tests/matchers.test.ts
- tests/snapshot-helpers.test.ts

Files removed:
- src/browser/page-helpers.ts (replaced by snapshot-helpers.ts)
- src/browser/selectors.ts (replaced by matchers.ts)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: gsiener

CLAUDE.md

@@ -8,22 +8,27 @@
 
 ### Why Browser Automation?
 
-Google's Docs API cannot create suggested edits—only read them. This is a known limitation ([issue #287903901](https://issuetracker.google.com/issues/287903901)). We use Playwright browser automation to write suggestions while using the API for reliable reading.
+Google's Docs API cannot create suggested edits—only read them. This is a known limitation ([issue #287903901](https://issuetracker.google.com/issues/287903901)). We use agent-browser (a Vercel Labs CLI wrapping Playwright) for browser automation to write suggestions while using the API for reliable reading.
 
 ### Hybrid Read/Write Strategy
 
 - **Read path**: Google Docs API via service account (fast, reliable)
-- **Write path**: Playwright browser automation (fragile but necessary)
+- **Write path**: agent-browser automation (more stable than raw Playwright)
 
 ### Text Anchoring Over Indexes
 
 Claude's suggestions use text content matching (`findText`) rather than document indexes. Indexes shift as edits are made; text anchoring is more robust for sequential operations.
 
+### Accessibility Tree-Based Element Selection
+
+Instead of CSS selectors, we use agent-browser's accessibility tree snapshots with refs (`@e1`, `@e2`) for more stable element selection. Elements are matched by role, name, and text properties. Fallback to CSS selectors when snapshots don't match.
+
 ## Development Practices
 
 - **TDD**: Write tests before implementation
-- **Isolated Selectors**: All DOM selectors in `src/browser/selectors.ts` for easy maintenance when Google's UI changes
-- **Keyboard Shortcuts**: Prefer keyboard shortcuts over DOM selectors (more stable)
+- **Isolated Matchers**: Element matchers in `src/browser/matchers.ts` for easy maintenance when Google's UI changes
+- **Keyboard Shortcuts**: Prefer keyboard shortcuts over UI selectors (most stable)
+- **Snapshot-First**: Try accessibility tree refs first, fall back to CSS selectors
 - **Retry Logic**: All browser operations use retry with exponential backoff
 
 ## Testing
@@ -38,10 +43,11 @@ npm run lint          # TypeScript type check
 
 ### Adding a New Browser Operation
 
-1. Add selector to `src/browser/selectors.ts`
-2. Write test in `tests/`
-3. Implement in `src/browser/docs-writer.ts`
-4. Wrap with `withRetry()` for resilience
+1. Add matcher to `src/browser/matchers.ts` (role, name, text patterns)
+2. Add CSS selector fallbacks if needed
+3. Write test in `tests/`
+4. Implement using `clickByMatcher()` / `fillByMatcher()` from `snapshot-helpers.ts`
+5. Wrap with `withRetry()` for resilience
 
 ### Modifying Claude's Response Format
 

package-lock.json

@@ -18,10 +18,11 @@
   },
   "dependencies": {
     "@anthropic-ai/sdk": "^0.30.0",
+    "agent-browser": "^0.5.0",
     "commander": "^12.0.0",
     "dotenv": "^16.0.0",
     "googleapis": "^130.0.0",
-    "playwright": "^1.40.0",
+    "playwright-core": "^1.57.0",
     "zod": "^3.22.0"
   },
   "devDependencies": {

package.json

@@ -0,0 +1,158 @@
+/**
+ * Agent Browser Client - Wrapper around BrowserManager for track-changes usage.
+ *
+ * This module provides a clean interface to agent-browser's BrowserManager,
+ * exposing only the functionality we need while providing better error handling
+ * and logging.
+ */
+
+import { BrowserManager } from "agent-browser/dist/browser.js";
+import type { Page, Locator } from "playwright-core";
+import type { RefMap, EnhancedSnapshot } from "agent-browser/dist/snapshot.js";
+import { logger } from "../utils/logger.js";
+
+export interface AgentBrowserOptions {
+  headless?: boolean;
+  viewport?: { width: number; height: number };
+  storageStatePath?: string;
+}
+
+/**
+ * Client wrapper for agent-browser's BrowserManager.
+ * Provides a cleaner interface and maintains compatibility with existing code patterns.
+ */
+export class AgentBrowserClient {
+  private manager: BrowserManager;
+  private launched = false;
+
+  constructor() {
+    this.manager = new BrowserManager();
+  }
+
+  /**
+   * Launch the browser with specified options.
+   */
+  async launch(options: AgentBrowserOptions = {}): Promise<void> {
+    if (this.launched) {
+      logger.debug("Browser already launched");
+      return;
+    }
+
+    logger.info("Launching browser via agent-browser", {
+      headless: options.headless ?? true,
+      viewport: options.viewport,
+    });
+
+    await this.manager.launch({
+      id: "launch",
+      action: "launch",
+      headless: options.headless ?? true,
+      viewport: options.viewport ?? { width: 1280, height: 800 },
+    });
+
+    this.launched = true;
+    logger.info("Browser launched successfully");
+  }
+
+  /**
+   * Get the current Playwright Page object.
+   * This allows using familiar Playwright APIs for interactions.
+   */
+  getPage(): Page {
+    if (!this.launched) {
+      throw new Error("Browser not launched. Call launch() first.");
+    }
+    return this.manager.getPage();
+  }
+
+  /**
+   * Get an accessibility tree snapshot with element refs.
+   * @param options.interactive - Only include interactive elements
+   * @param options.compact - Remove structural elements without content
+   */
+  async getSnapshot(options?: {
+    interactive?: boolean;
+    compact?: boolean;
+    maxDepth?: number;
+    selector?: string;
+  }): Promise<EnhancedSnapshot> {
+    if (!this.launched) {
+      throw new Error("Browser not launched. Call launch() first.");
+    }
+    return this.manager.getSnapshot(options);
+  }
+
+  /**
+   * Get the cached ref map from the last snapshot.
+   */
+  getRefMap(): RefMap {
+    return this.manager.getRefMap();
+  }
+
+  /**
+   * Get a Playwright Locator from a ref (e.g., "e1", "@e1", "ref=e1").
+   * Returns null if ref doesn't exist or is invalid.
+   */
+  getLocatorFromRef(ref: string): Locator | null {
+    if (!this.launched) {
+      throw new Error("Browser not launched. Call launch() first.");
+    }
+    return this.manager.getLocatorFromRef(ref);
+  }
+
+  /**
+   * Get a Playwright Locator - supports both refs and regular selectors.
+   */
+  getLocator(selectorOrRef: string): Locator {
+    if (!this.launched) {
+      throw new Error("Browser not launched. Call launch() first.");
+    }
+    return this.manager.getLocator(selectorOrRef);
+  }
+
+  /**
+   * Check if a selector string looks like a ref (e.g., "@e1").
+   */
+  isRef(selector: string): boolean {
+    return this.manager.isRef(selector);
+  }
+
+  /**
+   * Save the current storage state (cookies, localStorage) to a file.
+   */
+  async saveStorageState(path: string): Promise<void> {
+    if (!this.launched) {
+      throw new Error("Browser not launched. Call launch() first.");
+    }
+    await this.manager.saveStorageState(path);
+    logger.info("Storage state saved", { path });
+  }
+
+  /**
+   * Set the viewport size.
+   */
+  async setViewport(width: number, height: number): Promise<void> {
+    if (!this.launched) {
+      throw new Error("Browser not launched. Call launch() first.");
+    }
+    await this.manager.setViewport(width, height);
+  }
+
+  /**
+   * Check if the browser is currently launched.
+   */
+  isLaunched(): boolean {
+    return this.launched && this.manager.isLaunched();
+  }
+
+  /**
+   * Close the browser and clean up resources.
+   */
+  async close(): Promise<void> {
+    if (this.launched) {
+      await this.manager.close();
+      this.launched = false;
+      logger.info("Browser closed");
+    }
+  }
+}