feat: improve scrollable element detection, enhance abort error handl…

packages/core/src/PageAgentCore.ts

@@ -20,7 +20,7 @@ import type {
 	MacroToolInput,
 	MacroToolResult,
 } from './types'
-import { assert, fetchLlmsTxt, normalizeResponse, uid, waitFor } from './utils'
+import { assert, fetchLlmsTxt, getEventDetail, isAbortError, normalizeResponse, uid, waitFor } from './utils'
 
 export { tool, type PageAgentTool } from './tools'
 export type * from './types'
@@ -104,27 +104,30 @@ export class PageAgentCore extends EventTarget {
 
 		// Listen to LLM retry events
 		this.#llm.addEventListener('retry', (e) => {
-			const { attempt, maxAttempts } = (e as CustomEvent).detail
-			this.#emitActivity({ type: 'retrying', attempt, maxAttempts })
+			const detail = getEventDetail<{ attempt: number; maxAttempts: number }>(e)
+			if (!detail) return
+			this.#emitActivity({ type: 'retrying', attempt: detail.attempt, maxAttempts: detail.maxAttempts })
 			// Also push to history for panel rendering
 			this.history.push({
 				type: 'retry',
-				message: `LLM retry attempt ${attempt} of ${maxAttempts}`,
-				attempt,
-				maxAttempts,
+				message: `LLM retry attempt ${detail.attempt} of ${detail.maxAttempts}`,
+				attempt: detail.attempt,
+				maxAttempts: detail.maxAttempts,
 			})
 			this.#emitHistoryChange()
 		})
 		this.#llm.addEventListener('error', (e) => {
-			const error = (e as CustomEvent).detail.error as Error | InvokeError
-			if ((error as any)?.rawError?.name === 'AbortError') return
-			const message = String(error)
+			const detail = getEventDetail<{ error: unknown }>(e)
+			if (!detail) return
+			const error = detail.error
+			if (isAbortError(error)) return
+			const message = error instanceof Error ? error.message : String(error)
 			this.#emitActivity({ type: 'error', message })
 			// Also push to history for panel rendering
 			this.history.push({
 				type: 'error',
 				message,
-				rawResponse: (error as InvokeError).rawResponse,
+				rawResponse: error instanceof Error ? (error as InvokeError).rawResponse : undefined,
 			})
 			this.#emitHistoryChange()
 		})
@@ -187,9 +190,12 @@ export class PageAgentCore extends EventTarget {
 
 	/** Stop the current task. Agent remains reusable. */
 	stop() {
+		if (this.#abortController.signal.aborted) return // Already stopped
 		this.pageController.cleanUpHighlights()
 		this.pageController.hideMask()
 		this.#abortController.abort()
+		this.#setStatus('idle')
+		this.#abortController = new AbortController()
 	}
 
 	async execute(task: string): Promise<ExecutionResult> {
@@ -311,10 +317,10 @@ export class PageAgentCore extends EventTarget {
 				}
 			} catch (error: unknown) {
 				console.groupEnd() // to prevent nested groups
-				const isAbortError = (error as any)?.rawError?.name === 'AbortError'
+				const isAborted = isAbortError(error)
 
 				console.error('Task failed', error)
-				const errorMessage = isAbortError ? 'Task stopped' : String(error)
+				const errorMessage = isAborted ? 'Task stopped' : String(error)
 				this.#emitActivity({ type: 'error', message: errorMessage })
 				this.history.push({ type: 'error', message: errorMessage, rawResponse: error })
 				this.#emitHistoryChange()
@@ -343,7 +349,7 @@ export class PageAgentCore extends EventTarget {
 				return result
 			}
 
-			await waitFor(0.4) // @TODO: configurable
+			await waitFor(this.config.stepDelay ?? 0.4)
 		}
 	}
 

packages/core/src/types.ts

@@ -152,6 +152,12 @@ export interface AgentConfig extends LLMConfig {
 	 * @experimental Use with caution - incorrect prompts may break agent behavior.
 	 */
 	customSystemPrompt?: string
+
+	/**
+	 * Delay between steps in seconds.
+	 * @default 0.4
+	 */
+	stepDelay?: number
 }
 
 /**

packages/core/src/utils/index.ts

@@ -61,7 +61,12 @@ const llmsTxtCache = new Map<string, string | null>()
 
 /** Fetch /llms.txt for a URL's origin. Cached per origin, `null` = tried and not found. */
 export async function fetchLlmsTxt(url: string): Promise<string | null> {
-	const origin = new URL(url).origin
+	let origin: string
+	try {
+		origin = new URL(url).origin
+	} catch {
+		return null // Invalid URL
+	}
 	if (llmsTxtCache.has(origin)) return llmsTxtCache.get(origin)!
 
 	const endpoint = `${origin}/llms.txt`
@@ -101,3 +106,30 @@ export function assert(condition: unknown, message?: string, silent?: boolean):
 		throw new Error(errorMessage)
 	}
 }
+
+/**
+ * Check if an error is an AbortError (from AbortController)
+ * Handles various forms: Error with name 'AbortError', or rawError property
+ */
+export function isAbortError(error: unknown): boolean {
+	if (error instanceof Error && error.name === 'AbortError') return true
+	if (
+		typeof error === 'object' &&
+		error !== null &&
+		'rawError' in error &&
+		(error as { rawError?: Error }).rawError?.name === 'AbortError'
+	)
+		return true
+	return false
+}
+
+/**
+ * Safely extract detail from CustomEvent
+ * @returns The detail object or null if not a CustomEvent
+ */
+export function getEventDetail<T>(event: Event): T | null {
+	if (event instanceof CustomEvent) {
+		return event.detail as T
+	}
+	return null
+}

packages/llms/src/OpenAIClient.ts

@@ -56,15 +56,15 @@
 				signal: abortSignal,
 			})
 		} catch (error: unknown) {
-			const isAbortError = (error as any)?.name === 'AbortError'
+			const isAborted = error instanceof Error && error.name === 'AbortError'
 			const errorMessage = isAbortError ? 'Network request aborted' : 'Network request failed'
 			if (!isAbortError) console.error(error)
 			throw new InvokeError(InvokeErrorType.NETWORK_ERROR, errorMessage, error)
 		}
 
 		// 3. Handle HTTP errors
 		if (!response.ok) {
-			const errorData = await response.json().catch()
+			const errorData = await response.json().catch(() => ({ error: { message: response.statusText } }))
 			const errorMessage =
 				(errorData as { error?: { message?: string } }).error?.message || response.statusText
 
@@ -135,7 +135,7 @@
 
 		// Apply normalizeResponse if provided (for fixing format issues automatically)
 		const normalizedData = options?.normalizeResponse ? options.normalizeResponse(data) : data
-		const normalizedChoice = (normalizedData as any).choices?.[0]
+		const normalizedChoice = (normalizedData as { choices?: Array<{ message?: { tool_calls?: Array<{ function?: { name?: string; arguments?: string } }> } }> })?.choices?.[0]
 
 		// Get tool name from response
 		const toolCallName = normalizedChoice?.message?.tool_calls?.[0]?.function?.name
@@ -201,7 +201,7 @@
 		} catch (e) {
 			throw new InvokeError(
 				InvokeErrorType.TOOL_EXECUTION_ERROR,
-				`Tool execution failed: ${(e as Error).message}`,
+				`Tool execution failed: ${e instanceof Error ? e.message : String(e)}`,
 				e,
 				data
 			)

packages/llms/src/errors.ts

@@ -40,8 +40,8 @@ export class InvokeError extends Error {
 	}
 
 	private isRetryable(type: InvokeErrorType, rawError?: unknown): boolean {
-		const isAbortError = (rawError as any)?.name === 'AbortError'
-		if (isAbortError) return false
+		const isAborted = rawError instanceof Error && rawError.name === 'AbortError'
+		if (isAborted) return false
 
 		const retryableTypes: InvokeErrorType[] = [
 			InvokeErrorType.NETWORK_ERROR,

packages/llms/src/index.ts

@@ -93,15 +93,21 @@ async function withRetry<T>(
 			return await fn()
 		} catch (error: unknown) {
 			// do not retry if aborted by user
-			if ((error as any)?.rawError?.name === 'AbortError') throw error
+			if (
+				error instanceof InvokeError &&
+				error.rawError instanceof Error &&
+				error.rawError.name === 'AbortError'
+			) {
+				throw error
+			}
 
 			console.error(error)
-			settings.onError(error as Error)
+			settings.onError(error instanceof Error ? error : new Error(String(error)))
 
 			// do not retry if error is not retryable (InvokeError)
 			if (error instanceof InvokeError && !error.retryable) throw error
 
-			lastError = error as Error
+			lastError = error instanceof Error ? error : new Error(String(error))
 			attempt++
 
 			await new Promise((resolve) => setTimeout(resolve, 100))

packages/page-controller/src/actions.ts

@@ -10,6 +10,52 @@ async function waitFor(seconds: number): Promise<void> {
 	await new Promise((resolve) => setTimeout(resolve, seconds * 1000))
 }
 
+/**
+ * Find a scrollable container without scanning the entire DOM.
+ * Tries common selectors first, then falls back to viewport elements.
+ */
+function findScrollableContainer(
+	canScroll: (el: HTMLElement | null) => boolean
+): HTMLElement | undefined {
+	// Try common scrollable container selectors first (fast path)
+	const commonSelectors = [
+		'[data-scrollable="true"]',
+		'.scrollable',
+		'.overflow-auto',
+		'.overflow-scroll',
+		'[class*="scroll"]',
+		'main',
+		'article',
+		'section',
+		'aside',
+	]
+
+	for (const selector of commonSelectors) {
+		const containers = document.querySelectorAll<HTMLElement>(selector)
+		for (const container of containers) {
+			if (canScroll(container)) return container
+		}
+	}
+
+	// Fallback: check elements near viewport center (more likely to be visible)
+	const viewportElements = document.elementsFromPoint(
+		window.innerWidth / 2,
+		window.innerHeight / 2
+	) as HTMLElement[]
+
+	for (const el of viewportElements) {
+		if (canScroll(el)) return el
+		// Check ancestors up to 3 levels
+		let parent = el.parentElement
+		for (let i = 0; i < 3 && parent; i++) {
+			if (canScroll(parent)) return parent
+			parent = parent.parentElement
+		}
+	}
+
+	return undefined
+}
+
 // ======= dom utils =======
 
 export async function movePointerToElement(element: HTMLElement) {
@@ -213,14 +259,18 @@ export async function selectOptionElement(selectElement: HTMLSelectElement, opti
 	await waitFor(0.1) // Wait to ensure change event processing completes
 }
 
+interface ScrollableElement extends HTMLElement {
+	scrollIntoViewIfNeeded?: (centerIfNeeded?: boolean) => void
+}
+
 export async function scrollIntoViewIfNeeded(element: HTMLElement) {
-	const el = element as any
-	if (el.scrollIntoViewIfNeeded) {
+	const el = element as ScrollableElement
+	if (typeof el.scrollIntoViewIfNeeded === 'function') {
 		el.scrollIntoViewIfNeeded()
 		// await waitFor(0.5) // Animation playback
 	} else {
 		// @todo visibility check
-		el.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'nearest' })
+		element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'nearest' })
 		// await waitFor(0.5) // Animation playback
 	}
 }
@@ -299,7 +349,7 @@ export async function scrollVertically(
 
 	el = canScroll(el)
 		? el
-		: Array.from(document.querySelectorAll<HTMLElement>('*')).find(canScroll) ||
+		: findScrollableContainer(canScroll) ||
 			(document.scrollingElement as HTMLElement) ||
 			(document.documentElement as HTMLElement)
 
@@ -427,7 +477,7 @@ export async function scrollHorizontally(
 
 	el = canScroll(el)
 		? el
-		: Array.from(document.querySelectorAll<HTMLElement>('*')).find(canScroll) ||
+		: findScrollableContainer(canScroll) ||
 			(document.scrollingElement as HTMLElement) ||
 			(document.documentElement as HTMLElement)
 

packages/page-controller/src/dom/dom_tree/index.js

@@ -141,7 +141,7 @@ export default (
 
 	const HIGHLIGHT_CONTAINER_ID = 'playwright-highlight-container'
 
-	// Add a WeakMap cache for XPath strings
+	// Add a WeakMap cache for XPath strings (WeakMap to avoid memory leaks with DOM nodes)
 	const xpathCache = new WeakMap()
 
 	// // Initialize once and reuse
@@ -1604,10 +1604,12 @@ export default (
 					// Call the dedicated highlighting function
 					nodeWasHighlighted = handleHighlighting(nodeData, node, parentIframe, isParentHighlighted)
 
-					/**
-					 * @edit direct dom ref
-					 */
-					nodeData.ref = node
+				/**
+				 * @edit direct dom ref
+				 * @note This creates a memory reference that persists until the next updateTree() call.
+				 * PageController.dispose() clears flatTree to release these references.
+				 */
+				nodeData.ref = node
 				}
 			}
 		}

packages/page-controller/src/mask/SimulatorMask.module.css

@@ -7,3 +7,7 @@
 
 	display: none;
 }
+
+.wrapper.visible {
+	display: block;
+}

packages/page-controller/src/mask/SimulatorMask.ts

@@ -151,7 +151,7 @@ export class SimulatorMask {
 		this.motion?.start()
 		this.motion?.fadeIn()
 
-		this.wrapper.style.display = 'block'
+		this.wrapper.classList.add(styles.visible)
 
 		// Initialize cursor position
 		this.#currentCursorX = window.innerWidth / 2
@@ -172,7 +172,7 @@ export class SimulatorMask {
 		this.#cursor.classList.remove(cursorStyles.clicking)
 
 		setTimeout(() => {
-			this.wrapper.style.display = 'none'
+			this.wrapper.classList.remove(styles.visible)
 		}, 800) // Match the animation duration
 	}