feat: add bounded web crawl tools

Author: waydelyle

README.md

@@ -409,6 +409,15 @@ Operational docs: https://swarmclaw.ai/docs/observability
 
 ## Releases
 
+### v1.9.22 Highlights
+
+Research tools release: agents now get direct `web_extract` and `web_crawl` tools alongside `web_search`, `web_fetch`, and the unified `web` tool.
+
+- **Source-grounded extraction.** `web_extract` returns a page title, canonical URL, and readable content for known source URLs.
+- **Bounded crawls.** `web_crawl` walks same-origin links by default with conservative page and depth caps, plus an explicit external-link opt-in.
+- **Better routing.** Tool aliases, capability policy, planning hints, continuation recovery, and the chat UI all recognize the granular research tools.
+- **Regression coverage.** New tests cover action inference, tool-call translation, direct tool registration, extraction cleanup, and same-origin crawl bounds.
+
 ### v1.9.21 Highlights
 
 Provider diagnostics release: connection checks now return a structured step timeline across setup, provider settings, and agent editing.

package-lock.json

@@ -1,6 +1,6 @@
 {
   "name": "@swarmclawai/swarmclaw",
-  "version": "1.9.21",
+  "version": "1.9.22",
   "description": "Build and run autonomous AI agents with OpenClaw, Hermes, multiple model providers, orchestration, delegation, memory, skills, schedules, and chat connectors.",
   "main": "electron-dist/main.js",
   "license": "MIT",
@@ -88,7 +88,7 @@
     "test:cli": "node --test src/cli/*.test.js bin/*.test.js scripts/electron-after-pack.test.mjs scripts/electron-signing-config.test.mjs scripts/ensure-sandbox-browser-image.test.mjs scripts/postinstall.test.mjs scripts/run-next-build.test.mjs scripts/run-next-typegen.test.mjs",
     "test:setup": "tsx --test src/app/api/setup/check-provider/route.test.ts src/lib/server/provider-model-discovery.test.ts src/components/auth/setup-wizard/utils.test.ts src/components/auth/setup-wizard/types.test.ts src/hooks/setup-done-detection.test.ts src/lib/setup-defaults.test.ts src/lib/server/storage-auth.test.ts src/lib/server/storage-auth-docker.test.ts",
     "test:openclaw": "tsx --test src/lib/openclaw/openclaw-agent-id.test.ts src/lib/openclaw/openclaw-endpoint.test.ts src/lib/server/agents/agent-runtime-config.test.ts src/lib/server/build-llm.test.ts src/lib/server/connectors/connector-routing.test.ts src/lib/server/connectors/openclaw.test.ts src/lib/server/connectors/swarmdock.test.ts src/lib/server/gateway/protocol.test.ts src/lib/server/gateways/gateway-topology.test.ts src/lib/server/llm-response-cache.test.ts src/lib/server/mcp-conformance.test.ts src/lib/server/openclaw/agent-resolver.test.ts src/lib/server/openclaw/deploy.test.ts src/lib/server/openclaw/skills-normalize.test.ts src/lib/server/session-tools/openclaw-nodes.test.ts src/lib/server/session-tools/swarmdock.test.ts src/lib/server/tasks/task-quality-gate.test.ts src/lib/server/tasks/task-validation.test.ts src/lib/server/tool-capability-policy.test.ts src/lib/providers/openai.test.ts src/lib/providers/openclaw-exports.test.ts src/app/api/gateways/topology-route.test.ts src/app/api/openclaw/dashboard-url/route.test.ts",
-    "test:runtime": "tsx --test src/lib/a2a/agent-card.test.ts src/lib/agent-planning-mode.test.ts src/lib/agent-config-history.test.ts src/lib/strip-internal-metadata.test.ts src/lib/provider-sets.test.ts src/lib/providers/opencode-cli.test.ts src/lib/providers/cli-provider-metadata.test.ts src/lib/providers/cli-utils.test.ts src/lib/providers/generic-cli.test.ts src/lib/server/agents/delegation-advisory.test.ts src/lib/server/cli-provider-readiness.test.ts src/lib/server/provider-health.test.ts src/lib/server/provider-diagnostics.test.ts src/lib/server/mcp-gateway-runtime.test.ts src/lib/server/mcp-connection-pool.test.ts src/lib/server/knowledge-sources.test.ts src/lib/server/extension-managed-resources.test.ts src/lib/server/eval/baseline.test.ts src/lib/server/eval/environment-plan.test.ts src/lib/server/chat-execution/chat-execution-grounding.test.ts src/lib/server/chat-execution/chat-turn-preparation.test.ts src/lib/server/chat-execution/iteration-timers.test.ts src/lib/server/chat-execution/post-stream-finalization.test.ts src/lib/server/chat-execution/prompt-sections.planning-mode.test.ts src/lib/server/chat-execution/reasoning-tag-scrubber.test.ts src/lib/server/chats/clear-undo-snapshots.test.ts src/lib/server/chats/session-context-pack.test.ts src/lib/server/connectors/email.test.ts src/lib/server/protocols/protocol-service.test.ts src/lib/server/runtime/run-ledger.test.ts src/lib/server/runtime/queue-retry-policy.test.ts src/lib/server/runs/run-brief.test.ts src/lib/server/runs/run-handoff.test.ts src/lib/server/operations/operation-pulse.test.ts src/lib/server/schedules/schedule-history.test.ts src/lib/server/schedules/schedule-preview.test.ts src/lib/quality/release-readiness.test.ts src/lib/quality/architecture-health.test.ts src/lib/server/artifacts/artifact-resolver.test.ts src/lib/server/observability/otel-config.test.ts src/lib/server/safe-parse-body.test.ts src/lib/server/missions/mission-templates.test.ts src/lib/server/sharing/share-link-repository.test.ts src/lib/server/sharing/share-resolver.test.ts src/lib/server/tasks/task-execution-workspace.test.ts src/lib/server/tasks/task-execution-policy.test.ts src/lib/server/tasks/task-handoff.test.ts src/lib/server/tasks/task-service.test.ts src/lib/server/session-tools/execute.test.ts src/lib/server/session-tools/manage-tasks.test.ts src/lib/app/view-constants.test.ts src/lib/quality/quality-summary.test.ts src/app/api/approvals/route.test.ts src/app/api/agents/agents-route.test.ts src/app/api/tasks/tasks-route.test.ts src/app/api/tasks/task-workspace-route.test.ts src/app/api/chats/chat-route.test.ts src/app/api/chats/clear-route.test.ts src/app/api/chats/compact-route.test.ts src/app/api/chats/context-pack-route.test.ts src/app/api/chats/context-status-route.test.ts src/app/api/config-versions/config-versions-route.test.ts src/app/api/runs/run-handoff-route.test.ts src/app/api/connectors/connector-doctor-route.test.ts src/app/api/extensions/managed-resources/route.test.ts src/app/api/healthz/route.test.ts src/app/api/logs/route.test.ts src/app/api/portability/export/route.test.ts src/app/api/portability/import/route.test.ts src/app/api/providers/[id]/route.test.ts src/app/api/schedules/preview/route.test.ts src/app/api/schedules/schedule-history-route.test.ts src/app/api/tts/route.test.ts",
+    "test:runtime": "tsx --test src/lib/a2a/agent-card.test.ts src/lib/agent-planning-mode.test.ts src/lib/agent-config-history.test.ts src/lib/strip-internal-metadata.test.ts src/lib/provider-sets.test.ts src/lib/providers/opencode-cli.test.ts src/lib/providers/cli-provider-metadata.test.ts src/lib/providers/cli-utils.test.ts src/lib/providers/generic-cli.test.ts src/lib/server/agents/delegation-advisory.test.ts src/lib/server/cli-provider-readiness.test.ts src/lib/server/provider-health.test.ts src/lib/server/provider-diagnostics.test.ts src/lib/server/mcp-gateway-runtime.test.ts src/lib/server/mcp-connection-pool.test.ts src/lib/server/knowledge-sources.test.ts src/lib/server/extension-managed-resources.test.ts src/lib/server/eval/baseline.test.ts src/lib/server/eval/environment-plan.test.ts src/lib/server/chat-execution/chat-execution-grounding.test.ts src/lib/server/chat-execution/chat-turn-preparation.test.ts src/lib/server/chat-execution/iteration-timers.test.ts src/lib/server/chat-execution/post-stream-finalization.test.ts src/lib/server/chat-execution/prompt-sections.planning-mode.test.ts src/lib/server/chat-execution/reasoning-tag-scrubber.test.ts src/lib/server/chats/clear-undo-snapshots.test.ts src/lib/server/chats/session-context-pack.test.ts src/lib/server/connectors/email.test.ts src/lib/server/protocols/protocol-service.test.ts src/lib/server/runtime/run-ledger.test.ts src/lib/server/runtime/queue-retry-policy.test.ts src/lib/server/runs/run-brief.test.ts src/lib/server/runs/run-handoff.test.ts src/lib/server/operations/operation-pulse.test.ts src/lib/server/schedules/schedule-history.test.ts src/lib/server/schedules/schedule-preview.test.ts src/lib/quality/release-readiness.test.ts src/lib/quality/architecture-health.test.ts src/lib/server/artifacts/artifact-resolver.test.ts src/lib/server/observability/otel-config.test.ts src/lib/server/safe-parse-body.test.ts src/lib/server/missions/mission-templates.test.ts src/lib/server/sharing/share-link-repository.test.ts src/lib/server/sharing/share-resolver.test.ts src/lib/server/tasks/task-execution-workspace.test.ts src/lib/server/tasks/task-execution-policy.test.ts src/lib/server/tasks/task-handoff.test.ts src/lib/server/tasks/task-service.test.ts src/lib/server/session-tools/execute.test.ts src/lib/server/session-tools/manage-tasks.test.ts src/lib/server/session-tools/web-crawl.test.ts src/lib/app/view-constants.test.ts src/lib/quality/quality-summary.test.ts src/app/api/approvals/route.test.ts src/app/api/agents/agents-route.test.ts src/app/api/tasks/tasks-route.test.ts src/app/api/tasks/task-workspace-route.test.ts src/app/api/chats/chat-route.test.ts src/app/api/chats/clear-route.test.ts src/app/api/chats/compact-route.test.ts src/app/api/chats/context-pack-route.test.ts src/app/api/chats/context-status-route.test.ts src/app/api/config-versions/config-versions-route.test.ts src/app/api/runs/run-handoff-route.test.ts src/app/api/connectors/connector-doctor-route.test.ts src/app/api/extensions/managed-resources/route.test.ts src/app/api/healthz/route.test.ts src/app/api/logs/route.test.ts src/app/api/portability/export/route.test.ts src/app/api/portability/import/route.test.ts src/app/api/providers/[id]/route.test.ts src/app/api/schedules/preview/route.test.ts src/app/api/schedules/schedule-history-route.test.ts src/app/api/tts/route.test.ts",
     "test:builder": "tsx --test src/features/protocols/builder/utils/nodes-to-template.test.ts src/features/protocols/builder/utils/template-to-nodes.test.ts src/features/protocols/builder/validators/dag-validator.test.ts",
     "test:e2e": "node --import tsx scripts/browser-e2e-smoke.ts",
     "test:mcp:conformance": "node --import tsx ./scripts/mcp-conformance-check.ts",

package.json

@@ -19,6 +19,9 @@ const NOTABLE_TOOLS: Record<string, { label: string; color: string; icon: 'brain
   delegate_to_agent: { label: 'Delegating task', color: '#6366F1', icon: 'delegate' },
   check_delegation_status: { label: 'Checking delegation', color: '#6366F1', icon: 'delegate' },
   web_search: { label: 'Searched the web', color: '#22C55E', icon: 'search' },
+  web_fetch: { label: 'Read a web page', color: '#22C55E', icon: 'search' },
+  web_extract: { label: 'Extracted a web page', color: '#22C55E', icon: 'search' },
+  web_crawl: { label: 'Crawled a site', color: '#22C55E', icon: 'search' },
   connector_message_tool: { label: 'Sent a message', color: '#F97316', icon: 'message' },
 }
 
@@ -35,6 +38,7 @@ function extractSnippet(toolName: string, toolInput: string): string | null {
     if (toolName === 'check_delegation_status' && parsed.agentName) return parsed.agentName
     if (toolName.startsWith('delegate_to_') && parsed.task) return parsed.task
     if (toolName === 'web_search' && parsed.query) return parsed.query
+    if ((toolName === 'web_fetch' || toolName === 'web_extract' || toolName === 'web_crawl') && parsed.url) return parsed.url
     if (toolName === 'connector_message_tool' && parsed.to) return parsed.to
   } catch { /* ignore parse errors */ }
   return null

src/components/chat/activity-moment.tsx

@@ -20,6 +20,8 @@ const TOOL_COLORS: Record<string, string> = {
   create_spreadsheet: '#10B981',
   web_search: '#3B82F6',
   web_fetch: '#3B82F6',
+  web_extract: '#3B82F6',
+  web_crawl: '#3B82F6',
   spawn_subagent: '#8B5CF6',
   delegate_to_agent: '#6366F1',
   check_delegation_status: '#6366F1',
@@ -77,6 +79,8 @@ export const TOOL_LABELS: Record<string, string> = {
   create_spreadsheet: 'Create Spreadsheet',
   web_search: 'Web Search',
   web_fetch: 'Web Fetch',
+  web_extract: 'Web Extract',
+  web_crawl: 'Web Crawl',
   claude_code: 'Claude Code',
   codex_cli: 'Codex CLI',
   opencode_cli: 'OpenCode CLI',
@@ -127,6 +131,8 @@ export const TOOL_DESCRIPTIONS: Record<string, string> = {
   create_spreadsheet: 'Create Excel or CSV files from structured data',
   web_search: 'Search the web for information',
   web_fetch: 'Fetch and read web page content',
+  web_extract: 'Extract readable content from a source URL',
+  web_crawl: 'Crawl a bounded set of pages from one site',
   claude_code: 'Enable delegation to Claude Code CLI',
   codex_cli: 'Enable delegation to OpenAI Codex CLI',
   opencode_cli: 'Enable delegation to OpenCode CLI',

src/components/chat/tool-call-bubble.tsx

@@ -26,7 +26,7 @@ test('routeTaskIntent keeps coding prompts prioritized over memory keywords', ()
 test('routeTaskIntent keeps hybrid research-plus-media prompts in research intent', () => {
   const decision = routeTaskIntent(
     'Can you tell me more if there is any news related to the US-Iran war, and can you send me some screenshots and give me a summary and maybe send me a voice note about it?',
-    ['web_search', 'web_fetch', 'browser', 'manage_connectors'],
+    ['web_search', 'web_fetch', 'web_crawl', 'browser', 'manage_connectors'],
     null,
     makeClassification({
       taskIntent: 'research',
@@ -39,7 +39,7 @@ test('routeTaskIntent keeps hybrid research-plus-media prompts in research inten
   )
 
   assert.equal(decision.intent, 'research')
-  assert.deepEqual(decision.preferredTools, ['web_search', 'web_fetch', 'browser', 'connector_message_tool'])
+  assert.deepEqual(decision.preferredTools, ['web_search', 'web_fetch', 'web_extract', 'web_crawl', 'browser', 'connector_message_tool'])
 })
 
 test('routeTaskIntent treats direct voice-note delivery as outreach', () => {
@@ -72,7 +72,7 @@ test('routeTaskIntent treats keep-watching update requests as research even with
   )
 
   assert.equal(decision.intent, 'research')
-  assert.deepEqual(decision.preferredTools, ['web_search', 'web_fetch'])
+  assert.deepEqual(decision.preferredTools, ['web_search', 'web_fetch', 'web_extract', 'web_crawl'])
 })
 
 test('routeTaskIntent uses structured classification when available', () => {
@@ -99,7 +99,7 @@ test('routeTaskIntent uses structured classification when available', () => {
   )
 
   assert.equal(decision.intent, 'browsing')
-  assert.deepEqual(decision.preferredTools, ['browser', 'web_fetch'])
+  assert.deepEqual(decision.preferredTools, ['browser', 'web_fetch', 'web_extract'])
 })
 
 function makeClassification(overrides: Partial<MessageClassification>): MessageClassification {

src/lib/server/capability-router.test.ts

@@ -144,6 +144,7 @@ export function routeTaskIntent(
       [
         TOOL_CAPABILITY.researchSearch,
         TOOL_CAPABILITY.researchFetch,
+        TOOL_CAPABILITY.researchCrawl,
         ...(wantsScreenshots ? [TOOL_CAPABILITY.browserCapture] : []),
         ...(wantsVoiceDelivery ? [TOOL_CAPABILITY.deliveryVoiceNote] : []),
         ...(wantsOutboundDelivery ? [TOOL_CAPABILITY.deliveryMedia, TOOL_CAPABILITY.deliveryMessage] : []),

src/lib/server/capability-router.ts

@@ -407,6 +407,33 @@ describe('translateRequestedToolInvocation advanced', () => {
     assert.equal(args.action, 'search')
     assert.equal(args.query, 'test query')
   })
+
+  it('maps web_extract to web with action=extract', () => {
+    const { toolName, args } = translateRequestedToolInvocation(
+      'web_extract',
+      { url: 'https://example.com/source' },
+      '',
+      ['web'],
+    )
+    assert.equal(toolName, 'web')
+    assert.equal(args.action, 'extract')
+    assert.equal(args.url, 'https://example.com/source')
+  })
+
+  it('maps web_crawl to web with bounded crawl arguments', () => {
+    const { toolName, args } = translateRequestedToolInvocation(
+      'web_crawl',
+      { url: 'https://example.com/', maxPages: 4, maxDepth: 1, includeExternal: false },
+      '',
+      ['web'],
+    )
+    assert.equal(toolName, 'web')
+    assert.equal(args.action, 'crawl')
+    assert.equal(args.url, 'https://example.com/')
+    assert.equal(args.maxPages, 4)
+    assert.equal(args.maxDepth, 1)
+    assert.equal(args.includeExternal, false)
+  })
 })
 
 // ---------------------------------------------------------------------------

src/lib/server/chat-execution/chat-execution-advanced.test.ts

@@ -127,6 +127,27 @@ export function translateRequestedToolInvocation(
       },
     }
   }
+  if (requestedName === 'web_extract') {
+    return {
+      toolName: 'web',
+      args: {
+        action: 'extract',
+        url: rawArgs.url,
+      },
+    }
+  }
+  if (requestedName === 'web_crawl') {
+    return {
+      toolName: 'web',
+      args: {
+        action: 'crawl',
+        url: rawArgs.url || rawArgs.query,
+        maxPages: rawArgs.maxPages ?? rawArgs.maxResults,
+        maxDepth: rawArgs.maxDepth,
+        includeExternal: rawArgs.includeExternal,
+      },
+    }
+  }
   if (requestedName === 'delegate_to_claude_code') {
     return { toolName: 'delegate', args: { ...rawArgs, backend: 'claude' } }
   }

src/lib/server/chat-execution/chat-execution-utils.ts

@@ -349,7 +349,7 @@ export async function processIterationEvents(opts: ProcessIterationEventsOpts):
       }
       if (
         boundedExternalExecutionTask
-        && ['http_request', 'web', 'web_search', 'web_fetch', 'browser'].includes(toolName)
+        && ['http_request', 'web', 'web_search', 'web_fetch', 'web_extract', 'web_crawl', 'browser'].includes(toolName)
         && countExternalExecutionResearchSteps(state.streamedToolEvents) >= 5
         && countDistinctExternalResearchHosts(state.streamedToolEvents) >= 3
       ) {