Web pipeline: retrieval rides with discovery; hallucinated fetch calls get steered

Packages/OsaurusCore/Services/Chat/AgentTaskState.swift

@@ -714,12 +714,15 @@ public final class AgentTaskState {
 
         // Reworded discovery loop: the model has URLs/snippets but keeps
         // searching instead of retrieving or processing a selected source.
-        // Name the real tool boundary and the dynamic-load path; if those
-        // capabilities are unavailable, require a truthful blocker instead of
-        // another cosmetic query rewrite.
+        // Name the real tool boundary directly. `search_and_extract` is
+        // composed into the schema whenever web search is enabled, so the old
+        // `capabilities_load tool/search_and_extract` detour here pointed at a
+        // loader round-trip for a tool already in the schema — and on the
+        // Default agent, at a loader gated to configure writes that would
+        // REJECT the load. Steer to the tool itself.
         if webDiscoveryRunCount >= Self.webDiscoveryRunThreshold {
             return
-                "You have called `web_search` \(Self.webDiscoveryRunThreshold)+ times in a row. `web_search` is discovery-only and returns URLs/snippets, not page bodies or downloadable data. Stop searching. Use a returned source with an available extraction, download, or file tool; if available, load `tool/search_and_extract` with `capabilities_load`, then process the retrieved data and call `render_chart` when that tool is available. If retrieval or chart rendering is unavailable, report that blocker clearly instead of rephrasing the search again."
+                "You have called `web_search` \(Self.webDiscoveryRunThreshold)+ times in a row. `web_search` is discovery-only and returns URLs/snippets, not page bodies or downloadable data. Stop searching. Pick the best returned URLs and call `search_and_extract` with them (`{\"urls\": [...]}`) to retrieve page content, then process the retrieved data and call `render_chart` when that tool is available. If retrieval or chart rendering is unavailable, report that blocker clearly instead of rephrasing the search again."
         }
 
         // Listing nudges are reactive: suppressed until the model is observed

Packages/OsaurusCore/Services/Chat/SystemPromptComposer.swift

@@ -2745,15 +2745,17 @@ public struct SystemPromptComposer: Sendable {
             )
         }
 
-        // A chart assembled from live web data needs both halves of the web
-        // pipeline in the same turn: discovery (`web_search`) and retrieval
-        // (`search_and_extract`). Keeping extraction dynamic for ordinary
-        // chat still preserves the lean baseline, while the explicit Charts
-        // + Web capability combination is a strong signal that retrieval is
-        // required. Expose the real tool contract up front so small local
-        // models do not get trapped rephrasing discovery queries while trying
-        // to guess the capabilities-load transition.
-        if !isManual, snapshot.renderChartEnabled, snapshot.webSearchEnabled {
+        // Web search enabled means BOTH halves of the web pipeline are in the
+        // schema from turn 1: discovery (`web_search`) and retrieval
+        // (`search_and_extract`). Retrieval used to stay dynamic outside the
+        // Charts+Web combination to keep the baseline lean, and that gap is
+        // exactly where models fell apart in practice: with only `web_search`
+        // visible they either rephrased the same discovery query in a loop or
+        // hallucinated a fetch tool (`web_fetch`) that has never existed —
+        // observed live as a Raptor research run producing zero page reads.
+        // The discovery→retrieval transition advertised inside `web_search`
+        // results must be callable the moment those results arrive.
+        if !isManual, snapshot.webSearchEnabled {
             add(
                 ToolRegistry.shared.specs(forTools: ["search_and_extract"]),
                 replacingExisting: true

Packages/OsaurusCore/Tests/Chat/SystemPromptComposerToolResolutionTests.swift

@@ -2126,4 +2126,31 @@ struct SystemPromptComposerToolResolutionTests {
             }
         }
     }
+
+    @Test("web search alone exposes retrieval — no charts required")
+    func webSearchAloneExposesSearchAndExtract() {
+        // The discovery→retrieval transition advertised inside `web_search`
+        // results must be callable on turn 1. Retrieval used to be composed
+        // in only for Charts+Web or the orchestrator; ordinary web-enabled
+        // chat got discovery alone, and models either rephrased the same
+        // query in a loop or hallucinated a fetch tool (`web_fetch`).
+        let tools = SystemPromptComposer.resolveTools(
+            snapshot: makeSnapshot(webSearchEnabled: true),
+            executionMode: .none,
+            query: "What changed in the latest mlx release?"
+        )
+        let names = Set(tools.map(\.function.name))
+        #expect(names.contains("web_search"))
+        #expect(names.contains("search_and_extract"))
+
+        // And the web gate still strips BOTH halves when off.
+        let withoutWeb = SystemPromptComposer.resolveTools(
+            snapshot: makeSnapshot(webSearchEnabled: false),
+            executionMode: .none,
+            query: "What changed in the latest mlx release?"
+        )
+        let namesWithoutWeb = Set(withoutWeb.map(\.function.name))
+        #expect(!namesWithoutWeb.contains("web_search"))
+        #expect(!namesWithoutWeb.contains("search_and_extract"))
+    }
 }

Packages/OsaurusCore/Tests/Search/WebSearchToolTests.swift

@@ -48,7 +48,13 @@ struct WebSearchToolTests {
         let description = WebSearchTool().description
         #expect(description.contains("does not fetch page bodies"))
         #expect(description.contains("search_and_extract"))
-        #expect(description.contains("tool/search_and_extract"))
+        // The old contract routed through `capabilities_load tool/…` when
+        // retrieval wasn't loaded. `search_and_extract` is now composed into
+        // the schema whenever web search is enabled, so the description must
+        // name the direct transition — a load detour here would send models
+        // to a loader for a tool already in their schema.
+        #expect(!description.contains("capabilities_load"))
+        #expect(description.contains("IS the fetch tool"))
     }
 
     @Test func searchAndExtractContractProvidesPageContent() {
@@ -316,7 +322,11 @@ struct WebSearchToolTests {
         )
         let names = Set(tools.map { $0.function.name })
         #expect(names.contains("web_search"))
-        #expect(!names.contains("search_and_extract"))
+        // Retrieval rides along with discovery now: web-enabled agents get
+        // `search_and_extract` on turn 1 instead of a capabilities_load
+        // round-trip. The old expectation pinned the lean baseline that left
+        // models looping `web_search` rephrases or hallucinating `web_fetch`.
+        #expect(names.contains("search_and_extract"))
     }
 
     @Test func chartAndWebExposeRetrievalOnTheFirstTurn() {

Packages/OsaurusCore/Tests/Tool/ToolScopeGateRecoveryTests.swift

@@ -278,6 +278,49 @@ struct ToolScopeGateRecoveryTests {
         #expect(!ToolEnvelope.isError(result))
     }
 
+    // MARK: - Hallucinated fetch-tool steering
+
+    @Test
+    func hallucinatedFetchName_isSteeredToSearchAndExtract_whenExposed() async throws {
+        // `web_fetch` has never existed in osaurus, but models trained on
+        // other harnesses call it as if it were universal — observed live as
+        // a research run that burned its whole turn on it and read zero
+        // pages. When the request exposes `search_and_extract`, the dead end
+        // must instead point at the fetch tool already in the schema.
+        let scope = ToolExecutionScope(exposed: [])
+        scope.activate(["web_search", "search_and_extract"])
+        let result = try await ChatExecutionContext.$toolExecutionScope.withValue(scope) {
+            try await ToolRegistry.shared.execute(
+                name: "web_fetch",
+                argumentsJSON: "{\"url\": \"https://example.com\"}"
+            )
+        }
+        let parsed = try envelope(result)
+        #expect(parsed?["ok"] as? Bool == false)
+        #expect(parsed?["kind"] as? String == "tool_not_found")
+        #expect(parsed?["retryable"] as? Bool == true)
+        let message = parsed?["message"] as? String ?? ""
+        #expect(message.contains("search_and_extract"))
+        #expect(message.contains("urls"))
+    }
+
+    @Test
+    func hallucinatedFetchName_staysOpaque_whenRetrievalIsNotExposed() async throws {
+        // Web disabled for this agent: the steering must not leak the name of
+        // a tool the request cannot call. The plain unregistered-name refusal
+        // applies.
+        let scope = ToolExecutionScope(exposed: [])
+        scope.activate(["get_current_time"])
+        let result = try await ChatExecutionContext.$toolExecutionScope.withValue(scope) {
+            try await ToolRegistry.shared.execute(name: "web_fetch", argumentsJSON: "{}")
+        }
+        let parsed = try envelope(result)
+        #expect(parsed?["ok"] as? Bool == false)
+        #expect(parsed?["kind"] as? String == "tool_not_found")
+        let message = parsed?["message"] as? String ?? ""
+        #expect(!message.contains("search_and_extract"))
+    }
+
     // MARK: - Workspace-tool dead end names the real next step
 
     @Test

Packages/OsaurusCore/Tools/ToolRegistry.swift

@@ -899,6 +899,39 @@ public final class ToolRegistry: ObservableObject {
         // `tool/` handling in CapabilityTools.resolve.
         let name = resolvedRegisteredName(for: rawName)
 
+        // Hallucinated fetch tools get steered, not dead-ended. Models trained
+        // on other harnesses call `web_fetch` / `fetch_url` / `open_url` as if
+        // they were universal — observed live: a Raptor research run burned its
+        // entire turn on `web_fetch` (a tool that has never existed in osaurus)
+        // and produced zero page reads. The general no-"did you mean" rule
+        // exists because naming other tools teaches models to invent siblings;
+        // it does not apply here, because the steering target is only named
+        // when THIS request already exposes it — the model is being pointed at
+        // a tool sitting in its own schema, not at a rumor. Guarded to names
+        // with no real registration so a plugin/MCP tool that legitimately
+        // registers one of these names always wins — and a user-installed
+        // plugin GROUP that happens to share an alias name (`plugin/fetch`)
+        // keeps its own load rescue instead of being steered away from the
+        // capability the user deliberately installed.
+        if toolsByName[name] == nil, Self.hallucinatedFetchToolNames.contains(name.lowercased()),
+            toolsByName["search_and_extract"] != nil,
+            ChatExecutionContext.toolExecutionScope?.permits("search_and_extract") == true,
+            groupIdCallRescueEnvelope(for: name) == nil
+        {
+            ToolRegistryLogger.registry.notice(
+                "steering hallucinated fetch tool '\(name, privacy: .public)' to search_and_extract"
+            )
+            return ToolErrorEnvelope(
+                kind: .toolNotFound,
+                reason:
+                    "There is no '\(name)' tool. To fetch a web page, call search_and_extract "
+                    + "with {\"urls\": [\"<url>\"]} — it fetches the pages and returns their "
+                    + "readable content. To find sources first, call web_search.",
+                toolName: name,
+                retryable: true
+            ).toJSONString()
+        }
+
         // A tool this request never exposed must not run, however convincingly the model asks
         // for it.
         //
@@ -2454,6 +2487,23 @@ public final class ToolRegistry: ObservableObject {
         "capabilities", "capabilities_discover", "capabilities_load",
     ]
 
+    /// Fetch-tool names models import from other harnesses (Claude Code's
+    /// WebFetch, LangChain requests_get, the reference MCP fetch server, …).
+    /// None have ever existed in osaurus; calls to them are steered to
+    /// `search_and_extract` when this request exposes it (see `execute`).
+    /// Lowercase; matched case-insensitively. Only consulted for names with
+    /// NO real registration, so a plugin/MCP tool legitimately claiming one
+    /// of these always wins. Deliberately excludes browser-intent names
+    /// (`browse`, `open_url`) — steering "log in and click" intent to a
+    /// read-only extractor points AWAY from a `browser_use` sitting in the
+    /// schema — and shell-intent names (`curl`), whose POST/API shapes
+    /// extraction cannot serve.
+    static let hallucinatedFetchToolNames: Set<String> = [
+        "web_fetch", "webfetch", "fetch", "fetch_url", "fetch_page",
+        "fetch_webpage", "http_get", "get_url", "get_webpage", "read_url",
+        "read_webpage", "visit_page", "load_url", "url_fetch",
+    ]
+
     /// Built-in tools that are authoritatively gated per-agent and must never
     /// surface through `capabilities_discover`. Unlike the lean-by-default
     /// built-in gates (render_chart, speak, search_memory, the scheduler trio,

Packages/OsaurusCore/Tools/WebSearchTools.swift

@@ -174,9 +174,9 @@ final class WebSearchTool: OsaurusTool, @unchecked Sendable {
         "Discover relevant web sources. Just pass `query`; results come from the user's "
         + "configured search providers with automatic fallback. Returns ranked titles, URLs, "
         + "and snippets only — it does not fetch page bodies or downloadable data. Once you "
-        + "select a source, retrieve its content with `search_and_extract`; if that tool is not "
-        + "loaded and `capabilities_load` is available, load `tool/search_and_extract`. Do not "
-        + "keep rephrasing `web_search` when you need source content."
+        + "select a source, retrieve its content with `search_and_extract` (pass the chosen "
+        + "URLs). Do not keep rephrasing `web_search` when you need source content, and do "
+        + "not invent fetch tools — `search_and_extract` IS the fetch tool."
 
     // Immutable by design: `web_search` is an always-loaded baseline tool, so
     // its schema is part of every composed prompt's static prefix. Deriving