diff --git a/Packages/OsaurusCore/Services/Chat/AgentTaskState.swift b/Packages/OsaurusCore/Services/Chat/AgentTaskState.swift index b141e9a36..9e0a7ab55 100644 --- a/Packages/OsaurusCore/Services/Chat/AgentTaskState.swift +++ b/Packages/OsaurusCore/Services/Chat/AgentTaskState.swift @@ -714,12 +714,15 @@ public final class AgentTaskState { // Reworded discovery loop: the model has URLs/snippets but keeps // searching instead of retrieving or processing a selected source. - // Name the real tool boundary and the dynamic-load path; if those - // capabilities are unavailable, require a truthful blocker instead of - // another cosmetic query rewrite. + // Name the real tool boundary directly. `search_and_extract` is + // composed into the schema whenever web search is enabled, so the old + // `capabilities_load tool/search_and_extract` detour here pointed at a + // loader round-trip for a tool already in the schema — and on the + // Default agent, at a loader gated to configure writes that would + // REJECT the load. Steer to the tool itself. if webDiscoveryRunCount >= Self.webDiscoveryRunThreshold { return - "You have called `web_search` \(Self.webDiscoveryRunThreshold)+ times in a row. `web_search` is discovery-only and returns URLs/snippets, not page bodies or downloadable data. Stop searching. Use a returned source with an available extraction, download, or file tool; if available, load `tool/search_and_extract` with `capabilities_load`, then process the retrieved data and call `render_chart` when that tool is available. If retrieval or chart rendering is unavailable, report that blocker clearly instead of rephrasing the search again." + "You have called `web_search` \(Self.webDiscoveryRunThreshold)+ times in a row. `web_search` is discovery-only and returns URLs/snippets, not page bodies or downloadable data. Stop searching. Pick the best returned URLs and call `search_and_extract` with them (`{\"urls\": [...]}`) to retrieve page content, then process the retrieved data and call `render_chart` when that tool is available. If retrieval or chart rendering is unavailable, report that blocker clearly instead of rephrasing the search again." } // Listing nudges are reactive: suppressed until the model is observed diff --git a/Packages/OsaurusCore/Services/Chat/SystemPromptComposer.swift b/Packages/OsaurusCore/Services/Chat/SystemPromptComposer.swift index f0debcf51..8bc000e00 100644 --- a/Packages/OsaurusCore/Services/Chat/SystemPromptComposer.swift +++ b/Packages/OsaurusCore/Services/Chat/SystemPromptComposer.swift @@ -2745,15 +2745,17 @@ public struct SystemPromptComposer: Sendable { ) } - // A chart assembled from live web data needs both halves of the web - // pipeline in the same turn: discovery (`web_search`) and retrieval - // (`search_and_extract`). Keeping extraction dynamic for ordinary - // chat still preserves the lean baseline, while the explicit Charts - // + Web capability combination is a strong signal that retrieval is - // required. Expose the real tool contract up front so small local - // models do not get trapped rephrasing discovery queries while trying - // to guess the capabilities-load transition. - if !isManual, snapshot.renderChartEnabled, snapshot.webSearchEnabled { + // Web search enabled means BOTH halves of the web pipeline are in the + // schema from turn 1: discovery (`web_search`) and retrieval + // (`search_and_extract`). Retrieval used to stay dynamic outside the + // Charts+Web combination to keep the baseline lean, and that gap is + // exactly where models fell apart in practice: with only `web_search` + // visible they either rephrased the same discovery query in a loop or + // hallucinated a fetch tool (`web_fetch`) that has never existed — + // observed live as a Raptor research run producing zero page reads. + // The discovery→retrieval transition advertised inside `web_search` + // results must be callable the moment those results arrive. + if !isManual, snapshot.webSearchEnabled { add( ToolRegistry.shared.specs(forTools: ["search_and_extract"]), replacingExisting: true diff --git a/Packages/OsaurusCore/Tests/Chat/SystemPromptComposerToolResolutionTests.swift b/Packages/OsaurusCore/Tests/Chat/SystemPromptComposerToolResolutionTests.swift index 205e813db..05c2461e0 100644 --- a/Packages/OsaurusCore/Tests/Chat/SystemPromptComposerToolResolutionTests.swift +++ b/Packages/OsaurusCore/Tests/Chat/SystemPromptComposerToolResolutionTests.swift @@ -2126,4 +2126,31 @@ struct SystemPromptComposerToolResolutionTests { } } } + + @Test("web search alone exposes retrieval — no charts required") + func webSearchAloneExposesSearchAndExtract() { + // The discovery→retrieval transition advertised inside `web_search` + // results must be callable on turn 1. Retrieval used to be composed + // in only for Charts+Web or the orchestrator; ordinary web-enabled + // chat got discovery alone, and models either rephrased the same + // query in a loop or hallucinated a fetch tool (`web_fetch`). + let tools = SystemPromptComposer.resolveTools( + snapshot: makeSnapshot(webSearchEnabled: true), + executionMode: .none, + query: "What changed in the latest mlx release?" + ) + let names = Set(tools.map(\.function.name)) + #expect(names.contains("web_search")) + #expect(names.contains("search_and_extract")) + + // And the web gate still strips BOTH halves when off. + let withoutWeb = SystemPromptComposer.resolveTools( + snapshot: makeSnapshot(webSearchEnabled: false), + executionMode: .none, + query: "What changed in the latest mlx release?" + ) + let namesWithoutWeb = Set(withoutWeb.map(\.function.name)) + #expect(!namesWithoutWeb.contains("web_search")) + #expect(!namesWithoutWeb.contains("search_and_extract")) + } } diff --git a/Packages/OsaurusCore/Tests/Search/WebSearchToolTests.swift b/Packages/OsaurusCore/Tests/Search/WebSearchToolTests.swift index a426109bd..51c9db5a5 100644 --- a/Packages/OsaurusCore/Tests/Search/WebSearchToolTests.swift +++ b/Packages/OsaurusCore/Tests/Search/WebSearchToolTests.swift @@ -48,7 +48,13 @@ struct WebSearchToolTests { let description = WebSearchTool().description #expect(description.contains("does not fetch page bodies")) #expect(description.contains("search_and_extract")) - #expect(description.contains("tool/search_and_extract")) + // The old contract routed through `capabilities_load tool/…` when + // retrieval wasn't loaded. `search_and_extract` is now composed into + // the schema whenever web search is enabled, so the description must + // name the direct transition — a load detour here would send models + // to a loader for a tool already in their schema. + #expect(!description.contains("capabilities_load")) + #expect(description.contains("IS the fetch tool")) } @Test func searchAndExtractContractProvidesPageContent() { @@ -316,7 +322,11 @@ struct WebSearchToolTests { ) let names = Set(tools.map { $0.function.name }) #expect(names.contains("web_search")) - #expect(!names.contains("search_and_extract")) + // Retrieval rides along with discovery now: web-enabled agents get + // `search_and_extract` on turn 1 instead of a capabilities_load + // round-trip. The old expectation pinned the lean baseline that left + // models looping `web_search` rephrases or hallucinating `web_fetch`. + #expect(names.contains("search_and_extract")) } @Test func chartAndWebExposeRetrievalOnTheFirstTurn() { diff --git a/Packages/OsaurusCore/Tests/Tool/ToolScopeGateRecoveryTests.swift b/Packages/OsaurusCore/Tests/Tool/ToolScopeGateRecoveryTests.swift index e8197e046..a13158faa 100644 --- a/Packages/OsaurusCore/Tests/Tool/ToolScopeGateRecoveryTests.swift +++ b/Packages/OsaurusCore/Tests/Tool/ToolScopeGateRecoveryTests.swift @@ -278,6 +278,49 @@ struct ToolScopeGateRecoveryTests { #expect(!ToolEnvelope.isError(result)) } + // MARK: - Hallucinated fetch-tool steering + + @Test + func hallucinatedFetchName_isSteeredToSearchAndExtract_whenExposed() async throws { + // `web_fetch` has never existed in osaurus, but models trained on + // other harnesses call it as if it were universal — observed live as + // a research run that burned its whole turn on it and read zero + // pages. When the request exposes `search_and_extract`, the dead end + // must instead point at the fetch tool already in the schema. + let scope = ToolExecutionScope(exposed: []) + scope.activate(["web_search", "search_and_extract"]) + let result = try await ChatExecutionContext.$toolExecutionScope.withValue(scope) { + try await ToolRegistry.shared.execute( + name: "web_fetch", + argumentsJSON: "{\"url\": \"https://example.com\"}" + ) + } + let parsed = try envelope(result) + #expect(parsed?["ok"] as? Bool == false) + #expect(parsed?["kind"] as? String == "tool_not_found") + #expect(parsed?["retryable"] as? Bool == true) + let message = parsed?["message"] as? String ?? "" + #expect(message.contains("search_and_extract")) + #expect(message.contains("urls")) + } + + @Test + func hallucinatedFetchName_staysOpaque_whenRetrievalIsNotExposed() async throws { + // Web disabled for this agent: the steering must not leak the name of + // a tool the request cannot call. The plain unregistered-name refusal + // applies. + let scope = ToolExecutionScope(exposed: []) + scope.activate(["get_current_time"]) + let result = try await ChatExecutionContext.$toolExecutionScope.withValue(scope) { + try await ToolRegistry.shared.execute(name: "web_fetch", argumentsJSON: "{}") + } + let parsed = try envelope(result) + #expect(parsed?["ok"] as? Bool == false) + #expect(parsed?["kind"] as? String == "tool_not_found") + let message = parsed?["message"] as? String ?? "" + #expect(!message.contains("search_and_extract")) + } + // MARK: - Workspace-tool dead end names the real next step @Test diff --git a/Packages/OsaurusCore/Tools/ToolRegistry.swift b/Packages/OsaurusCore/Tools/ToolRegistry.swift index 6cdf07734..1d3647359 100644 --- a/Packages/OsaurusCore/Tools/ToolRegistry.swift +++ b/Packages/OsaurusCore/Tools/ToolRegistry.swift @@ -899,6 +899,39 @@ public final class ToolRegistry: ObservableObject { // `tool/` handling in CapabilityTools.resolve. let name = resolvedRegisteredName(for: rawName) + // Hallucinated fetch tools get steered, not dead-ended. Models trained + // on other harnesses call `web_fetch` / `fetch_url` / `open_url` as if + // they were universal — observed live: a Raptor research run burned its + // entire turn on `web_fetch` (a tool that has never existed in osaurus) + // and produced zero page reads. The general no-"did you mean" rule + // exists because naming other tools teaches models to invent siblings; + // it does not apply here, because the steering target is only named + // when THIS request already exposes it — the model is being pointed at + // a tool sitting in its own schema, not at a rumor. Guarded to names + // with no real registration so a plugin/MCP tool that legitimately + // registers one of these names always wins — and a user-installed + // plugin GROUP that happens to share an alias name (`plugin/fetch`) + // keeps its own load rescue instead of being steered away from the + // capability the user deliberately installed. + if toolsByName[name] == nil, Self.hallucinatedFetchToolNames.contains(name.lowercased()), + toolsByName["search_and_extract"] != nil, + ChatExecutionContext.toolExecutionScope?.permits("search_and_extract") == true, + groupIdCallRescueEnvelope(for: name) == nil + { + ToolRegistryLogger.registry.notice( + "steering hallucinated fetch tool '\(name, privacy: .public)' to search_and_extract" + ) + return ToolErrorEnvelope( + kind: .toolNotFound, + reason: + "There is no '\(name)' tool. To fetch a web page, call search_and_extract " + + "with {\"urls\": [\"\"]} — it fetches the pages and returns their " + + "readable content. To find sources first, call web_search.", + toolName: name, + retryable: true + ).toJSONString() + } + // A tool this request never exposed must not run, however convincingly the model asks // for it. // @@ -2454,6 +2487,23 @@ public final class ToolRegistry: ObservableObject { "capabilities", "capabilities_discover", "capabilities_load", ] + /// Fetch-tool names models import from other harnesses (Claude Code's + /// WebFetch, LangChain requests_get, the reference MCP fetch server, …). + /// None have ever existed in osaurus; calls to them are steered to + /// `search_and_extract` when this request exposes it (see `execute`). + /// Lowercase; matched case-insensitively. Only consulted for names with + /// NO real registration, so a plugin/MCP tool legitimately claiming one + /// of these always wins. Deliberately excludes browser-intent names + /// (`browse`, `open_url`) — steering "log in and click" intent to a + /// read-only extractor points AWAY from a `browser_use` sitting in the + /// schema — and shell-intent names (`curl`), whose POST/API shapes + /// extraction cannot serve. + static let hallucinatedFetchToolNames: Set = [ + "web_fetch", "webfetch", "fetch", "fetch_url", "fetch_page", + "fetch_webpage", "http_get", "get_url", "get_webpage", "read_url", + "read_webpage", "visit_page", "load_url", "url_fetch", + ] + /// Built-in tools that are authoritatively gated per-agent and must never /// surface through `capabilities_discover`. Unlike the lean-by-default /// built-in gates (render_chart, speak, search_memory, the scheduler trio, diff --git a/Packages/OsaurusCore/Tools/WebSearchTools.swift b/Packages/OsaurusCore/Tools/WebSearchTools.swift index f61e2c6b0..02bcc8c95 100644 --- a/Packages/OsaurusCore/Tools/WebSearchTools.swift +++ b/Packages/OsaurusCore/Tools/WebSearchTools.swift @@ -174,9 +174,9 @@ final class WebSearchTool: OsaurusTool, @unchecked Sendable { "Discover relevant web sources. Just pass `query`; results come from the user's " + "configured search providers with automatic fallback. Returns ranked titles, URLs, " + "and snippets only — it does not fetch page bodies or downloadable data. Once you " - + "select a source, retrieve its content with `search_and_extract`; if that tool is not " - + "loaded and `capabilities_load` is available, load `tool/search_and_extract`. Do not " - + "keep rephrasing `web_search` when you need source content." + + "select a source, retrieve its content with `search_and_extract` (pass the chosen " + + "URLs). Do not keep rephrasing `web_search` when you need source content, and do " + + "not invent fetch tools — `search_and_extract` IS the fetch tool." // Immutable by design: `web_search` is an always-loaded baseline tool, so // its schema is part of every composed prompt's static prefix. Deriving