Skip to content

Commit 2d6a935

Browse files
fix(keenable): read result snippet from snippet, not description
The Keenable API returns both fields on every result. `description` is frequently empty and `snippet` carries the page text, so WebSearchResult came back with a title and a link and an empty snippet. Verified against the live keyless endpoint: 10/10 results had `len(description) == 0` and `len(snippet)` around 2000-2600. Reads `snippet` first with a `description` fallback. Keenable returns whole-page text where the other providers return a short snippet, so the text is whitespace-collapsed (it arrives with newlines) and capped at 500 characters to keep `WebSearchResult.snippet` preview-sized; the full page is still available through the contents step. Adds a unit test covering the mapping with a realistic fixture (empty `description`, populated `snippet`), the fallback, the whitespace collapse and the cap, and link-less rows.
1 parent 4fbb3ca commit 2d6a935

2 files changed

Lines changed: 116 additions & 1 deletion

File tree

backend/onyx/tools/tool_implementations/web_search/clients/keenable_client.py

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414

1515
KEENABLE_DEFAULT_BASE_URL = "https://api.keenable.ai"
1616
KEENABLE_REQUEST_TIMEOUT_SECONDS = 30
17+
# Keenable returns whole-page text where the other providers return a short
18+
# snippet, so cap it to keep WebSearchResult.snippet preview-sized. The full
19+
# page is available through the contents step.
20+
KEENABLE_MAX_SNIPPET_CHARS = 500
1721

1822

1923
class RetryableKeenableSearchError(Exception):
@@ -113,7 +117,7 @@ def _search_with_retries(self, query: str) -> list[WebSearchResult]:
113117
WebSearchResult(
114118
title=(result.get("title") or "").strip(),
115119
link=link,
116-
snippet=(result.get("description") or "").strip(),
120+
snippet=_extract_snippet(result),
117121
author=result.get("author"),
118122
published_date=None,
119123
)
@@ -164,6 +168,20 @@ def test_connection(self) -> dict[str, str]:
164168
return {"status": "ok"}
165169

166170

171+
def _extract_snippet(result: dict[str, Any]) -> str:
172+
"""Pull a result's text out of the Keenable response.
173+
174+
Keenable returns both `snippet` and `description`. `snippet` carries the
175+
page text and `description` is frequently empty, so prefer whichever has
176+
content. Snippets are raw page text with newlines in them, so collapse
177+
whitespace and cap the length.
178+
"""
179+
text = " ".join(
180+
str(result.get("snippet") or result.get("description") or "").split()
181+
)
182+
return text[:KEENABLE_MAX_SNIPPET_CHARS]
183+
184+
167185
def _build_error_message(response: requests.Response) -> str:
168186
return (
169187
f"Keenable search failed (status {response.status_code}): "
Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
from typing import Any
2+
from unittest.mock import MagicMock
3+
from unittest.mock import patch
4+
5+
from onyx.tools.tool_implementations.web_search.clients.keenable_client import (
6+
KEENABLE_MAX_SNIPPET_CHARS,
7+
)
8+
from onyx.tools.tool_implementations.web_search.clients.keenable_client import (
9+
KeenableClient,
10+
)
11+
12+
13+
def _response(results: list[dict[str, Any]]) -> MagicMock:
14+
response = MagicMock()
15+
response.json.return_value = {"results": results}
16+
response.raise_for_status.return_value = None
17+
return response
18+
19+
20+
@patch(
21+
"onyx.tools.tool_implementations.web_search.clients.keenable_client.requests.post"
22+
)
23+
def test_search_reads_the_snippet_field(mock_post: MagicMock) -> None:
24+
"""Keenable returns both fields and `description` is frequently empty."""
25+
mock_post.return_value = _response(
26+
[
27+
{
28+
"url": "https://example.com/one",
29+
"title": "One",
30+
"description": "",
31+
"snippet": "First page text",
32+
}
33+
]
34+
)
35+
36+
results = KeenableClient().search("test query")
37+
38+
assert len(results) == 1
39+
assert results[0].snippet == "First page text"
40+
41+
42+
@patch(
43+
"onyx.tools.tool_implementations.web_search.clients.keenable_client.requests.post"
44+
)
45+
def test_search_falls_back_to_description(mock_post: MagicMock) -> None:
46+
mock_post.return_value = _response(
47+
[
48+
{
49+
"url": "https://example.com/one",
50+
"title": "One",
51+
"description": "A description",
52+
}
53+
]
54+
)
55+
56+
results = KeenableClient().search("test query")
57+
58+
assert results[0].snippet == "A description"
59+
60+
61+
@patch(
62+
"onyx.tools.tool_implementations.web_search.clients.keenable_client.requests.post"
63+
)
64+
def test_search_collapses_whitespace_and_caps_the_snippet(mock_post: MagicMock) -> None:
65+
"""Snippets are raw page text: newlines in them, and far longer than a snippet."""
66+
mock_post.return_value = _response(
67+
[
68+
{
69+
"url": "https://example.com/one",
70+
"title": "One",
71+
"description": "",
72+
"snippet": "line one\n\nline two" + " padding" * 500,
73+
}
74+
]
75+
)
76+
77+
snippet = KeenableClient().search("test query")[0].snippet
78+
79+
assert len(snippet) == KEENABLE_MAX_SNIPPET_CHARS
80+
assert "\n" not in snippet
81+
assert snippet.startswith("line one line two")
82+
83+
84+
@patch(
85+
"onyx.tools.tool_implementations.web_search.clients.keenable_client.requests.post"
86+
)
87+
def test_search_skips_results_without_a_link(mock_post: MagicMock) -> None:
88+
mock_post.return_value = _response(
89+
[
90+
{"title": "No URL", "snippet": "text"},
91+
{"url": "https://example.com/one", "title": "One", "snippet": "text"},
92+
]
93+
)
94+
95+
results = KeenableClient().search("test query")
96+
97+
assert [result.link for result in results] == ["https://example.com/one"]

0 commit comments

Comments
 (0)