From 142937835a4ee3c64804c13a2ab43e7f5ba47b3e Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:09:56 +0200 Subject: [PATCH 01/15] feat(apify): add apify-client dep, lazy_deps entry, and cache slots --- plugins/web/apify/plugin.yaml | 7 +++++++ pyproject.toml | 1 + tools/lazy_deps.py | 1 + tools/web_tools.py | 2 ++ 4 files changed, 11 insertions(+) create mode 100644 plugins/web/apify/plugin.yaml diff --git a/plugins/web/apify/plugin.yaml b/plugins/web/apify/plugin.yaml new file mode 100644 index 000000000000..4c4f70595bd8 --- /dev/null +++ b/plugins/web/apify/plugin.yaml @@ -0,0 +1,7 @@ +name: web-apify +version: 1.0.0 +description: "Apify web search (RAG Web Browser) and page extract (Website Content Crawler)" +author: Apify +kind: backend +provides_web_providers: + - apify diff --git a/pyproject.toml b/pyproject.toml index ae2472b7a105..7d272b99a5c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -74,6 +74,7 @@ anthropic = ["anthropic==0.86.0"] # search provider (configured via `hermes tools` or config.yaml). exa = ["exa-py==2.10.2"] firecrawl = ["firecrawl-py==4.17.0"] +apify = ["apify-client==3.0.1"] parallel-web = ["parallel-web==0.4.2"] # Image generation backends fal = ["fal-client==0.13.1"] diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 1a8708ef25c0..e1ff6ea50f9d 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -90,6 +90,7 @@ # ─── Web search backends ─────────────────────────────────────────────── "search.exa": ("exa-py==2.10.2",), "search.firecrawl": ("firecrawl-py==4.17.0",), + "search.apify": ("apify-client==3.0.1",), "search.parallel": ("parallel-web==0.4.2",), # ─── TTS providers ───────────────────────────────────────────────────── diff --git a/tools/web_tools.py b/tools/web_tools.py index a55fe78c41e4..466acbda546e 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -96,6 +96,8 @@ _parallel_client: Optional[Any] = None _async_parallel_client: Optional[Any] = None _exa_client: Optional[Any] = None +_apify_client: Optional[Any] = None +_apify_client_config: Optional[Any] = None from agent.auxiliary_client import ( async_call_llm, From 53f760ab5c3e86e8289c31883db80c700359d42f Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:35:30 +0200 Subject: [PATCH 02/15] =?UTF-8?q?feat(apify):=20provider=20skeleton=20?= =?UTF-8?q?=E2=80=94=20registry,=20capability=20flags,=20setup=20schema,?= =?UTF-8?q?=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Sonnet 4.6 --- plugins/web/apify/provider.py | 117 ++++++++++++++++++ .../web/test_web_search_provider_plugins.py | 50 +++++++- 2 files changed, 164 insertions(+), 3 deletions(-) create mode 100644 plugins/web/apify/provider.py diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py new file mode 100644 index 000000000000..524446d0335f --- /dev/null +++ b/plugins/web/apify/provider.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import asyncio +import logging +import os +from typing import Any, Dict, List, Optional + +from agent.web_search_provider import WebSearchProvider +from tools.website_policy import check_website_access + +logger = logging.getLogger(__name__) + +_APIFY_CLIENT_CLS_CACHE: Optional[type] = None + + +def _load_apify_client_cls() -> type: + """Import and cache apify_client.ApifyClient (lazy, deferred on first use).""" + global _APIFY_CLIENT_CLS_CACHE + if _APIFY_CLIENT_CLS_CACHE is None: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("search.apify", prompt=False) + except ImportError: + pass + except Exception as exc: + raise ImportError(str(exc)) + from apify_client import ApifyClient + _APIFY_CLIENT_CLS_CACHE = ApifyClient + return _APIFY_CLIENT_CLS_CACHE + + +def _get_apify_client() -> Any: + """Return cached ApifyClient, constructing it from APIFY_API_TOKEN. + + Raises ValueError when APIFY_API_TOKEN is not set. + Cache stored on tools.web_tools._apify_client so tests can reset it via + ``tools.web_tools._apify_client = None``. + """ + import tools.web_tools as _wt + + api_token = os.getenv("APIFY_API_TOKEN", "").strip() + if not api_token: + raise ValueError( + "Apify tools are not configured. " + "Set APIFY_API_TOKEN (get one at https://apify.com/account/integrations)." + ) + + client_config = ("direct", api_token) + cached = getattr(_wt, "_apify_client", None) + cached_config = getattr(_wt, "_apify_client_config", None) + if cached is not None and cached_config == client_config: + return cached + + ApifyClient = _load_apify_client_cls() + _wt._apify_client = ApifyClient(token=api_token) + _wt._apify_client_config = client_config + return _wt._apify_client + + +def _reset_client_for_tests() -> None: + """Drop cached Apify client so tests can re-instantiate cleanly.""" + import tools.web_tools as _wt + _wt._apify_client = None + _wt._apify_client_config = None + + +class ApifyWebSearchProvider(WebSearchProvider): + """Apify web search + extract provider. + + search() — apify/rag-web-browser Actor (sync, 60s timeout in run input) + extract() — apify/website-content-crawler Actor (async, per-URL, 60s asyncio guard) + crawl() — not supported in v1 (supports_crawl returns False) + """ + + @property + def name(self) -> str: + return "apify" + + @property + def display_name(self) -> str: + return "Apify" + + def is_available(self) -> bool: + return bool(os.getenv("APIFY_API_TOKEN", "").strip()) + + def supports_search(self) -> bool: + return True + + def supports_extract(self) -> bool: + return True + + def supports_crawl(self) -> bool: + return False + + def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + raise NotImplementedError("search() implemented in Task 3") + + async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: + raise NotImplementedError("extract() implemented in Task 4") + + def get_setup_schema(self) -> Dict[str, Any]: + return { + "name": "Apify", + "badge": "paid · free tier", + "tag": ( + "JS-rendered, bot-protected, and geo-gated pages via Apify's " + "residential proxy infrastructure. Uses RAG Web Browser for search " + "and Website Content Crawler for extract." + ), + "env_vars": [ + { + "key": "APIFY_API_TOKEN", + "prompt": "Apify API token", + "url": "https://apify.com/account/integrations", + }, + ], + } diff --git a/tests/plugins/web/test_web_search_provider_plugins.py b/tests/plugins/web/test_web_search_provider_plugins.py index 47d7791977b9..2f2dee7f99f7 100644 --- a/tests/plugins/web/test_web_search_provider_plugins.py +++ b/tests/plugins/web/test_web_search_provider_plugins.py @@ -48,6 +48,7 @@ def _clear_web_env(monkeypatch: pytest.MonkeyPatch) -> None: "TOOL_GATEWAY_DOMAIN", "TOOL_GATEWAY_USER_TOKEN", "XAI_API_KEY", + "APIFY_API_TOKEN", ): monkeypatch.delenv(k, raising=False) @@ -73,12 +74,13 @@ def _isolate_env(monkeypatch: pytest.MonkeyPatch) -> None: class TestBundledPluginsRegister: """All eight bundled web plugins discover and register correctly.""" - def test_all_seven_plugins_present_in_registry(self) -> None: + def test_all_bundled_plugins_present_in_registry(self) -> None: _ensure_plugins_loaded() from agent.web_search_registry import list_providers names = sorted(p.name for p in list_providers()) assert names == [ + "apify", "brave-free", "ddgs", "exa", @@ -104,6 +106,7 @@ def test_all_seven_plugins_present_in_registry(self) -> None: ("firecrawl", True, True, True), # xai: search-only via Grok's agentic web_search tool. ("xai", True, False, False), + ("apify", True, True, False), ], ) def test_capability_flags_match_spec( @@ -124,7 +127,7 @@ def test_capability_flags_match_spec( @pytest.mark.parametrize( "plugin_name", - ["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"], + ["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai", "apify"], ) def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None: _ensure_plugins_loaded() @@ -137,7 +140,7 @@ def test_each_plugin_has_name_and_display_name(self, plugin_name: str) -> None: @pytest.mark.parametrize( "plugin_name", - ["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai"], + ["brave-free", "ddgs", "searxng", "exa", "parallel", "tavily", "firecrawl", "xai", "apify"], ) def test_each_plugin_has_setup_schema(self, plugin_name: str) -> None: """``get_setup_schema()`` returns a dict the picker can consume.""" @@ -254,6 +257,16 @@ def test_xai_requires_api_key_or_oauth(self, monkeypatch: pytest.MonkeyPatch) -> monkeypatch.setenv("XAI_API_KEY", "real") assert p.is_available() is True + def test_apify_requires_api_token(self, monkeypatch: pytest.MonkeyPatch) -> None: + _ensure_plugins_loaded() + from agent.web_search_registry import get_provider + + p = get_provider("apify") + assert p is not None + assert p.is_available() is False # no APIFY_API_TOKEN + monkeypatch.setenv("APIFY_API_TOKEN", "test-token") + assert p.is_available() is True + # --------------------------------------------------------------------------- # Registry resolution semantics (Option B — conservative smart fallback) @@ -377,6 +390,14 @@ def test_tavily_extract_is_sync(self) -> None: assert p is not None assert inspect.iscoroutinefunction(p.extract) is False + def test_apify_extract_is_async(self) -> None: + _ensure_plugins_loaded() + from agent.web_search_registry import get_provider + + p = get_provider("apify") + assert p is not None + assert inspect.iscoroutinefunction(p.extract) is True + # --------------------------------------------------------------------------- # Error response shape (preserved bit-for-bit from legacy) @@ -500,3 +521,26 @@ def test_xai_search_returns_error_dict_when_unconfigured(self) -> None: assert isinstance(result, dict) assert result.get("success") is False assert "error" in result + + def test_apify_search_returns_error_dict_when_unconfigured(self) -> None: + _ensure_plugins_loaded() + from agent.web_search_registry import get_provider + + p = get_provider("apify") + assert p is not None + result = p.search("test query", limit=5) + assert isinstance(result, dict) + assert result.get("success") is False + assert "error" in result + + def test_apify_extract_returns_per_url_errors_when_unconfigured(self) -> None: + _ensure_plugins_loaded() + from agent.web_search_registry import get_provider + + p = get_provider("apify") + assert p is not None + result = asyncio.run(p.extract(["https://example.com"])) + assert isinstance(result, list) + assert len(result) == 1 + assert "error" in result[0] + assert result[0]["url"] == "https://example.com" From d82f0e411a79bbd1f5810d40409c05e5472ae732 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:41:54 +0200 Subject: [PATCH 03/15] fix(apify): add credential-check stubs to search/extract for error-shape tests --- plugins/web/apify/provider.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 524446d0335f..0e7a37df01f3 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -93,9 +93,17 @@ def supports_crawl(self) -> bool: return False def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + try: + _get_apify_client() + except (ValueError, ImportError) as exc: + return {"success": False, "error": str(exc)} raise NotImplementedError("search() implemented in Task 3") async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: + try: + _get_apify_client() + except (ValueError, ImportError) as exc: + return [{"url": u, "title": "", "content": "", "raw_content": "", "error": str(exc)} for u in urls] raise NotImplementedError("extract() implemented in Task 4") def get_setup_schema(self) -> Dict[str, Any]: From 1bee9e1f46e6e1345fc2dc5e619a55423d708b10 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:44:34 +0200 Subject: [PATCH 04/15] feat(apify): implement search() via RAG Web Browser Actor --- plugins/web/apify/provider.py | 59 ++++++++++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 0e7a37df01f3..5a6e967154a0 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -64,6 +64,29 @@ def _reset_client_for_tests() -> None: _wt._apify_client_config = None +def _normalize_rag_search_results(items: List[Any], limit: int) -> List[Dict[str, Any]]: + """Normalize RAG Web Browser dataset items to the registry web search shape.""" + results: List[Dict[str, Any]] = [] + for i, item in enumerate(items[:limit]): + if not isinstance(item, dict): + continue + sr = item.get("searchResult") or {} + if not isinstance(sr, dict): + sr = {} + title = sr.get("title") or item.get("title", "") + url = sr.get("url") or item.get("url", "") + description = sr.get("description") or item.get("markdown", "") + if description and len(description) > 500: + description = description[:500] + results.append({ + "title": title, + "url": url, + "description": description, + "position": i + 1, + }) + return results + + class ApifyWebSearchProvider(WebSearchProvider): """Apify web search + extract provider. @@ -93,11 +116,39 @@ def supports_crawl(self) -> bool: return False def search(self, query: str, limit: int = 5) -> Dict[str, Any]: + """Execute a web search via Apify RAG Web Browser Actor. + + Sync; blocks until the Actor run completes (up to requestTimeoutSecs). + """ + from tools.interrupt import is_interrupted + + if is_interrupted(): + return {"success": False, "error": "Interrupted"} + + logger.info("Apify search: '%s' (limit=%d)", query, limit) try: - _get_apify_client() - except (ValueError, ImportError) as exc: - return {"success": False, "error": str(exc)} - raise NotImplementedError("search() implemented in Task 3") + client = _get_apify_client() + run = client.actor("apify/rag-web-browser").call( + run_input={ + "query": query, + "maxResults": limit, + "requestTimeoutSecs": 60, + } + ) + if run is None: + return {"success": False, "error": "Apify actor run returned no result"} + + dataset_id = run.get("defaultDatasetId") + if not dataset_id: + return {"success": False, "error": "Apify run missing defaultDatasetId"} + + items = client.dataset(dataset_id).list_items().items + web_results = _normalize_rag_search_results(items, limit) + logger.info("Apify search: found %d results", len(web_results)) + return {"success": True, "data": {"web": web_results}} + except Exception as exc: + logger.warning("Apify search error: %s", exc) + return {"success": False, "error": f"Apify search failed: {exc}"} async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: try: From 3ab1f739a8a91abfe1a5a60512d507db3d3471cd Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:48:27 +0200 Subject: [PATCH 05/15] fix(apify): use sequential position counter in _normalize_rag_search_results --- plugins/web/apify/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 5a6e967154a0..356175370f07 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -82,7 +82,7 @@ def _normalize_rag_search_results(items: List[Any], limit: int) -> List[Dict[str "title": title, "url": url, "description": description, - "position": i + 1, + "position": len(results) + 1, }) return results From 8ac6082901d51fd19ead4e42c49834a9adc11b8a Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:48:57 +0200 Subject: [PATCH 06/15] fix(apify): remove unused loop var in _normalize_rag_search_results --- plugins/web/apify/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 356175370f07..68c87911a073 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -67,7 +67,7 @@ def _reset_client_for_tests() -> None: def _normalize_rag_search_results(items: List[Any], limit: int) -> List[Dict[str, Any]]: """Normalize RAG Web Browser dataset items to the registry web search shape.""" results: List[Dict[str, Any]] = [] - for i, item in enumerate(items[:limit]): + for item in items[:limit]: if not isinstance(item, dict): continue sr = item.get("searchResult") or {} From fbce2beaa79614ac033765e53bb2451cb4b2d456 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:51:05 +0200 Subject: [PATCH 07/15] feat(apify): implement extract() via Website Content Crawler Actor Add _run_website_content_crawler helper and replace the extract() stub with the full async implementation: per-URL crawl via asyncio.to_thread, 60s wait_for guard, pre/post-redirect website policy checks, format selection (markdown/html/both), and per-URL error items on any failure. Co-Authored-By: Claude Sonnet 4.6 --- plugins/web/apify/provider.py | 171 +++++++++++++++++++++++++++++++++- 1 file changed, 166 insertions(+), 5 deletions(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 68c87911a073..73d0537a7b75 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -87,6 +87,32 @@ def _normalize_rag_search_results(items: List[Any], limit: int) -> List[Dict[str return results +def _run_website_content_crawler(url: str, output_formats: List[str]) -> Optional[Dict[str, Any]]: + """Blocking call to apify/website-content-crawler for a single URL. + + Returns the first dataset item (single-page crawl), or None if no items. + Intended to be called via asyncio.to_thread from extract(). + """ + client = _get_apify_client() + run = client.actor("apify/website-content-crawler").call( + run_input={ + "startUrls": [{"url": url}], + "maxCrawlPages": 1, + "outputFormats": output_formats, + } + ) + if run is None: + return None + dataset_id = run.get("defaultDatasetId") + if not dataset_id: + return None + items = client.dataset(dataset_id).list_items().items + if not items: + return None + item = items[0] + return item if isinstance(item, dict) else None + + class ApifyWebSearchProvider(WebSearchProvider): """Apify web search + extract provider. @@ -151,11 +177,146 @@ def search(self, query: str, limit: int = 5) -> Dict[str, Any]: return {"success": False, "error": f"Apify search failed: {exc}"} async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: - try: - _get_apify_client() - except (ValueError, ImportError) as exc: - return [{"url": u, "title": "", "content": "", "raw_content": "", "error": str(exc)} for u in urls] - raise NotImplementedError("extract() implemented in Task 4") + """Extract content from URLs via Apify Website Content Crawler. + + Each URL is crawled in a background thread (asyncio.to_thread) with a + 60s asyncio.wait_for guard. Website-access policy is checked before + each Actor call. Per-URL failures are returned as error items, not raised. + + kwargs: + format: "markdown" | "html" | None (default: both) + """ + from tools.interrupt import is_interrupted as _is_interrupted + + if _is_interrupted(): + return [{"url": u, "error": "Interrupted", "title": ""} for u in urls] + + fmt = kwargs.get("format") + if fmt == "markdown": + output_formats = ["markdown"] + elif fmt == "html": + output_formats = ["html"] + else: + output_formats = ["markdown", "html"] + + results: List[Dict[str, Any]] = [] + + for url in urls: + if _is_interrupted(): + results.append({"url": url, "error": "Interrupted", "title": ""}) + continue + + blocked = check_website_access(url) + if blocked: + logger.info( + "Blocked web_extract for %s by rule %s", + blocked["host"], + blocked["rule"], + ) + results.append( + { + "url": url, + "title": "", + "content": "", + "error": blocked["message"], + "blocked_by_policy": { + "host": blocked["host"], + "rule": blocked["rule"], + "source": blocked["source"], + }, + } + ) + continue + + try: + logger.info("Apify extracting: %s", url) + try: + item = await asyncio.wait_for( + asyncio.to_thread( + _run_website_content_crawler, + url, + output_formats, + ), + timeout=60, + ) + except asyncio.TimeoutError: + logger.warning("Apify WCC timed out for %s", url) + results.append( + { + "url": url, + "title": "", + "content": "", + "error": ( + "Extract timed out after 60s — page may be too large " + "or unresponsive. Try browser_navigate instead." + ), + } + ) + continue + + if item is None: + results.append( + {"url": url, "title": "", "content": "", "error": "Actor returned no content"} + ) + continue + + final_url = item.get("url", url) + + final_blocked = check_website_access(final_url) + if final_blocked: + logger.info( + "Blocked redirected web_extract for %s by rule %s", + final_blocked["host"], + final_blocked["rule"], + ) + results.append( + { + "url": final_url, + "title": item.get("title", ""), + "content": "", + "raw_content": "", + "error": final_blocked["message"], + "blocked_by_policy": { + "host": final_blocked["host"], + "rule": final_blocked["rule"], + "source": final_blocked["source"], + }, + } + ) + continue + + content_markdown = item.get("markdown") + content_html = item.get("html") + title = item.get("title", "") + metadata = item.get("metadata") or {} + + if fmt == "markdown" or (fmt is None and content_markdown): + chosen_content = content_markdown or "" + else: + chosen_content = content_html or content_markdown or "" + + results.append( + { + "url": final_url, + "title": title, + "content": chosen_content, + "raw_content": chosen_content, + "metadata": metadata, + } + ) + except Exception as exc: + logger.debug("Apify extract failed for %s: %s", url, exc) + results.append( + { + "url": url, + "title": "", + "content": "", + "raw_content": "", + "error": str(exc), + } + ) + + return results def get_setup_schema(self) -> Dict[str, Any]: return { From 02bf7ede156c713b3435ed2b77ad0897c6e44cf3 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:54:04 +0200 Subject: [PATCH 08/15] fix(apify): add noqa BLE001 to broad except in extract() for linter parity --- plugins/web/apify/provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 73d0537a7b75..841509fa34ea 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -304,7 +304,7 @@ async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: "metadata": metadata, } ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 logger.debug("Apify extract failed for %s: %s", url, exc) results.append( { From 53cd5f0e9a5a17121c2dd046689ba458efcf4832 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Mon, 25 May 2026 12:59:22 +0200 Subject: [PATCH 09/15] fix(apify): add noqa BLE001 to remaining broad excepts for linter parity --- plugins/web/apify/provider.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 841509fa34ea..b981cd487888 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -22,7 +22,7 @@ def _load_apify_client_cls() -> type: _lazy_ensure("search.apify", prompt=False) except ImportError: pass - except Exception as exc: + except Exception as exc: # noqa: BLE001 raise ImportError(str(exc)) from apify_client import ApifyClient _APIFY_CLIENT_CLS_CACHE = ApifyClient @@ -172,7 +172,7 @@ def search(self, query: str, limit: int = 5) -> Dict[str, Any]: web_results = _normalize_rag_search_results(items, limit) logger.info("Apify search: found %d results", len(web_results)) return {"success": True, "data": {"web": web_results}} - except Exception as exc: + except Exception as exc: # noqa: BLE001 logger.warning("Apify search error: %s", exc) return {"success": False, "error": f"Apify search failed: {exc}"} From b6d20fe8ec6d382ff246856866e3021985ce1cf8 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Tue, 26 May 2026 12:36:12 +0200 Subject: [PATCH 10/15] fix: config UI, tool registration --- hermes_cli/config.py | 10 +++++++++- plugins/web/apify/__init__.py | 7 +++++++ plugins/web/apify/provider.py | 6 +++--- tools/web_tools.py | 8 +++++--- 4 files changed, 24 insertions(+), 7 deletions(-) create mode 100644 plugins/web/apify/__init__.py diff --git a/hermes_cli/config.py b/hermes_cli/config.py index e927996c3c85..177c20418c4b 100644 --- a/hermes_cli/config.py +++ b/hermes_cli/config.py @@ -2299,6 +2299,14 @@ def _ensure_hermes_home_managed(home: Path): "password": True, "category": "tool", }, + "APIFY_API_TOKEN": { + "description": "Apify API token for web search (RAG Web Browser) and page extract (Website Content Crawler)", + "prompt": "Apify API token", + "url": "https://apify.com/account/integrations", + "tools": ["web_search", "web_extract"], + "password": True, + "category": "tool", + }, "SEARXNG_URL": { "description": "URL of your SearXNG instance for free self-hosted web search", "prompt": "SearXNG URL (e.g. http://localhost:8080)", @@ -5257,7 +5265,7 @@ def set_config_value(key: str, value: str): 'OPENROUTER_API_KEY', 'OPENAI_API_KEY', 'ANTHROPIC_API_KEY', 'VOICE_TOOLS_OPENAI_KEY', 'EXA_API_KEY', 'PARALLEL_API_KEY', 'FIRECRAWL_API_KEY', 'FIRECRAWL_API_URL', 'FIRECRAWL_GATEWAY_URL', 'TOOL_GATEWAY_DOMAIN', 'TOOL_GATEWAY_SCHEME', - 'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY', + 'TOOL_GATEWAY_USER_TOKEN', 'TAVILY_API_KEY', 'APIFY_API_TOKEN', 'BROWSERBASE_API_KEY', 'BROWSERBASE_PROJECT_ID', 'BROWSER_USE_API_KEY', 'FAL_KEY', 'TELEGRAM_BOT_TOKEN', 'DISCORD_BOT_TOKEN', 'TERMINAL_SSH_HOST', 'TERMINAL_SSH_USER', 'TERMINAL_SSH_KEY', diff --git a/plugins/web/apify/__init__.py b/plugins/web/apify/__init__.py new file mode 100644 index 000000000000..e9310213f5a2 --- /dev/null +++ b/plugins/web/apify/__init__.py @@ -0,0 +1,7 @@ +from __future__ import annotations + +from plugins.web.apify.provider import ApifyWebSearchProvider + +def register(ctx) -> None: + """Register the Apify provider with the plugin context.""" + ctx.register_web_search_provider(ApifyWebSearchProvider()) \ No newline at end of file diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index b981cd487888..44748b115160 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -103,7 +103,7 @@ def _run_website_content_crawler(url: str, output_formats: List[str]) -> Optiona ) if run is None: return None - dataset_id = run.get("defaultDatasetId") + dataset_id = run.default_dataset_id if not dataset_id: return None items = client.dataset(dataset_id).list_items().items @@ -164,7 +164,7 @@ def search(self, query: str, limit: int = 5) -> Dict[str, Any]: if run is None: return {"success": False, "error": "Apify actor run returned no result"} - dataset_id = run.get("defaultDatasetId") + dataset_id = run.default_dataset_id if not dataset_id: return {"success": False, "error": "Apify run missing defaultDatasetId"} @@ -321,7 +321,7 @@ async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: def get_setup_schema(self) -> Dict[str, Any]: return { "name": "Apify", - "badge": "paid · free tier", + "badge": "paid", "tag": ( "JS-rendered, bot-protected, and geo-gated pages via Apify's " "residential proxy infrastructure. Uses RAG Web Browser for search " diff --git a/tools/web_tools.py b/tools/web_tools.py index 466acbda546e..ff38b5cebd5b 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -142,7 +142,7 @@ def _get_backend() -> str: keys manually without running setup. """ configured = (_load_web_config().get("backend") or "").lower().strip() - if configured in {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai"}: + if configured in {"parallel", "firecrawl", "tavily", "exa", "searxng", "brave-free", "ddgs", "xai", "apify"}: return configured # Fallback for manual / legacy config — pick the highest-priority @@ -230,6 +230,8 @@ def _is_backend_available(backend: str) -> bool: return has_xai_credentials() except Exception: return False + if backend == "apify": + return _has_env("APIFY_API_TOKEN") return False @@ -1369,11 +1371,11 @@ async def _process_tavily_crawl(result): def check_web_api_key() -> bool: """Check whether the configured web backend is available.""" configured = _load_web_config().get("backend", "").lower().strip() - if configured in {"exa", "parallel", "firecrawl", "tavily", "searxng", "brave-free", "ddgs"}: + if configured in {"exa", "parallel", "firecrawl", "tavily", "searxng", "brave-free", "ddgs", "apify"}: return _is_backend_available(configured) return any( _is_backend_available(backend) - for backend in ("exa", "parallel", "firecrawl", "tavily", "searxng", "brave-free", "ddgs") + for backend in ("exa", "parallel", "firecrawl", "tavily", "searxng", "brave-free", "ddgs", "apify") ) From 508b84c8bc7a3b3c594315969219f21eb29b5190 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Tue, 26 May 2026 13:01:15 +0200 Subject: [PATCH 11/15] feat: added progress logging --- plugins/web/apify/provider.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 44748b115160..7bdbfbecca17 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -10,9 +10,13 @@ logger = logging.getLogger(__name__) + + _APIFY_CLIENT_CLS_CACHE: Optional[type] = None + + def _load_apify_client_cls() -> type: """Import and cache apify_client.ApifyClient (lazy, deferred on first use).""" global _APIFY_CLIENT_CLS_CACHE @@ -94,15 +98,18 @@ def _run_website_content_crawler(url: str, output_formats: List[str]) -> Optiona Intended to be called via asyncio.to_thread from extract(). """ client = _get_apify_client() - run = client.actor("apify/website-content-crawler").call( + run = client.actor("apify/website-content-crawler").start( run_input={ "startUrls": [{"url": url}], "maxCrawlPages": 1, "outputFormats": output_formats, } ) + logger.info("Apify website-content-crawler started — https://console.apify.com/actors/runs/%s", run.id) + run = client.run(run.id).wait_for_finish() if run is None: return None + logger.info("Apify website-content-crawler: %s", run.status) dataset_id = run.default_dataset_id if not dataset_id: return None @@ -154,15 +161,18 @@ def search(self, query: str, limit: int = 5) -> Dict[str, Any]: logger.info("Apify search: '%s' (limit=%d)", query, limit) try: client = _get_apify_client() - run = client.actor("apify/rag-web-browser").call( + run = client.actor("apify/rag-web-browser").start( run_input={ "query": query, "maxResults": limit, "requestTimeoutSecs": 60, } ) + logger.info("Apify rag-web-browser started — https://console.apify.com/actors/runs/%s", run.id) + run = client.run(run.id).wait_for_finish() if run is None: return {"success": False, "error": "Apify actor run returned no result"} + logger.info("Apify rag-web-browser: %s", run.status) dataset_id = run.default_dataset_id if not dataset_id: From 42edf9677bfc9989c299ecaac3c2904b33086658 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Tue, 26 May 2026 13:27:53 +0200 Subject: [PATCH 12/15] feat: implemented web_crawl_tool() --- plugins/web/apify/provider.py | 123 +++++++++++++++++- .../web/test_web_search_provider_plugins.py | 15 ++- 2 files changed, 134 insertions(+), 4 deletions(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 7bdbfbecca17..1658ed09d944 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -91,6 +91,34 @@ def _normalize_rag_search_results(items: List[Any], limit: int) -> List[Dict[str return results +def _run_wcc_crawl(url: str, max_pages: int, max_depth: int) -> List[Dict[str, Any]]: + """Blocking multi-page crawl via apify/website-content-crawler. + + Intended to be called via asyncio.to_thread from crawl(). + Returns raw dataset items (plain dicts). + """ + client = _get_apify_client() + run = client.actor("apify/website-content-crawler").start( + run_input={ + "startUrls": [{"url": url}], + "maxCrawlPages": max_pages, + "maxCrawlDepth": max_depth, + "outputFormats": ["markdown"], + "saveMarkdown": True, + "saveHtml": False, + } + ) + logger.info("Apify WCC crawl started — https://console.apify.com/actors/runs/%s", run.id) + run = client.run(run.id).wait_for_finish() + if run is None: + return [] + logger.info("Apify WCC crawl: %s", run.status) + dataset_id = run.default_dataset_id + if not dataset_id: + return [] + return client.dataset(dataset_id).list_items().items + + def _run_website_content_crawler(url: str, output_formats: List[str]) -> Optional[Dict[str, Any]]: """Blocking call to apify/website-content-crawler for a single URL. @@ -121,11 +149,11 @@ def _run_website_content_crawler(url: str, output_formats: List[str]) -> Optiona class ApifyWebSearchProvider(WebSearchProvider): - """Apify web search + extract provider. + """Apify web search + extract + crawl provider. search() — apify/rag-web-browser Actor (sync, 60s timeout in run input) extract() — apify/website-content-crawler Actor (async, per-URL, 60s asyncio guard) - crawl() — not supported in v1 (supports_crawl returns False) + crawl() — apify/website-content-crawler Actor (async, multi-page, 300s ceiling) """ @property @@ -146,7 +174,7 @@ def supports_extract(self) -> bool: return True def supports_crawl(self) -> bool: - return False + return True def search(self, query: str, limit: int = 5) -> Dict[str, Any]: """Execute a web search via Apify RAG Web Browser Actor. @@ -328,6 +356,95 @@ async def extract(self, urls: List[str], **kwargs: Any) -> List[Dict[str, Any]]: return results + async def crawl(self, url: str, **kwargs: Any) -> Dict[str, Any]: + """Crawl a seed URL via Apify Website Content Crawler. + + Multi-page crawl wrapped in asyncio.to_thread with a 300s ceiling. + Per-page URLs are re-checked against website-access policy. Per-page + failures are returned as error items, not raised. + + kwargs: + instructions: str — logged and dropped (WCC has no NL instructions param) + limit: int — max pages to crawl (default 20) + depth: "basic" → maxCrawlDepth=2, "advanced" → maxCrawlDepth=5, + int → direct, None → 0 (unlimited, capped by limit) + """ + from tools.interrupt import is_interrupted as _is_interrupted + + if _is_interrupted(): + return {"results": [{"url": url, "title": "", "content": "", "error": "Interrupted"}]} + + instructions = kwargs.get("instructions") + limit = int(kwargs.get("limit", 20)) + depth_raw = kwargs.get("depth") + + if depth_raw == "basic": + max_depth = 2 + elif depth_raw == "advanced": + max_depth = 5 + elif isinstance(depth_raw, int): + max_depth = depth_raw + else: + max_depth = 0 + + if instructions: + logger.info("Apify crawl: 'instructions' ignored (not supported by WCC)") + + logger.info("Apify crawl: %s (limit=%d, depth=%s)", url, limit, depth_raw or "unlimited") + + try: + items = await asyncio.wait_for( + asyncio.to_thread(_run_wcc_crawl, url, limit, max_depth), + timeout=300, + ) + except asyncio.TimeoutError: + logger.warning("Apify crawl timed out for %s", url) + return {"results": [{"url": url, "title": "", "content": "", + "error": "Crawl timed out after 300s"}]} + except Exception as exc: # noqa: BLE001 + logger.warning("Apify crawl error: %s", exc) + return {"results": [{"url": url, "title": "", "content": "", "error": str(exc)}]} + + pages: List[Dict[str, Any]] = [] + for item in items: + if not isinstance(item, dict): + continue + page_url = item.get("url", url) + title = item.get("title", "") + + blocked = check_website_access(page_url) + if blocked: + logger.info( + "Blocked crawled page %s by rule %s", + blocked["host"], + blocked["rule"], + ) + pages.append({ + "url": page_url, + "title": title, + "content": "", + "raw_content": "", + "error": blocked["message"], + "blocked_by_policy": { + "host": blocked["host"], + "rule": blocked["rule"], + "source": blocked["source"], + }, + }) + continue + + content = item.get("markdown") or item.get("text") or "" + pages.append({ + "url": page_url, + "title": title, + "content": content, + "raw_content": content, + "metadata": item.get("metadata") or {}, + }) + + logger.info("Apify crawl: %d pages collected", len(pages)) + return {"results": pages} + def get_setup_schema(self) -> Dict[str, Any]: return { "name": "Apify", diff --git a/tests/plugins/web/test_web_search_provider_plugins.py b/tests/plugins/web/test_web_search_provider_plugins.py index 2f2dee7f99f7..de9cc9b2d9c4 100644 --- a/tests/plugins/web/test_web_search_provider_plugins.py +++ b/tests/plugins/web/test_web_search_provider_plugins.py @@ -106,7 +106,7 @@ def test_all_bundled_plugins_present_in_registry(self) -> None: ("firecrawl", True, True, True), # xai: search-only via Grok's agentic web_search tool. ("xai", True, False, False), - ("apify", True, True, False), + ("apify", True, True, True), ], ) def test_capability_flags_match_spec( @@ -544,3 +544,16 @@ def test_apify_extract_returns_per_url_errors_when_unconfigured(self) -> None: assert len(result) == 1 assert "error" in result[0] assert result[0]["url"] == "https://example.com" + + def test_apify_crawl_returns_error_dict_when_unconfigured(self) -> None: + _ensure_plugins_loaded() + from agent.web_search_registry import get_provider + + p = get_provider("apify") + assert p is not None + result = asyncio.run(p.crawl("https://example.com")) + assert isinstance(result, dict) + assert "results" in result + assert len(result["results"]) >= 1 + assert "error" in result["results"][0] + assert result["results"][0]["url"] == "https://example.com" From 46a2748234cc4a41579942624fc6875c4de6b01a Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Tue, 26 May 2026 13:35:32 +0200 Subject: [PATCH 13/15] refactor(apify): extract shared actor helper, add section structure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add _run_actor_blocking() shared helper — eliminates duplicated .start() → log → wait_for_finish() → status check → dataset fetch pattern that existed in both _run_wcc_crawl and _run_website_content_crawler - Add _RAG_ACTOR / _WCC_ACTOR constants for repeated actor ID strings - Extract check_apify_api_key() so is_available() delegates to it, matching the pattern used by all other providers - Save run_id before reassigning run to avoid ambiguous variable reuse - Add status check in _run_actor_blocking: non-SUCCEEDED runs log a warning and return [] rather than silently returning empty results - search() simplified to call _run_actor_blocking directly - Add module docstring and # --- section separators matching Firecrawl style; add __init__.py module docstring Co-Authored-By: Claude Sonnet 4.6 --- plugins/web/apify/__init__.py | 7 ++ plugins/web/apify/provider.py | 181 +++++++++++++++++++++------------- 2 files changed, 119 insertions(+), 69 deletions(-) diff --git a/plugins/web/apify/__init__.py b/plugins/web/apify/__init__.py index e9310213f5a2..9b1b611e861f 100644 --- a/plugins/web/apify/__init__.py +++ b/plugins/web/apify/__init__.py @@ -1,7 +1,14 @@ +"""Apify web search + extract + crawl plugin — bundled, auto-loaded. + +Uses apify/rag-web-browser for search and apify/website-content-crawler +for both single-page extract and multi-page crawl. Requires APIFY_API_TOKEN. +""" + from __future__ import annotations from plugins.web.apify.provider import ApifyWebSearchProvider + def register(ctx) -> None: """Register the Apify provider with the plugin context.""" ctx.register_web_search_provider(ApifyWebSearchProvider()) \ No newline at end of file diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 1658ed09d944..2bbf71077275 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -1,3 +1,27 @@ +"""Apify web search + extract + crawl — plugin form. + +Subclasses :class:`agent.web_search_provider.WebSearchProvider`. Three +capabilities advertised: + +- ``supports_search()`` -> True (apify/rag-web-browser Actor) +- ``supports_extract()`` -> True (apify/website-content-crawler, single-page) +- ``supports_crawl()`` -> True (apify/website-content-crawler, multi-page) + +search() is sync; extract() and crawl() are async (each Actor call runs in +``asyncio.to_thread`` with an ``asyncio.wait_for`` guard). + +Config keys this provider responds to:: + + web: + search_backend: "apify" # explicit per-capability + extract_backend: "apify" # explicit per-capability + backend: "apify" # shared fallback + +Env vars:: + + APIFY_API_TOKEN=... # https://apify.com/account/integrations +""" + from __future__ import annotations import asyncio @@ -10,15 +34,19 @@ logger = logging.getLogger(__name__) +_RAG_ACTOR = "apify/rag-web-browser" +_WCC_ACTOR = "apify/website-content-crawler" -_APIFY_CLIENT_CLS_CACHE: Optional[type] = None - +# --------------------------------------------------------------------------- +# SDK lazy import + client cache +# --------------------------------------------------------------------------- +_APIFY_CLIENT_CLS_CACHE: Optional[type] = None def _load_apify_client_cls() -> type: - """Import and cache apify_client.ApifyClient (lazy, deferred on first use).""" + """Import and cache apify_client.ApifyClient (deferred to first use).""" global _APIFY_CLIENT_CLS_CACHE if _APIFY_CLIENT_CLS_CACHE is None: try: @@ -33,6 +61,11 @@ def _load_apify_client_cls() -> type: return _APIFY_CLIENT_CLS_CACHE +def check_apify_api_key() -> bool: + """Return True when APIFY_API_TOKEN is configured.""" + return bool(os.getenv("APIFY_API_TOKEN", "").strip()) + + def _get_apify_client() -> Any: """Return cached ApifyClient, constructing it from APIFY_API_TOKEN. @@ -68,86 +101,107 @@ def _reset_client_for_tests() -> None: _wt._apify_client_config = None -def _normalize_rag_search_results(items: List[Any], limit: int) -> List[Dict[str, Any]]: - """Normalize RAG Web Browser dataset items to the registry web search shape.""" - results: List[Dict[str, Any]] = [] - for item in items[:limit]: - if not isinstance(item, dict): - continue - sr = item.get("searchResult") or {} - if not isinstance(sr, dict): - sr = {} - title = sr.get("title") or item.get("title", "") - url = sr.get("url") or item.get("url", "") - description = sr.get("description") or item.get("markdown", "") - if description and len(description) > 500: - description = description[:500] - results.append({ - "title": title, - "url": url, - "description": description, - "position": len(results) + 1, - }) - return results +# --------------------------------------------------------------------------- +# Actor execution +# --------------------------------------------------------------------------- + + +def _run_actor_blocking(actor_id: str, run_input: Dict[str, Any]) -> List[Dict[str, Any]]: + """Start an Apify Actor, wait for completion, return dataset items. + + Blocking — intended to be called via asyncio.to_thread from async methods. + Returns [] when the Actor fails, is aborted, or produces no dataset. + Raises ValueError if the client is unconfigured. + """ + client = _get_apify_client() + started = client.actor(actor_id).start(run_input=run_input) + run_id = started.id + logger.info("Apify %s started — https://console.apify.com/actors/runs/%s", actor_id, run_id) + run = client.run(run_id).wait_for_finish() + if run is None: + logger.warning("Apify %s run %s: wait_for_finish returned None", actor_id, run_id) + return [] + if run.status != "SUCCEEDED": + logger.warning("Apify %s run %s finished with status %s", actor_id, run_id, run.status) + return [] + dataset_id = run.default_dataset_id + if not dataset_id: + return [] + return list(client.dataset(dataset_id).list_items().items) def _run_wcc_crawl(url: str, max_pages: int, max_depth: int) -> List[Dict[str, Any]]: - """Blocking multi-page crawl via apify/website-content-crawler. + """Multi-page crawl via apify/website-content-crawler. Intended to be called via asyncio.to_thread from crawl(). Returns raw dataset items (plain dicts). """ - client = _get_apify_client() - run = client.actor("apify/website-content-crawler").start( - run_input={ + return _run_actor_blocking( + _WCC_ACTOR, + { "startUrls": [{"url": url}], "maxCrawlPages": max_pages, "maxCrawlDepth": max_depth, "outputFormats": ["markdown"], "saveMarkdown": True, "saveHtml": False, - } + }, ) - logger.info("Apify WCC crawl started — https://console.apify.com/actors/runs/%s", run.id) - run = client.run(run.id).wait_for_finish() - if run is None: - return [] - logger.info("Apify WCC crawl: %s", run.status) - dataset_id = run.default_dataset_id - if not dataset_id: - return [] - return client.dataset(dataset_id).list_items().items def _run_website_content_crawler(url: str, output_formats: List[str]) -> Optional[Dict[str, Any]]: - """Blocking call to apify/website-content-crawler for a single URL. + """Single-page extract via apify/website-content-crawler. - Returns the first dataset item (single-page crawl), or None if no items. Intended to be called via asyncio.to_thread from extract(). + Returns the first dataset item, or None if the Actor returned no items. """ - client = _get_apify_client() - run = client.actor("apify/website-content-crawler").start( - run_input={ + items = _run_actor_blocking( + _WCC_ACTOR, + { "startUrls": [{"url": url}], "maxCrawlPages": 1, "outputFormats": output_formats, - } + }, ) - logger.info("Apify website-content-crawler started — https://console.apify.com/actors/runs/%s", run.id) - run = client.run(run.id).wait_for_finish() - if run is None: - return None - logger.info("Apify website-content-crawler: %s", run.status) - dataset_id = run.default_dataset_id - if not dataset_id: - return None - items = client.dataset(dataset_id).list_items().items if not items: return None item = items[0] return item if isinstance(item, dict) else None +# --------------------------------------------------------------------------- +# Response normalization +# --------------------------------------------------------------------------- + + +def _normalize_rag_search_results(items: List[Any], limit: int) -> List[Dict[str, Any]]: + """Normalize RAG Web Browser dataset items to the registry web search shape.""" + results: List[Dict[str, Any]] = [] + for item in items[:limit]: + if not isinstance(item, dict): + continue + sr = item.get("searchResult") or {} + if not isinstance(sr, dict): + sr = {} + title = sr.get("title") or item.get("title", "") + url = sr.get("url") or item.get("url", "") + description = sr.get("description") or item.get("markdown", "") + if description and len(description) > 500: + description = description[:500] + results.append({ + "title": title, + "url": url, + "description": description, + "position": len(results) + 1, + }) + return results + + +# --------------------------------------------------------------------------- +# Provider class +# --------------------------------------------------------------------------- + + class ApifyWebSearchProvider(WebSearchProvider): """Apify web search + extract + crawl provider. @@ -165,7 +219,7 @@ def display_name(self) -> str: return "Apify" def is_available(self) -> bool: - return bool(os.getenv("APIFY_API_TOKEN", "").strip()) + return check_apify_api_key() def supports_search(self) -> bool: return True @@ -188,25 +242,14 @@ def search(self, query: str, limit: int = 5) -> Dict[str, Any]: logger.info("Apify search: '%s' (limit=%d)", query, limit) try: - client = _get_apify_client() - run = client.actor("apify/rag-web-browser").start( - run_input={ + items = _run_actor_blocking( + _RAG_ACTOR, + { "query": query, "maxResults": limit, "requestTimeoutSecs": 60, - } + }, ) - logger.info("Apify rag-web-browser started — https://console.apify.com/actors/runs/%s", run.id) - run = client.run(run.id).wait_for_finish() - if run is None: - return {"success": False, "error": "Apify actor run returned no result"} - logger.info("Apify rag-web-browser: %s", run.status) - - dataset_id = run.default_dataset_id - if not dataset_id: - return {"success": False, "error": "Apify run missing defaultDatasetId"} - - items = client.dataset(dataset_id).list_items().items web_results = _normalize_rag_search_results(items, limit) logger.info("Apify search: found %d results", len(web_results)) return {"success": True, "data": {"web": web_results}} From 27e3744cf601cb99278201623b1f73d9322d7e39 Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Thu, 28 May 2026 13:06:13 +0200 Subject: [PATCH 14/15] fix: PR fixes --- plugins/web/apify/provider.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index 2bbf71077275..b65c601936c0 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -110,7 +110,8 @@ def _run_actor_blocking(actor_id: str, run_input: Dict[str, Any]) -> List[Dict[s """Start an Apify Actor, wait for completion, return dataset items. Blocking — intended to be called via asyncio.to_thread from async methods. - Returns [] when the Actor fails, is aborted, or produces no dataset. + Returns [] when the Actor succeeds but produces no dataset items. + Raises RuntimeError on non-SUCCEEDED status so callers can surface the failure. Raises ValueError if the client is unconfigured. """ client = _get_apify_client() @@ -119,11 +120,15 @@ def _run_actor_blocking(actor_id: str, run_input: Dict[str, Any]) -> List[Dict[s logger.info("Apify %s started — https://console.apify.com/actors/runs/%s", actor_id, run_id) run = client.run(run_id).wait_for_finish() if run is None: - logger.warning("Apify %s run %s: wait_for_finish returned None", actor_id, run_id) - return [] + raise RuntimeError( + f"Apify {actor_id} run {run_id}: wait_for_finish returned None — " + f"check https://console.apify.com/actors/runs/{run_id}" + ) if run.status != "SUCCEEDED": - logger.warning("Apify %s run %s finished with status %s", actor_id, run_id, run.status) - return [] + raise RuntimeError( + f"Apify {actor_id} run {run_id} finished with status {run.status} — " + f"check https://console.apify.com/actors/runs/{run_id} for details" + ) dataset_id = run.default_dataset_id if not dataset_id: return [] @@ -418,7 +423,10 @@ async def crawl(self, url: str, **kwargs: Any) -> Dict[str, Any]: return {"results": [{"url": url, "title": "", "content": "", "error": "Interrupted"}]} instructions = kwargs.get("instructions") - limit = int(kwargs.get("limit", 20)) + try: + limit = int(kwargs.get("limit", 20)) + except (TypeError, ValueError): + limit = 20 depth_raw = kwargs.get("depth") if depth_raw == "basic": From 50e4e2a7fe2b365c4b1ccc01b0009ce83d5410be Mon Sep 17 00:00:00 2001 From: JanHranicky Date: Thu, 4 Jun 2026 10:57:36 +0200 Subject: [PATCH 15/15] refactor(apify): extract shared client to tools/apify_client.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Moves _get_apify_client / check_apify_api_key / reset_client_for_tests out of the web-search provider into a standalone module so the upcoming Actor-execution tools (apify_tool.py) can import the client without depending on the provider — enabling both to be submitted as independent PRs targeting main. Co-Authored-By: Claude Sonnet 4.6 --- plugins/web/apify/provider.py | 65 ++--------------------------------- tools/apify_client.py | 62 +++++++++++++++++++++++++++++++++ tools/web_tools.py | 2 -- 3 files changed, 64 insertions(+), 65 deletions(-) create mode 100644 tools/apify_client.py diff --git a/plugins/web/apify/provider.py b/plugins/web/apify/provider.py index b65c601936c0..a0cf39566de2 100644 --- a/plugins/web/apify/provider.py +++ b/plugins/web/apify/provider.py @@ -26,10 +26,10 @@ import asyncio import logging -import os from typing import Any, Dict, List, Optional from agent.web_search_provider import WebSearchProvider +from tools.apify_client import check_apify_api_key, get_apify_client from tools.website_policy import check_website_access logger = logging.getLogger(__name__) @@ -37,68 +37,7 @@ _RAG_ACTOR = "apify/rag-web-browser" _WCC_ACTOR = "apify/website-content-crawler" - -# --------------------------------------------------------------------------- -# SDK lazy import + client cache -# --------------------------------------------------------------------------- - -_APIFY_CLIENT_CLS_CACHE: Optional[type] = None - - -def _load_apify_client_cls() -> type: - """Import and cache apify_client.ApifyClient (deferred to first use).""" - global _APIFY_CLIENT_CLS_CACHE - if _APIFY_CLIENT_CLS_CACHE is None: - try: - from tools.lazy_deps import ensure as _lazy_ensure - _lazy_ensure("search.apify", prompt=False) - except ImportError: - pass - except Exception as exc: # noqa: BLE001 - raise ImportError(str(exc)) - from apify_client import ApifyClient - _APIFY_CLIENT_CLS_CACHE = ApifyClient - return _APIFY_CLIENT_CLS_CACHE - - -def check_apify_api_key() -> bool: - """Return True when APIFY_API_TOKEN is configured.""" - return bool(os.getenv("APIFY_API_TOKEN", "").strip()) - - -def _get_apify_client() -> Any: - """Return cached ApifyClient, constructing it from APIFY_API_TOKEN. - - Raises ValueError when APIFY_API_TOKEN is not set. - Cache stored on tools.web_tools._apify_client so tests can reset it via - ``tools.web_tools._apify_client = None``. - """ - import tools.web_tools as _wt - - api_token = os.getenv("APIFY_API_TOKEN", "").strip() - if not api_token: - raise ValueError( - "Apify tools are not configured. " - "Set APIFY_API_TOKEN (get one at https://apify.com/account/integrations)." - ) - - client_config = ("direct", api_token) - cached = getattr(_wt, "_apify_client", None) - cached_config = getattr(_wt, "_apify_client_config", None) - if cached is not None and cached_config == client_config: - return cached - - ApifyClient = _load_apify_client_cls() - _wt._apify_client = ApifyClient(token=api_token) - _wt._apify_client_config = client_config - return _wt._apify_client - - -def _reset_client_for_tests() -> None: - """Drop cached Apify client so tests can re-instantiate cleanly.""" - import tools.web_tools as _wt - _wt._apify_client = None - _wt._apify_client_config = None +_get_apify_client = get_apify_client # alias kept for tests that patch this name # --------------------------------------------------------------------------- diff --git a/tools/apify_client.py b/tools/apify_client.py new file mode 100644 index 000000000000..62e14ce09d91 --- /dev/null +++ b/tools/apify_client.py @@ -0,0 +1,62 @@ +"""Shared Apify SDK client — lazy import, token validation, and cache. + +Extracted so both the web-search provider (plugins/web/apify/provider.py) and +the Actor execution tools (tools/apify_tool.py) can import the client without +depending on each other. +""" +from __future__ import annotations + +import os +from typing import Any, Optional + +_CLIENT_CLS: Optional[type] = None +_CLIENT: Optional[Any] = None +_CLIENT_CONFIG: Optional[Any] = None + + +def _load_client_cls() -> type: + global _CLIENT_CLS + if _CLIENT_CLS is None: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("search.apify", prompt=False) + except ImportError: + pass + except Exception as exc: # noqa: BLE001 + raise ImportError(str(exc)) + from apify_client import ApifyClient + _CLIENT_CLS = ApifyClient + return _CLIENT_CLS + + +def check_apify_api_key() -> bool: + """Return True when APIFY_API_TOKEN is configured.""" + return bool(os.getenv("APIFY_API_TOKEN", "").strip()) + + +def get_apify_client() -> Any: + """Return a cached ApifyClient built from APIFY_API_TOKEN. + + Raises ValueError when the token is not set. + """ + global _CLIENT, _CLIENT_CONFIG + api_token = os.getenv("APIFY_API_TOKEN", "").strip() + if not api_token: + raise ValueError( + "Apify tools are not configured. " + "Set APIFY_API_TOKEN (get one at https://apify.com/account/integrations)." + ) + client_config = ("direct", api_token) + if _CLIENT is not None and _CLIENT_CONFIG == client_config: + return _CLIENT + _CLIENT = _load_client_cls()(token=api_token) + _CLIENT_CONFIG = client_config + return _CLIENT + + +def reset_client_for_tests() -> None: + """Drop cached client so tests can re-instantiate cleanly.""" + global _CLIENT, _CLIENT_CONFIG, _CLIENT_CLS + _CLIENT = None + _CLIENT_CONFIG = None + _CLIENT_CLS = None diff --git a/tools/web_tools.py b/tools/web_tools.py index ff38b5cebd5b..ac9f00b6275a 100644 --- a/tools/web_tools.py +++ b/tools/web_tools.py @@ -96,8 +96,6 @@ _parallel_client: Optional[Any] = None _async_parallel_client: Optional[Any] = None _exa_client: Optional[Any] = None -_apify_client: Optional[Any] = None -_apify_client_config: Optional[Any] = None from agent.auxiliary_client import ( async_call_llm,