diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md index 44e3e8c71a..8a8f2bb4f0 100644 --- a/docs/en/api/02-resources.md +++ b/docs/en/api/02-resources.md @@ -53,7 +53,7 @@ OpenViking supports various resource types, categorized by functionality: | Type | Resource Name | Description | |------|---------------|-------------| -| Single page / recursive crawl | `https://host/path` | By default only the entry page is fetched. Set `args.depth > 0` to crawl same-host links breadth-first; `args.max_pages` only bounds how many pages are collected. Each page is extracted to Markdown with trafilatura, with an automatic Playwright fallback for SPAs whose static HTML carries no body text. Supported `args`: `depth`, `max_pages`, `include_paths`, `exclude_paths`, `allow_external_links`, `skip_download_links`. Download links discovered on pages are skipped by default (`skip_download_links=true`) to avoid importing sidecar files such as `llms.txt`; set it to `false` to download same-host file links and count them toward `max_pages`. `include_paths`/`exclude_paths` use **path-prefix** matching (e.g. `/docs/` matches only paths starting with `/docs/`, never substrings like `/blog/docs-tips`). | +| Single page / recursive crawl | `https://host/path` | By default only the entry page is fetched. Set `args.depth > 0` to crawl same-host links breadth-first; `args.max_pages` only bounds how many pages are collected. Each page is extracted to Markdown with trafilatura. Supported `args`: `depth`, `max_pages`, `include_paths`, `exclude_paths`, `allow_external_links`, `skip_download_links`. Download links discovered on pages are skipped by default (`skip_download_links=true`) to avoid importing sidecar files such as `llms.txt`; set it to `false` to download same-host file links and count them toward `max_pages`. `include_paths`/`exclude_paths` use **path-prefix** matching (e.g. `/docs/` matches only paths starting with `/docs/`, never substrings like `/blog/docs-tips`). | > Routing: sitemap-looking URLs (`https://host/sitemap.xml`, `https://host/feed.xml`, `*.atom`, ...) and explicit `args.site=true` are delegated to the whole-website ingestion below; Git hosting URLs such as `https://github.com/{org}/{repo}` are delegated to the Code section above. diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md index f5b66de842..5f70e83319 100644 --- a/docs/zh/api/02-resources.md +++ b/docs/zh/api/02-resources.md @@ -47,7 +47,7 @@ OpenViking 支持多种资源类型，按照功能分类如下： 网页类（递归网页爬虫） | 类型 | 资源名 | 说明 | |------|--------|------| -| 单页 / 递归抓取 | `https://host/path` | 默认仅抓入口页；设置 `args.depth > 0` 后，沿同域链接 BFS 递归展开，`args.max_pages` 只限制最多收集的页面数。每页用 trafilatura 抽成 Markdown，对 SPA 等静态 HTML 拿不到正文的站点自动降级到 Playwright 渲染。可选 `args`：`depth`、`max_pages`、`include_paths`、`exclude_paths`、`allow_external_links`、`skip_download_links`。页面中发现的下载链接默认跳过（`skip_download_links=true`），避免导入 `llms.txt` 等 sidecar 文件造成重复；设为 `false` 时会下载同域文件链接，并计入 `max_pages`。`include_paths`/`exclude_paths` 按**路径前缀**匹配（例如 `/docs/` 仅匹配以 `/docs/` 开头的路径，不会误命中 `/blog/docs-tips`）。| +| 单页 / 递归抓取 | `https://host/path` | 默认仅抓入口页；设置 `args.depth > 0` 后，沿同域链接 BFS 递归展开，`args.max_pages` 只限制最多收集的页面数。每页用 trafilatura 抽成 Markdown。可选 `args`：`depth`、`max_pages`、`include_paths`、`exclude_paths`、`allow_external_links`、`skip_download_links`。页面中发现的下载链接默认跳过（`skip_download_links=true`），避免导入 `llms.txt` 等 sidecar 文件造成重复；设为 `false` 时会下载同域文件链接，并计入 `max_pages`。`include_paths`/`exclude_paths` 按**路径前缀**匹配（例如 `/docs/` 仅匹配以 `/docs/` 开头的路径，不会误命中 `/blog/docs-tips`）。| > 路由说明：`https://host/sitemap.xml`、`https://host/feed.xml`、`*.atom` 等 sitemap-looking URL 和显式 `args.site=true` 让出给下表的整站导入；`https://github.com/{org}/{repo}` 等 Git 托管平台 URL 让出给上文的代码导入。 diff --git a/openviking/parse/accessors/web_crawler/config.py b/openviking/parse/accessors/web_crawler/config.py index bcab21bab4..d32f9e9897 100644 --- a/openviking/parse/accessors/web_crawler/config.py +++ b/openviking/parse/accessors/web_crawler/config.py @@ -22,8 +22,6 @@ class CrawlConfig: retry_times: int = 2 max_links_per_page: int = 500 max_html_bytes: int = 10 * 1024 * 1024 - fallback_playwright: bool = True - playwright_timeout: float = 30.0 request_validator: Optional[Callable[[str], None]] = None def __post_init__(self) -> None: @@ -43,5 +41,3 @@ def __post_init__(self) -> None: raise ValueError("max_links_per_page must be >= 1.") if self.max_html_bytes < 1: raise ValueError("max_html_bytes must be >= 1.") - if self.playwright_timeout <= 0: - raise ValueError("playwright_timeout must be > 0 for recursive web crawling.") diff --git a/openviking/parse/accessors/web_crawler/models.py b/openviking/parse/accessors/web_crawler/models.py index 771bfb8779..863a472d40 100644 --- a/openviking/parse/accessors/web_crawler/models.py +++ b/openviking/parse/accessors/web_crawler/models.py @@ -31,4 +31,3 @@ class CrawlResult: total_downloads: int = 0 total_skipped: int = 0 total_failed: int = 0 - fallback_rendered: int = 0 diff --git a/openviking/parse/accessors/web_crawler/playwright_renderer.py b/openviking/parse/accessors/web_crawler/playwright_renderer.py deleted file mode 100644 index b3063e02fe..0000000000 --- a/openviking/parse/accessors/web_crawler/playwright_renderer.py +++ /dev/null @@ -1,191 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -"""Playwright renderer used only as low-content fallback.""" - -import asyncio -from collections.abc import Callable -from dataclasses import dataclass -from typing import Optional - -from openviking.parse.accessors.web_crawler.render_heuristics import ( - CHALLENGE_MARKERS, - SHELL_VISIBLE_TEXT_CHARS, -) - - -PLAYWRIGHT_PACKAGE_INSTALL_HINT = ( - "Playwright fallback was needed, but the Python package is not installed. " - "Install it with `pip install playwright` and install Chromium with " - "`python -m playwright install chromium`." -) -PLAYWRIGHT_CHROMIUM_INSTALL_HINT = ( - "Playwright fallback was needed, but Chromium is not installed or cannot be " - "launched. Run `python -m playwright install chromium` and retry." -) - -# Cap the networkidle wait: pages with continuous background activity (e.g. the -# GraphiQL playground, polling/websocket apps) never go idle and would otherwise -# block until the full render timeout. Content is usually ready right after -# domcontentloaded, and ``_wait_past_challenge`` handles late-arriving text. -_NETWORKIDLE_TIMEOUT_MS = 8000 - - -@dataclass -class RenderResult: - html: str = "" - status_code: int = 0 - final_url: str = "" - content_type: str = "" - error: Optional[str] = None - - @property - def is_success(self) -> bool: - return 200 <= self.status_code < 400 and self.error is None - - -class PlaywrightRenderer: - def __init__(self, request_validator: Optional[Callable[[str], None]] = None) -> None: - self._request_validator = request_validator - self._playwright_manager = None - self._browser = None - self._browser_lock = asyncio.Lock() - - async def render(self, url: str, timeout: float) -> RenderResult: - page = None - try: - if self._request_validator: - self._request_validator(url) - browser = await self._get_browser() - page = await browser.new_page(accept_downloads=False) - - if self._request_validator: - - async def _validate_route(route): - await self._guard_route(route, self._request_validator) - - await page.route("**/*", _validate_route) - - response = await page.goto( - url, - wait_until="domcontentloaded", - timeout=timeout * 1000, - ) - try: - await page.wait_for_load_state("networkidle", timeout=_NETWORKIDLE_TIMEOUT_MS) - except Exception: - pass - await self._wait_past_challenge(page, timeout * 1000) - html = await self._read_content(page) - final_url = page.url - if self._request_validator: - self._request_validator(final_url) - return RenderResult( - html=html, - status_code=response.status if response else 200, - final_url=final_url, - content_type=response.headers.get("content-type", "") if response else "", - ) - except ImportError: - return RenderResult(final_url=url, error=PLAYWRIGHT_PACKAGE_INSTALL_HINT) - except Exception as exc: - error = str(exc) - if ( - "Executable doesn't exist" in error - or "playwright install" in error - or "BrowserType.launch" in error - ): - error = PLAYWRIGHT_CHROMIUM_INSTALL_HINT - return RenderResult(final_url=url, error=error) - finally: - if page: - try: - await page.close() - except Exception: - pass - - @staticmethod - async def _guard_route(route, request_validator) -> None: - """SSRF guard for sub-resource requests. - - Block requests to disallowed hosts (e.g. private-network probe - endpoints) by aborting them, but never let a blocked sub-resource - fail the whole page render. Only the main document URL and the final - URL (validated in ``render``) gate the overall result. - """ - try: - request_validator(route.request.url) - except Exception: - await route.abort() - return - await route.continue_() - - @staticmethod - async def _wait_past_challenge(page, timeout_ms: float, poll_ms: int = 500) -> None: - """Wait out JS anti-bot interstitials (e.g. "Please wait..."). - - These challenge pages run a CPU-bound JS proof-of-work and then - auto-redirect to the real content without further network activity, - so ``networkidle`` returns while the interstitial is still showing. - Poll the body text until real content appears or we run out of time. - """ - import time - - deadline = time.monotonic() + max(timeout_ms, 0) / 1000 - while True: - try: - body = (await page.inner_text("body")).strip() - except Exception: - body = "" - lowered = body.lower() - looks_like_challenge = ( - not body - or len(body) < SHELL_VISIBLE_TEXT_CHARS - or any(marker in lowered for marker in CHALLENGE_MARKERS) - ) - if not looks_like_challenge or time.monotonic() >= deadline: - return - await page.wait_for_timeout(poll_ms) - - @staticmethod - async def _read_content(page) -> str: - """Read page HTML, retrying through in-flight client-side navigation. - - Heavy SPAs (client-side redirects, late hydration) can still be - navigating when we first ask for content, which raises "page is - navigating and changing the content". Retry until the page settles. - """ - last_exc = None - for _ in range(6): - try: - return await page.content() - except Exception as exc: - if "navigating and changing the content" not in str(exc): - raise - last_exc = exc - try: - await page.wait_for_load_state("load", timeout=5000) - except Exception: - pass - await page.wait_for_timeout(400) - if last_exc: - raise last_exc - return await page.content() - - async def close(self) -> None: - async with self._browser_lock: - if self._browser: - await self._browser.close() - self._browser = None - if self._playwright_manager: - await self._playwright_manager.stop() - self._playwright_manager = None - - async def _get_browser(self): - async with self._browser_lock: - if self._browser is None or not self._browser.is_connected(): - from playwright.async_api import async_playwright - - if self._playwright_manager is None: - self._playwright_manager = await async_playwright().start() - self._browser = await self._playwright_manager.chromium.launch(headless=True) - return self._browser diff --git a/openviking/parse/accessors/web_crawler/render_heuristics.py b/openviking/parse/accessors/web_crawler/render_heuristics.py deleted file mode 100644 index 91283e54f0..0000000000 --- a/openviking/parse/accessors/web_crawler/render_heuristics.py +++ /dev/null @@ -1,83 +0,0 @@ -# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd. -# SPDX-License-Identifier: AGPL-3.0 -"""Heuristics for deciding whether a page likely needs browser rendering.""" - -import re - -from bs4 import BeautifulSoup - - -_SPA_EMPTY_PATTERNS = ( - "You need to enable JavaScript to run this app.", - "This app works best with JavaScript enabled.", - "Please enable JavaScript to continue.", - "JavaScript is required to use this application.", - "Enable JavaScript to view this page.", -) - -_MIN_VISIBLE_TEXT_CHARS = 200 -SHELL_VISIBLE_TEXT_CHARS = 40 - -# Text markers of JS anti-bot interstitials that briefly show before the real -# page loads (e.g. volcengine's proof-of-work gate renders only "Please wait..."). -CHALLENGE_MARKERS = ( - "please wait", - "checking your browser", - "verifying you are human", - "just a moment", - "attention required", -) - - -def should_render_with_playwright(html: str) -> bool: - """Return True when static HTML looks like a client-rendered shell.""" - html = html or "" - html_lower = html.lower() - if any(pattern.lower() in html_lower for pattern in _SPA_EMPTY_PATTERNS): - return True - visible_len = visible_body_text_len(html) - if visible_len < SHELL_VISIBLE_TEXT_CHARS: - return True - # __NEXT_DATA__ only signals a Next.js app; most such pages are SSR/SSG and - # already ship full body text in the static HTML. Only render when the - # static body is also too thin to be the real content. - if "__next_data__" in html_lower and visible_len < _MIN_VISIBLE_TEXT_CHARS: - return True - if re.search(r'id=["\'](?:root|app)["\']', html_lower) and visible_len < _MIN_VISIBLE_TEXT_CHARS: - return True - if html_lower.count("= 5 and visible_len < _MIN_VISIBLE_TEXT_CHARS: - return True - return False - - -def looks_like_unrendered_page(html: str) -> bool: - """Return True when HTML is an empty shell or an anti-bot challenge page. - - Used to reject content that must not be stored as real page text: the - static SPA shell or a JS interstitial such as "Please wait...". - """ - try: - html = html or "" - soup = BeautifulSoup(html, "html.parser") - body = soup.body or soup - for el in body(["script", "style", "noscript"]): - el.decompose() - text = body.get_text(strip=True) - lowered = text.lower() - if any(marker in lowered for marker in CHALLENGE_MARKERS): - return True - return len(text) < SHELL_VISIBLE_TEXT_CHARS - except Exception: - return True - - -def visible_body_text_len(html: str) -> int: - """Approximate visible text length without running page JavaScript.""" - try: - soup = BeautifulSoup(html or "", "html.parser") - body = soup.body or soup - for el in body(["script", "style", "noscript"]): - el.decompose() - return len(body.get_text(strip=True)) - except Exception: - return 0 diff --git a/openviking/parse/accessors/web_crawler/scrapy_spider.py b/openviking/parse/accessors/web_crawler/scrapy_spider.py index 2b72e70e9d..9cf344242c 100644 --- a/openviking/parse/accessors/web_crawler/scrapy_spider.py +++ b/openviking/parse/accessors/web_crawler/scrapy_spider.py @@ -2,24 +2,17 @@ # SPDX-License-Identifier: AGPL-3.0 """Scrapy spider for recursive web resource import.""" -import asyncio import os from collections.abc import Iterator from urllib.parse import urljoin, urlparse import scrapy from parsel import Selector -from scrapy import signals from scrapy.exceptions import CloseSpider from openviking.parse.accessors.http_accessor import URLType, URLTypeDetector from openviking.parse.accessors.web_crawler.config import CrawlConfig from openviking.parse.accessors.web_crawler.models import CrawledDownload, CrawledPage -from openviking.parse.accessors.web_crawler.playwright_renderer import PlaywrightRenderer -from openviking.parse.accessors.web_crawler.render_heuristics import ( - looks_like_unrendered_page, - should_render_with_playwright, -) _DOWNLOAD_URL_TYPES = frozenset( @@ -78,23 +71,9 @@ def __init__( self.config = config self.collector = collector self.download_collector = download_collector - self.renderer = PlaywrightRenderer(config.request_validator) self.root_host = urlparse(root_url).netloc.lower() self._success_count = 0 self._seen_download_urls: set[str] = set() - self._render_semaphore: asyncio.Semaphore | None = None - - @classmethod - def from_crawler(cls, crawler, *args, **kwargs): - spider = super().from_crawler(crawler, *args, **kwargs) - crawler.signals.connect(spider._on_spider_closed, signal=signals.spider_closed) - return spider - - async def _on_spider_closed(self, spider, reason): - try: - await asyncio.wait_for(self.renderer.close(), timeout=5.0) - except Exception: - pass def _success_at_limit(self) -> bool: return 0 < self.config.max_pages <= self._success_count @@ -128,39 +107,6 @@ async def parse(self, response): return page_html = response.text - page_source = "scrapy_static" - needs_render = self.config.fallback_playwright and should_render_with_playwright( - response.text - ) - try: - if needs_render: - if self._render_semaphore is None: - self._render_semaphore = asyncio.Semaphore(self.config.concurrency) - async with self._render_semaphore: - if self._success_at_limit(): - return - rendered = await self.renderer.render( - final_url, self.config.playwright_timeout - ) - if rendered.is_success and rendered.html: - final_url = rendered.final_url or final_url - page_html = rendered.html - page_source = "playwright" - elif rendered.error: - self._add_failed(response.url, final_url, depth, rendered.error) - return - except Exception as exc: - self._add_failed(response.url, final_url, depth, str(exc)) - return - - if needs_render and looks_like_unrendered_page(page_html): - self._add_failed( - response.url, - final_url, - depth, - "page did not render real content (empty shell or anti-bot challenge page)", - ) - return if self._stop_if_success_at_limit(): return @@ -170,7 +116,7 @@ async def parse(self, response): final_url=final_url, depth=depth, html=page_html, - source=page_source, + source="scrapy_static", ) ) self._success_count += 1 diff --git a/openviking/parse/accessors/web_crawler/web_crawler.py b/openviking/parse/accessors/web_crawler/web_crawler.py index 39ee7345d9..05bb2e46e0 100644 --- a/openviking/parse/accessors/web_crawler/web_crawler.py +++ b/openviking/parse/accessors/web_crawler/web_crawler.py @@ -146,6 +146,4 @@ def _build_result(pages: list[CrawledPage], downloads: list[CrawledDownload]) -> result.total_skipped += 1 else: result.total_failed += 1 - if page.source == "playwright": - result.fallback_rendered += 1 return result diff --git a/openviking/parse/accessors/web_importer.py b/openviking/parse/accessors/web_importer.py index ea3e61acdc..06a0995872 100644 --- a/openviking/parse/accessors/web_importer.py +++ b/openviking/parse/accessors/web_importer.py @@ -13,10 +13,6 @@ from openviking.parse.accessors.http_accessor import HTTPAccessor from openviking.parse.accessors.web_crawler import CrawlConfig, ScrapyWebCrawler -from openviking.parse.accessors.web_crawler.playwright_renderer import ( - PLAYWRIGHT_CHROMIUM_INSTALL_HINT, - PLAYWRIGHT_PACKAGE_INSTALL_HINT, -) from openviking.parse.accessors.web_feed_accessor import ( FeedEntry, _dedup_relpath, @@ -25,24 +21,10 @@ url_to_relpath, ) from openviking_cli.exceptions import InvalidArgumentError -from openviking_cli.utils.logger import get_logger - -logger = get_logger(__name__) DEPTH_UNLIMITED = -1 MAX_PAGES_UNLIMITED = -1 -# Actionable install hints the renderer records on CrawledPage.error when the -# Playwright fallback is needed but unavailable. These are surfaced to the user -# even for non-entry pages, which would otherwise only show up as a failure -# count. -_RENDER_INSTALL_HINTS = frozenset( - { - PLAYWRIGHT_PACKAGE_INSTALL_HINT, - PLAYWRIGHT_CHROMIUM_INSTALL_HINT, - } -) - @dataclass(frozen=True) class WebImportOptions: @@ -111,19 +93,7 @@ async def import_to_directory( success_pages = self._dedupe_success_pages(crawl_result.pages) if not any(page.depth == 0 for page in success_pages): detail = self._entry_failure_detail(crawl_result.pages) - message = f"Failed to fetch entry page: {root_url}" - if detail: - message = f"{message} ({detail})" - raise RuntimeError(message) - - render_hints = self._render_install_hints(crawl_result.pages) - if render_hints: - logger.warning( - "Some pages could not be rendered during web import of %s and were " - "skipped: %s", - root_url, - " ".join(render_hints), - ) + raise RuntimeError(_entry_failure_message(root_url, detail)) temp_root = Path(tempfile.mkdtemp(prefix="ov_web_")) temp_dir = temp_root / _host_name(root_url) @@ -155,35 +125,16 @@ async def import_to_directory( "page_count": len(success_pages), "download_count": len(downloaded_files), "crawl_result": _crawl_summary(crawl_result), - "render_hints": render_hints, "original_filename": _host_name(root_url), }, ) - @staticmethod - def _render_install_hints(pages) -> list[str]: - """Collect actionable render-install hints from failed pages (any depth). - - Child SPA pages that fail because Playwright is unavailable only record - the hint on ``CrawledPage.error``; the entry page still succeeds, so the - caller would otherwise see the hint nowhere. Surface each distinct hint - once so the user can act on it. - """ - hints: list[str] = [] - for page in pages: - if page.status == "success" or not page.error: - continue - if page.error in _RENDER_INSTALL_HINTS and page.error not in hints: - hints.append(page.error) - return hints - @staticmethod def _entry_failure_detail(pages) -> str: """Return the failure reason of the entry page, if one was recorded. - The renderer surfaces actionable hints (e.g. missing Playwright - install) via ``CrawledPage.error``; without this the caller only sees - the generic "Failed to fetch entry page" message. + Without this the caller only sees the generic "Failed to fetch entry + page" message instead of the underlying error (e.g. an SSRF rejection). """ for page in pages: if page.depth == 0 and page.status != "success" and page.error: @@ -269,10 +220,28 @@ def _crawl_summary(crawl_result: Any) -> Dict[str, Any]: "total_downloads": getattr(crawl_result, "total_downloads", 0), "total_failed": crawl_result.total_failed, "total_skipped": crawl_result.total_skipped, - "fallback_rendered": crawl_result.fallback_rendered, } +def _entry_failure_message(root_url: str, detail: str) -> str: + """Build a human-readable error for an entry page that could not be fetched. + + The most common cause is the site's robots.txt disallowing crawlers, which + Scrapy reports with the terse "Forbidden by robots.txt". Turn that into a + short compliance-oriented hint instead. + """ + if detail and "robots.txt" in detail.lower(): + return ( + "This URL cannot be imported for compliance reasons " + "(the site disallows crawlers). Please save the page and import it " + "as a local file instead." + ) + message = f"Failed to fetch entry page: {root_url}" + if detail: + message = f"{message} ({detail})" + return message + + def _normalize_page_url(url: str) -> str: parts = urlsplit(url or "") scheme = parts.scheme.lower() diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py index a169cd2763..452067842e 100644 --- a/openviking/parse/parsers/html.py +++ b/openviking/parse/parsers/html.py @@ -24,14 +24,6 @@ logger = __import__("openviking_cli.utils.logger").utils.logger.get_logger(__name__) -_SPA_EMPTY_PATTERNS = ( - "You need to enable JavaScript to run this app.", - "This app works best with JavaScript enabled.", - "Please enable JavaScript to continue.", - "JavaScript is required to use this application.", - "Enable JavaScript to view this page.", -) - class HTMLParser(BaseParser): """ @@ -133,10 +125,33 @@ def _html_to_markdown(self, html: str, base_url: str = "") -> str: content = self._extract_markdown(html, base_url or "") title = self._extract_title(html, base_url or "") content = self._clean_markdown(content) + if not content: + content = self._extract_noscript_notice(html) if title and title not in content: content = f"# {title}\n\n{content}" if content else f"# {title}" return content + @staticmethod + def _extract_noscript_notice(html: str) -> str: + """Return the