From d5c16ba1ab94369619382a5fc5815cde724cc7f6 Mon Sep 17 00:00:00 2001
From: sponge225 <1670519171@qq.com>
Date: Mon, 6 Jul 2026 17:28:39 +0800
Subject: [PATCH 1/2] refactor(web-crawler): remove Playwright rendering, keep
 static SPA shells

Drop the Playwright-based fallback rendering path so SPA pages are stored
as their static shell HTML instead of being rendered headlessly. SPA
shells now surface the <noscript> notice (e.g. "You need to enable
JavaScript to run this app.") rather than producing an empty document,
and robots.txt-blocked entry pages get a human-readable error message.

- Delete playwright_renderer.py, render_heuristics.py and their tests
- Strip fallback_playwright/playwright_timeout config, fallback_rendered
  counter, and render-hint plumbing from the crawler and web importer
- HTMLParser: drop the SPA-empty-pattern stripping and fall back to
  <noscript> text when trafilatura extracts nothing
- Humanize the robots.txt entry-failure message
- Migrate the orphaned _convert_to_raw_url tests to HTTPAccessor (the
  method moved there in an earlier reorg) and remove the stale file
---
 docs/en/api/02-resources.md                   |   2 +-
 docs/zh/api/02-resources.md                   |   2 +-
 .../parse/accessors/web_crawler/config.py     |   4 -
 .../parse/accessors/web_crawler/models.py     |   1 -
 .../web_crawler/playwright_renderer.py        | 191 ------------------
 .../web_crawler/render_heuristics.py          |  83 --------
 .../accessors/web_crawler/scrapy_spider.py    |  56 +----
 .../accessors/web_crawler/web_crawler.py      |   2 -
 openviking/parse/accessors/web_importer.py    |  75 ++-----
 openviking/parse/parsers/html.py              |  33 ++-
 .../accessors/web_crawler/test_config.py      |   4 -
 .../web_crawler/test_playwright_renderer.py   |  87 --------
 .../web_crawler/test_render_heuristics.py     |  63 ------
 .../web_crawler/test_scrapy_spider.py         |  13 +-
 tests/parse/parsers/test_html.py              |  31 ++-
 tests/parse/test_html_parser_utils.py         |  53 -----
 tests/unit/test_accessors_http.py             |  61 ++++++
 tests/unit/test_web_importer.py               |  51 ++---
 18 files changed, 159 insertions(+), 653 deletions(-)
 delete mode 100644 openviking/parse/accessors/web_crawler/playwright_renderer.py
 delete mode 100644 openviking/parse/accessors/web_crawler/render_heuristics.py
 delete mode 100644 tests/parse/accessors/web_crawler/test_playwright_renderer.py
 delete mode 100644 tests/parse/accessors/web_crawler/test_render_heuristics.py
 delete mode 100644 tests/parse/test_html_parser_utils.py

diff --git a/docs/en/api/02-resources.md b/docs/en/api/02-resources.md
index 44e3e8c71a..8a8f2bb4f0 100644
--- a/docs/en/api/02-resources.md
+++ b/docs/en/api/02-resources.md
@@ -53,7 +53,7 @@ OpenViking supports various resource types, categorized by functionality:
 
 | Type | Resource Name | Description |
 |------|---------------|-------------|
-| Single page / recursive crawl | `https://host/path` | By default only the entry page is fetched. Set `args.depth > 0` to crawl same-host links breadth-first; `args.max_pages` only bounds how many pages are collected. Each page is extracted to Markdown with trafilatura, with an automatic Playwright fallback for SPAs whose static HTML carries no body text. Supported `args`: `depth`, `max_pages`, `include_paths`, `exclude_paths`, `allow_external_links`, `skip_download_links`. Download links discovered on pages are skipped by default (`skip_download_links=true`) to avoid importing sidecar files such as `llms.txt`; set it to `false` to download same-host file links and count them toward `max_pages`. `include_paths`/`exclude_paths` use **path-prefix** matching (e.g. `/docs/` matches only paths starting with `/docs/`, never substrings like `/blog/docs-tips`). |
+| Single page / recursive crawl | `https://host/path` | By default only the entry page is fetched. Set `args.depth > 0` to crawl same-host links breadth-first; `args.max_pages` only bounds how many pages are collected. Each page is extracted to Markdown with trafilatura. Supported `args`: `depth`, `max_pages`, `include_paths`, `exclude_paths`, `allow_external_links`, `skip_download_links`. Download links discovered on pages are skipped by default (`skip_download_links=true`) to avoid importing sidecar files such as `llms.txt`; set it to `false` to download same-host file links and count them toward `max_pages`. `include_paths`/`exclude_paths` use **path-prefix** matching (e.g. `/docs/` matches only paths starting with `/docs/`, never substrings like `/blog/docs-tips`). |
 
 > Routing: sitemap-looking URLs (`https://host/sitemap.xml`, `https://host/feed.xml`, `*.atom`, ...) and explicit `args.site=true` are delegated to the whole-website ingestion below; Git hosting URLs such as `https://github.com/{org}/{repo}` are delegated to the Code section above.
 
diff --git a/docs/zh/api/02-resources.md b/docs/zh/api/02-resources.md
index f5b66de842..5f70e83319 100644
--- a/docs/zh/api/02-resources.md
+++ b/docs/zh/api/02-resources.md
@@ -47,7 +47,7 @@ OpenViking 支持多种资源类型，按照功能分类如下：
 网页类（递归网页爬虫）
 | 类型 | 资源名 | 说明 |
 |------|--------|------|
-| 单页 / 递归抓取 | `https://host/path` | 默认仅抓入口页；设置 `args.depth > 0` 后，沿同域链接 BFS 递归展开，`args.max_pages` 只限制最多收集的页面数。每页用 trafilatura 抽成 Markdown，对 SPA 等静态 HTML 拿不到正文的站点自动降级到 Playwright 渲染。可选 `args`：`depth`、`max_pages`、`include_paths`、`exclude_paths`、`allow_external_links`、`skip_download_links`。页面中发现的下载链接默认跳过（`skip_download_links=true`），避免导入 `llms.txt` 等 sidecar 文件造成重复；设为 `false` 时会下载同域文件链接，并计入 `max_pages`。`include_paths`/`exclude_paths` 按**路径前缀**匹配（例如 `/docs/` 仅匹配以 `/docs/` 开头的路径，不会误命中 `/blog/docs-tips`）。|
+| 单页 / 递归抓取 | `https://host/path` | 默认仅抓入口页；设置 `args.depth > 0` 后，沿同域链接 BFS 递归展开，`args.max_pages` 只限制最多收集的页面数。每页用 trafilatura 抽成 Markdown。可选 `args`：`depth`、`max_pages`、`include_paths`、`exclude_paths`、`allow_external_links`、`skip_download_links`。页面中发现的下载链接默认跳过（`skip_download_links=true`），避免导入 `llms.txt` 等 sidecar 文件造成重复；设为 `false` 时会下载同域文件链接，并计入 `max_pages`。`include_paths`/`exclude_paths` 按**路径前缀**匹配（例如 `/docs/` 仅匹配以 `/docs/` 开头的路径，不会误命中 `/blog/docs-tips`）。|
 
 > 路由说明：`https://host/sitemap.xml`、`https://host/feed.xml`、`*.atom` 等 sitemap-looking URL 和显式 `args.site=true` 让出给下表的整站导入；`https://github.com/{org}/{repo}` 等 Git 托管平台 URL 让出给上文的代码导入。
 
diff --git a/openviking/parse/accessors/web_crawler/config.py b/openviking/parse/accessors/web_crawler/config.py
index bcab21bab4..d32f9e9897 100644
--- a/openviking/parse/accessors/web_crawler/config.py
+++ b/openviking/parse/accessors/web_crawler/config.py
@@ -22,8 +22,6 @@ class CrawlConfig:
     retry_times: int = 2
     max_links_per_page: int = 500
     max_html_bytes: int = 10 * 1024 * 1024
-    fallback_playwright: bool = True
-    playwright_timeout: float = 30.0
     request_validator: Optional[Callable[[str], None]] = None
 
     def __post_init__(self) -> None:
@@ -43,5 +41,3 @@ def __post_init__(self) -> None:
             raise ValueError("max_links_per_page must be >= 1.")
         if self.max_html_bytes < 1:
             raise ValueError("max_html_bytes must be >= 1.")
-        if self.playwright_timeout <= 0:
-            raise ValueError("playwright_timeout must be > 0 for recursive web crawling.")
diff --git a/openviking/parse/accessors/web_crawler/models.py b/openviking/parse/accessors/web_crawler/models.py
index 771bfb8779..863a472d40 100644
--- a/openviking/parse/accessors/web_crawler/models.py
+++ b/openviking/parse/accessors/web_crawler/models.py
@@ -31,4 +31,3 @@ class CrawlResult:
     total_downloads: int = 0
     total_skipped: int = 0
     total_failed: int = 0
-    fallback_rendered: int = 0
diff --git a/openviking/parse/accessors/web_crawler/playwright_renderer.py b/openviking/parse/accessors/web_crawler/playwright_renderer.py
deleted file mode 100644
index b3063e02fe..0000000000
--- a/openviking/parse/accessors/web_crawler/playwright_renderer.py
+++ /dev/null
@@ -1,191 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Playwright renderer used only as low-content fallback."""
-
-import asyncio
-from collections.abc import Callable
-from dataclasses import dataclass
-from typing import Optional
-
-from openviking.parse.accessors.web_crawler.render_heuristics import (
-    CHALLENGE_MARKERS,
-    SHELL_VISIBLE_TEXT_CHARS,
-)
-
-
-PLAYWRIGHT_PACKAGE_INSTALL_HINT = (
-    "Playwright fallback was needed, but the Python package is not installed. "
-    "Install it with `pip install playwright` and install Chromium with "
-    "`python -m playwright install chromium`."
-)
-PLAYWRIGHT_CHROMIUM_INSTALL_HINT = (
-    "Playwright fallback was needed, but Chromium is not installed or cannot be "
-    "launched. Run `python -m playwright install chromium` and retry."
-)
-
-# Cap the networkidle wait: pages with continuous background activity (e.g. the
-# GraphiQL playground, polling/websocket apps) never go idle and would otherwise
-# block until the full render timeout. Content is usually ready right after
-# domcontentloaded, and ``_wait_past_challenge`` handles late-arriving text.
-_NETWORKIDLE_TIMEOUT_MS = 8000
-
-
-@dataclass
-class RenderResult:
-    html: str = ""
-    status_code: int = 0
-    final_url: str = ""
-    content_type: str = ""
-    error: Optional[str] = None
-
-    @property
-    def is_success(self) -> bool:
-        return 200 <= self.status_code < 400 and self.error is None
-
-
-class PlaywrightRenderer:
-    def __init__(self, request_validator: Optional[Callable[[str], None]] = None) -> None:
-        self._request_validator = request_validator
-        self._playwright_manager = None
-        self._browser = None
-        self._browser_lock = asyncio.Lock()
-
-    async def render(self, url: str, timeout: float) -> RenderResult:
-        page = None
-        try:
-            if self._request_validator:
-                self._request_validator(url)
-            browser = await self._get_browser()
-            page = await browser.new_page(accept_downloads=False)
-
-            if self._request_validator:
-
-                async def _validate_route(route):
-                    await self._guard_route(route, self._request_validator)
-
-                await page.route("**/*", _validate_route)
-
-            response = await page.goto(
-                url,
-                wait_until="domcontentloaded",
-                timeout=timeout * 1000,
-            )
-            try:
-                await page.wait_for_load_state("networkidle", timeout=_NETWORKIDLE_TIMEOUT_MS)
-            except Exception:
-                pass
-            await self._wait_past_challenge(page, timeout * 1000)
-            html = await self._read_content(page)
-            final_url = page.url
-            if self._request_validator:
-                self._request_validator(final_url)
-            return RenderResult(
-                html=html,
-                status_code=response.status if response else 200,
-                final_url=final_url,
-                content_type=response.headers.get("content-type", "") if response else "",
-            )
-        except ImportError:
-            return RenderResult(final_url=url, error=PLAYWRIGHT_PACKAGE_INSTALL_HINT)
-        except Exception as exc:
-            error = str(exc)
-            if (
-                "Executable doesn't exist" in error
-                or "playwright install" in error
-                or "BrowserType.launch" in error
-            ):
-                error = PLAYWRIGHT_CHROMIUM_INSTALL_HINT
-            return RenderResult(final_url=url, error=error)
-        finally:
-            if page:
-                try:
-                    await page.close()
-                except Exception:
-                    pass
-
-    @staticmethod
-    async def _guard_route(route, request_validator) -> None:
-        """SSRF guard for sub-resource requests.
-
-        Block requests to disallowed hosts (e.g. private-network probe
-        endpoints) by aborting them, but never let a blocked sub-resource
-        fail the whole page render. Only the main document URL and the final
-        URL (validated in ``render``) gate the overall result.
-        """
-        try:
-            request_validator(route.request.url)
-        except Exception:
-            await route.abort()
-            return
-        await route.continue_()
-
-    @staticmethod
-    async def _wait_past_challenge(page, timeout_ms: float, poll_ms: int = 500) -> None:
-        """Wait out JS anti-bot interstitials (e.g. "Please wait...").
-
-        These challenge pages run a CPU-bound JS proof-of-work and then
-        auto-redirect to the real content without further network activity,
-        so ``networkidle`` returns while the interstitial is still showing.
-        Poll the body text until real content appears or we run out of time.
-        """
-        import time
-
-        deadline = time.monotonic() + max(timeout_ms, 0) / 1000
-        while True:
-            try:
-                body = (await page.inner_text("body")).strip()
-            except Exception:
-                body = ""
-            lowered = body.lower()
-            looks_like_challenge = (
-                not body
-                or len(body) < SHELL_VISIBLE_TEXT_CHARS
-                or any(marker in lowered for marker in CHALLENGE_MARKERS)
-            )
-            if not looks_like_challenge or time.monotonic() >= deadline:
-                return
-            await page.wait_for_timeout(poll_ms)
-
-    @staticmethod
-    async def _read_content(page) -> str:
-        """Read page HTML, retrying through in-flight client-side navigation.
-
-        Heavy SPAs (client-side redirects, late hydration) can still be
-        navigating when we first ask for content, which raises "page is
-        navigating and changing the content". Retry until the page settles.
-        """
-        last_exc = None
-        for _ in range(6):
-            try:
-                return await page.content()
-            except Exception as exc:
-                if "navigating and changing the content" not in str(exc):
-                    raise
-                last_exc = exc
-                try:
-                    await page.wait_for_load_state("load", timeout=5000)
-                except Exception:
-                    pass
-                await page.wait_for_timeout(400)
-        if last_exc:
-            raise last_exc
-        return await page.content()
-
-    async def close(self) -> None:
-        async with self._browser_lock:
-            if self._browser:
-                await self._browser.close()
-                self._browser = None
-            if self._playwright_manager:
-                await self._playwright_manager.stop()
-                self._playwright_manager = None
-
-    async def _get_browser(self):
-        async with self._browser_lock:
-            if self._browser is None or not self._browser.is_connected():
-                from playwright.async_api import async_playwright
-
-                if self._playwright_manager is None:
-                    self._playwright_manager = await async_playwright().start()
-                self._browser = await self._playwright_manager.chromium.launch(headless=True)
-            return self._browser
diff --git a/openviking/parse/accessors/web_crawler/render_heuristics.py b/openviking/parse/accessors/web_crawler/render_heuristics.py
deleted file mode 100644
index 91283e54f0..0000000000
--- a/openviking/parse/accessors/web_crawler/render_heuristics.py
+++ /dev/null
@@ -1,83 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-"""Heuristics for deciding whether a page likely needs browser rendering."""
-
-import re
-
-from bs4 import BeautifulSoup
-
-
-_SPA_EMPTY_PATTERNS = (
-    "You need to enable JavaScript to run this app.",
-    "This app works best with JavaScript enabled.",
-    "Please enable JavaScript to continue.",
-    "JavaScript is required to use this application.",
-    "Enable JavaScript to view this page.",
-)
-
-_MIN_VISIBLE_TEXT_CHARS = 200
-SHELL_VISIBLE_TEXT_CHARS = 40
-
-# Text markers of JS anti-bot interstitials that briefly show before the real
-# page loads (e.g. volcengine's proof-of-work gate renders only "Please wait...").
-CHALLENGE_MARKERS = (
-    "please wait",
-    "checking your browser",
-    "verifying you are human",
-    "just a moment",
-    "attention required",
-)
-
-
-def should_render_with_playwright(html: str) -> bool:
-    """Return True when static HTML looks like a client-rendered shell."""
-    html = html or ""
-    html_lower = html.lower()
-    if any(pattern.lower() in html_lower for pattern in _SPA_EMPTY_PATTERNS):
-        return True
-    visible_len = visible_body_text_len(html)
-    if visible_len < SHELL_VISIBLE_TEXT_CHARS:
-        return True
-    # __NEXT_DATA__ only signals a Next.js app; most such pages are SSR/SSG and
-    # already ship full body text in the static HTML. Only render when the
-    # static body is also too thin to be the real content.
-    if "__next_data__" in html_lower and visible_len < _MIN_VISIBLE_TEXT_CHARS:
-        return True
-    if re.search(r'id=["\'](?:root|app)["\']', html_lower) and visible_len < _MIN_VISIBLE_TEXT_CHARS:
-        return True
-    if html_lower.count("<script") >= 5 and visible_len < _MIN_VISIBLE_TEXT_CHARS:
-        return True
-    return False
-
-
-def looks_like_unrendered_page(html: str) -> bool:
-    """Return True when HTML is an empty shell or an anti-bot challenge page.
-
-    Used to reject content that must not be stored as real page text: the
-    static SPA shell or a JS interstitial such as "Please wait...".
-    """
-    try:
-        html = html or ""
-        soup = BeautifulSoup(html, "html.parser")
-        body = soup.body or soup
-        for el in body(["script", "style", "noscript"]):
-            el.decompose()
-        text = body.get_text(strip=True)
-        lowered = text.lower()
-        if any(marker in lowered for marker in CHALLENGE_MARKERS):
-            return True
-        return len(text) < SHELL_VISIBLE_TEXT_CHARS
-    except Exception:
-        return True
-
-
-def visible_body_text_len(html: str) -> int:
-    """Approximate visible text length without running page JavaScript."""
-    try:
-        soup = BeautifulSoup(html or "", "html.parser")
-        body = soup.body or soup
-        for el in body(["script", "style", "noscript"]):
-            el.decompose()
-        return len(body.get_text(strip=True))
-    except Exception:
-        return 0
diff --git a/openviking/parse/accessors/web_crawler/scrapy_spider.py b/openviking/parse/accessors/web_crawler/scrapy_spider.py
index 2b72e70e9d..9cf344242c 100644
--- a/openviking/parse/accessors/web_crawler/scrapy_spider.py
+++ b/openviking/parse/accessors/web_crawler/scrapy_spider.py
@@ -2,24 +2,17 @@
 # SPDX-License-Identifier: AGPL-3.0
 """Scrapy spider for recursive web resource import."""
 
-import asyncio
 import os
 from collections.abc import Iterator
 from urllib.parse import urljoin, urlparse
 
 import scrapy
 from parsel import Selector
-from scrapy import signals
 from scrapy.exceptions import CloseSpider
 
 from openviking.parse.accessors.http_accessor import URLType, URLTypeDetector
 from openviking.parse.accessors.web_crawler.config import CrawlConfig
 from openviking.parse.accessors.web_crawler.models import CrawledDownload, CrawledPage
-from openviking.parse.accessors.web_crawler.playwright_renderer import PlaywrightRenderer
-from openviking.parse.accessors.web_crawler.render_heuristics import (
-    looks_like_unrendered_page,
-    should_render_with_playwright,
-)
 
 
 _DOWNLOAD_URL_TYPES = frozenset(
@@ -78,23 +71,9 @@ def __init__(
         self.config = config
         self.collector = collector
         self.download_collector = download_collector
-        self.renderer = PlaywrightRenderer(config.request_validator)
         self.root_host = urlparse(root_url).netloc.lower()
         self._success_count = 0
         self._seen_download_urls: set[str] = set()
-        self._render_semaphore: asyncio.Semaphore | None = None
-
-    @classmethod
-    def from_crawler(cls, crawler, *args, **kwargs):
-        spider = super().from_crawler(crawler, *args, **kwargs)
-        crawler.signals.connect(spider._on_spider_closed, signal=signals.spider_closed)
-        return spider
-
-    async def _on_spider_closed(self, spider, reason):
-        try:
-            await asyncio.wait_for(self.renderer.close(), timeout=5.0)
-        except Exception:
-            pass
 
     def _success_at_limit(self) -> bool:
         return 0 < self.config.max_pages <= self._success_count
@@ -128,39 +107,6 @@ async def parse(self, response):
             return
 
         page_html = response.text
-        page_source = "scrapy_static"
-        needs_render = self.config.fallback_playwright and should_render_with_playwright(
-            response.text
-        )
-        try:
-            if needs_render:
-                if self._render_semaphore is None:
-                    self._render_semaphore = asyncio.Semaphore(self.config.concurrency)
-                async with self._render_semaphore:
-                    if self._success_at_limit():
-                        return
-                    rendered = await self.renderer.render(
-                        final_url, self.config.playwright_timeout
-                    )
-                if rendered.is_success and rendered.html:
-                    final_url = rendered.final_url or final_url
-                    page_html = rendered.html
-                    page_source = "playwright"
-                elif rendered.error:
-                    self._add_failed(response.url, final_url, depth, rendered.error)
-                    return
-        except Exception as exc:
-            self._add_failed(response.url, final_url, depth, str(exc))
-            return
-
-        if needs_render and looks_like_unrendered_page(page_html):
-            self._add_failed(
-                response.url,
-                final_url,
-                depth,
-                "page did not render real content (empty shell or anti-bot challenge page)",
-            )
-            return
 
         if self._stop_if_success_at_limit():
             return
@@ -170,7 +116,7 @@ async def parse(self, response):
                 final_url=final_url,
                 depth=depth,
                 html=page_html,
-                source=page_source,
+                source="scrapy_static",
             )
         )
         self._success_count += 1
diff --git a/openviking/parse/accessors/web_crawler/web_crawler.py b/openviking/parse/accessors/web_crawler/web_crawler.py
index 39ee7345d9..05bb2e46e0 100644
--- a/openviking/parse/accessors/web_crawler/web_crawler.py
+++ b/openviking/parse/accessors/web_crawler/web_crawler.py
@@ -146,6 +146,4 @@ def _build_result(pages: list[CrawledPage], downloads: list[CrawledDownload]) ->
                 result.total_skipped += 1
             else:
                 result.total_failed += 1
-            if page.source == "playwright":
-                result.fallback_rendered += 1
         return result
diff --git a/openviking/parse/accessors/web_importer.py b/openviking/parse/accessors/web_importer.py
index ea3e61acdc..c838b3616d 100644
--- a/openviking/parse/accessors/web_importer.py
+++ b/openviking/parse/accessors/web_importer.py
@@ -13,10 +13,6 @@
 
 from openviking.parse.accessors.http_accessor import HTTPAccessor
 from openviking.parse.accessors.web_crawler import CrawlConfig, ScrapyWebCrawler
-from openviking.parse.accessors.web_crawler.playwright_renderer import (
-    PLAYWRIGHT_CHROMIUM_INSTALL_HINT,
-    PLAYWRIGHT_PACKAGE_INSTALL_HINT,
-)
 from openviking.parse.accessors.web_feed_accessor import (
     FeedEntry,
     _dedup_relpath,
@@ -25,24 +21,10 @@
     url_to_relpath,
 )
 from openviking_cli.exceptions import InvalidArgumentError
-from openviking_cli.utils.logger import get_logger
-
-logger = get_logger(__name__)
 
 DEPTH_UNLIMITED = -1
 MAX_PAGES_UNLIMITED = -1
 
-# Actionable install hints the renderer records on CrawledPage.error when the
-# Playwright fallback is needed but unavailable. These are surfaced to the user
-# even for non-entry pages, which would otherwise only show up as a failure
-# count.
-_RENDER_INSTALL_HINTS = frozenset(
-    {
-        PLAYWRIGHT_PACKAGE_INSTALL_HINT,
-        PLAYWRIGHT_CHROMIUM_INSTALL_HINT,
-    }
-)
-
 
 @dataclass(frozen=True)
 class WebImportOptions:
@@ -111,19 +93,7 @@ async def import_to_directory(
         success_pages = self._dedupe_success_pages(crawl_result.pages)
         if not any(page.depth == 0 for page in success_pages):
             detail = self._entry_failure_detail(crawl_result.pages)
-            message = f"Failed to fetch entry page: {root_url}"
-            if detail:
-                message = f"{message} ({detail})"
-            raise RuntimeError(message)
-
-        render_hints = self._render_install_hints(crawl_result.pages)
-        if render_hints:
-            logger.warning(
-                "Some pages could not be rendered during web import of %s and were "
-                "skipped: %s",
-                root_url,
-                " ".join(render_hints),
-            )
+            raise RuntimeError(_entry_failure_message(root_url, detail))
 
         temp_root = Path(tempfile.mkdtemp(prefix="ov_web_"))
         temp_dir = temp_root / _host_name(root_url)
@@ -155,35 +125,16 @@ async def import_to_directory(
                 "page_count": len(success_pages),
                 "download_count": len(downloaded_files),
                 "crawl_result": _crawl_summary(crawl_result),
-                "render_hints": render_hints,
                 "original_filename": _host_name(root_url),
             },
         )
 
-    @staticmethod
-    def _render_install_hints(pages) -> list[str]:
-        """Collect actionable render-install hints from failed pages (any depth).
-
-        Child SPA pages that fail because Playwright is unavailable only record
-        the hint on ``CrawledPage.error``; the entry page still succeeds, so the
-        caller would otherwise see the hint nowhere. Surface each distinct hint
-        once so the user can act on it.
-        """
-        hints: list[str] = []
-        for page in pages:
-            if page.status == "success" or not page.error:
-                continue
-            if page.error in _RENDER_INSTALL_HINTS and page.error not in hints:
-                hints.append(page.error)
-        return hints
-
     @staticmethod
     def _entry_failure_detail(pages) -> str:
         """Return the failure reason of the entry page, if one was recorded.
 
-        The renderer surfaces actionable hints (e.g. missing Playwright
-        install) via ``CrawledPage.error``; without this the caller only sees
-        the generic "Failed to fetch entry page" message.
+        Without this the caller only sees the generic "Failed to fetch entry
+        page" message instead of the underlying error (e.g. an SSRF rejection).
         """
         for page in pages:
             if page.depth == 0 and page.status != "success" and page.error:
@@ -269,10 +220,28 @@ def _crawl_summary(crawl_result: Any) -> Dict[str, Any]:
         "total_downloads": getattr(crawl_result, "total_downloads", 0),
         "total_failed": crawl_result.total_failed,
         "total_skipped": crawl_result.total_skipped,
-        "fallback_rendered": crawl_result.fallback_rendered,
     }
 
 
+def _entry_failure_message(root_url: str, detail: str) -> str:
+    """Build a human-readable error for an entry page that could not be fetched.
+
+    The most common cause is the site's robots.txt disallowing crawlers, which
+    Scrapy reports with the terse "Forbidden by robots.txt". Turn that into a
+    plain-language explanation instead.
+    """
+    if detail and "robots.txt" in detail.lower():
+        return (
+            f"Cannot crawl this page: {root_url}\n"
+            "Reason: the site's robots.txt disallows crawlers for this URL, and "
+            "OpenViking honors robots.txt by default, so the crawl was stopped."
+        )
+    message = f"Failed to fetch entry page: {root_url}"
+    if detail:
+        message = f"{message} ({detail})"
+    return message
+
+
 def _normalize_page_url(url: str) -> str:
     parts = urlsplit(url or "")
     scheme = parts.scheme.lower()
diff --git a/openviking/parse/parsers/html.py b/openviking/parse/parsers/html.py
index a169cd2763..452067842e 100644
--- a/openviking/parse/parsers/html.py
+++ b/openviking/parse/parsers/html.py
@@ -24,14 +24,6 @@
 
 logger = __import__("openviking_cli.utils.logger").utils.logger.get_logger(__name__)
 
-_SPA_EMPTY_PATTERNS = (
-    "You need to enable JavaScript to run this app.",
-    "This app works best with JavaScript enabled.",
-    "Please enable JavaScript to continue.",
-    "JavaScript is required to use this application.",
-    "Enable JavaScript to view this page.",
-)
-
 
 class HTMLParser(BaseParser):
     """
@@ -133,10 +125,33 @@ def _html_to_markdown(self, html: str, base_url: str = "") -> str:
         content = self._extract_markdown(html, base_url or "")
         title = self._extract_title(html, base_url or "")
         content = self._clean_markdown(content)
+        if not content:
+            content = self._extract_noscript_notice(html)
         if title and title not in content:
             content = f"# {title}\n\n{content}" if content else f"# {title}"
         return content
 
+    @staticmethod
+    def _extract_noscript_notice(html: str) -> str:
+        """Return the <noscript> text when the page has no extractable body.
+
+        SPA shells (e.g. React apps) render their real content via JavaScript
+        and only ship a static ``<noscript>You need to enable JavaScript...``
+        placeholder. trafilatura ignores <noscript>, so such pages otherwise
+        extract to an empty string. Surfacing the notice keeps at least a hint
+        of why the page looks empty instead of writing a blank file.
+        """
+        try:
+            from bs4 import BeautifulSoup
+
+            soup = BeautifulSoup(html, "html.parser")
+            texts = [
+                node.get_text(" ", strip=True) for node in soup.find_all("noscript")
+            ]
+            return "\n\n".join(text for text in texts if text)
+        except Exception:
+            return ""
+
     @staticmethod
     def _extract_markdown(html: str, url: str) -> str:
         """Extract clean Markdown body from HTML."""
@@ -182,8 +197,6 @@ def _clean_markdown(markdown: str) -> str:
             flags=re.IGNORECASE,
         )
         markdown = re.sub(r"<(span|a)\s+(?:id|name)=['\"][^'\"]*['\"]\s*>\s*</\1>", "", markdown)
-        for pattern in _SPA_EMPTY_PATTERNS:
-            markdown = markdown.replace(pattern, "")
         markdown = re.sub(r"\n{3,}", "\n\n", markdown)
         return markdown.strip()
 
diff --git a/tests/parse/accessors/web_crawler/test_config.py b/tests/parse/accessors/web_crawler/test_config.py
index 501aeb0d5c..cb5a2e5193 100644
--- a/tests/parse/accessors/web_crawler/test_config.py
+++ b/tests/parse/accessors/web_crawler/test_config.py
@@ -47,7 +47,3 @@ def test_max_links_per_page_zero_rejected(self):
     def test_max_html_bytes_zero_rejected(self):
         with pytest.raises(ValueError):
             CrawlConfig(max_html_bytes=0)
-
-    def test_playwright_timeout_non_positive_rejected(self):
-        with pytest.raises(ValueError):
-            CrawlConfig(playwright_timeout=0)
diff --git a/tests/parse/accessors/web_crawler/test_playwright_renderer.py b/tests/parse/accessors/web_crawler/test_playwright_renderer.py
deleted file mode 100644
index bc1cc8250f..0000000000
--- a/tests/parse/accessors/web_crawler/test_playwright_renderer.py
+++ /dev/null
@@ -1,87 +0,0 @@
-from openviking.parse.accessors.web_crawler.playwright_renderer import PlaywrightRenderer
-
-
-class _FakePage:
-    """Minimal page stub returning a scripted sequence of body texts."""
-
-    def __init__(self, body_sequence):
-        self._bodies = list(body_sequence)
-        self.inner_text_calls = 0
-        self.slept_ms = 0
-
-    async def inner_text(self, _selector):
-        self.inner_text_calls += 1
-        idx = min(self.inner_text_calls - 1, len(self._bodies) - 1)
-        return self._bodies[idx]
-
-    async def wait_for_timeout(self, ms):
-        self.slept_ms += ms
-
-
-class TestRenderer:
-    async def test_close_is_idempotent_when_browser_never_started(self):
-        renderer = PlaywrightRenderer()
-        await renderer.close()
-        await renderer.close()
-
-    def test_request_validator_stored(self):
-        called = []
-        renderer = PlaywrightRenderer(request_validator=lambda u: called.append(u))
-        assert renderer._request_validator is not None
-        renderer._request_validator("http://x.com/")
-        assert called == ["http://x.com/"]
-
-
-class _FakeRoute:
-    def __init__(self, url):
-        self.request = type("Req", (), {"url": url})()
-        self.aborted = False
-        self.continued = False
-
-    async def abort(self):
-        self.aborted = True
-
-    async def continue_(self):
-        self.continued = True
-
-
-class TestGuardRoute:
-    async def test_blocks_disallowed_subresource_without_raising(self):
-        def validator(url):
-            if "private" in url:
-                raise ValueError("private network blocked")
-
-        route = _FakeRoute("http://private.internal/probe")
-        # Must not raise: a blocked sub-resource cannot fail the whole render.
-        await PlaywrightRenderer._guard_route(route, validator)
-        assert route.aborted is True
-        assert route.continued is False
-
-    async def test_allows_valid_subresource(self):
-        route = _FakeRoute("https://cdn.example.com/app.js")
-        await PlaywrightRenderer._guard_route(route, lambda _u: None)
-        assert route.continued is True
-        assert route.aborted is False
-
-
-class TestWaitPastChallenge:
-    async def test_waits_until_challenge_redirects_to_content(self):
-        real = "Real documentation body with plenty of visible content here." * 3
-        page = _FakePage(["Please wait...", "Please wait...", real])
-        await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=10_000, poll_ms=10)
-        assert page.inner_text_calls == 3
-        assert page.slept_ms == 20
-
-    async def test_returns_immediately_when_content_present(self):
-        real = "Real documentation body with plenty of visible content here." * 3
-        page = _FakePage([real])
-        await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=10_000, poll_ms=10)
-        assert page.inner_text_calls == 1
-        assert page.slept_ms == 0
-
-    async def test_gives_up_after_timeout_on_persistent_challenge(self):
-        page = _FakePage(["Please wait..."])
-        await PlaywrightRenderer._wait_past_challenge(page, timeout_ms=0, poll_ms=10)
-        assert page.inner_text_calls == 1
-        assert page.slept_ms == 0
-
diff --git a/tests/parse/accessors/web_crawler/test_render_heuristics.py b/tests/parse/accessors/web_crawler/test_render_heuristics.py
deleted file mode 100644
index a1bcff0b1c..0000000000
--- a/tests/parse/accessors/web_crawler/test_render_heuristics.py
+++ /dev/null
@@ -1,63 +0,0 @@
-from openviking.parse.accessors.web_crawler.render_heuristics import (
-    looks_like_unrendered_page,
-    should_render_with_playwright,
-)
-
-
-def test_js_required_pattern_triggers_render():
-    html = "<html>You need to enable JavaScript to run this app.</html>"
-    assert should_render_with_playwright(html) is True
-
-
-def test_react_root_triggers_render():
-    html = "<html><body><div id='root'></div></body></html>"
-    assert should_render_with_playwright(html) is True
-
-
-def test_ssr_app_root_with_rich_content_skips_render():
-    body = "OpenViking 快速开始文档，包含安装、配置、添加资源、创建集合和检索示例。" * 8
-    html = f"<html><body><div id='app'><main><p>{body}</p></main></div></body></html>"
-    assert should_render_with_playwright(html) is False
-
-
-def test_next_data_triggers_render():
-    html = "<html><script>__NEXT_DATA__</script></html>"
-    assert should_render_with_playwright(html) is True
-
-
-def test_many_scripts_with_tiny_body_triggers_render():
-    html = "<html><body>x</body>" + "<script></script>" * 6 + "</html>"
-    assert should_render_with_playwright(html) is True
-
-
-def test_content_rich_static_page_skips_render():
-    body = "This is a fully server-rendered article with plenty of readable text. " * 3
-    html = f"<html><body><p>{body}</p></body></html>"
-    assert should_render_with_playwright(html) is False
-
-
-def test_challenge_page_is_unrendered():
-    html = "<html><body>Please wait...</body></html>"
-    assert looks_like_unrendered_page(html) is True
-
-
-def test_empty_shell_is_unrendered():
-    html = "<html><body><div id='root'></div></body></html>"
-    assert looks_like_unrendered_page(html) is True
-
-
-def test_real_content_is_not_unrendered():
-    body = "Collection 数据过滤删除任务的完整说明文档，包含请求参数、示例与返回值。" * 3
-    html = f"<html><body><article>{body}</article></body></html>"
-    assert looks_like_unrendered_page(html) is False
-
-
-def test_unrendered_check_fails_closed_on_parser_error(monkeypatch):
-    from openviking.parse.accessors.web_crawler import render_heuristics
-
-    def raise_parser_error(*_args, **_kwargs):
-        raise RuntimeError("parser failed")
-
-    monkeypatch.setattr(render_heuristics, "BeautifulSoup", raise_parser_error)
-
-    assert render_heuristics.looks_like_unrendered_page("<html></html>") is True
diff --git a/tests/parse/accessors/web_crawler/test_scrapy_spider.py b/tests/parse/accessors/web_crawler/test_scrapy_spider.py
index 66dccb4e9c..9e39775664 100644
--- a/tests/parse/accessors/web_crawler/test_scrapy_spider.py
+++ b/tests/parse/accessors/web_crawler/test_scrapy_spider.py
@@ -183,7 +183,7 @@ def test_below_limit(self):
 
 class TestParseGate:
     async def test_early_return_when_limit_reached(self):
-        spider = _make_spider(CrawlConfig(max_pages=10, fallback_playwright=False))
+        spider = _make_spider(CrawlConfig(max_pages=10))
         spider._success_count = 10
         resp = _make_response(body=_RICH_HTML)
         items = await _drive_parse(spider, resp)
@@ -192,7 +192,7 @@ async def test_early_return_when_limit_reached(self):
         assert spider.collector == []
 
     async def test_appends_success_and_increments_count(self):
-        spider = _make_spider(CrawlConfig(max_pages=10, fallback_playwright=False))
+        spider = _make_spider(CrawlConfig(max_pages=10))
         resp = _make_response(body=_RICH_HTML)
         await _drive_parse(spider, resp)
         assert spider._success_count == 1
@@ -207,7 +207,6 @@ async def test_download_links_count_toward_max_pages(self):
                 depth=1,
                 max_pages=2,
                 skip_download_links=False,
-                fallback_playwright=False,
             )
         )
         body = (
@@ -225,7 +224,7 @@ async def test_download_links_count_toward_max_pages(self):
         assert items == []
 
     async def test_second_call_blocked_after_reaching_max(self):
-        spider = _make_spider(CrawlConfig(max_pages=1, fallback_playwright=False))
+        spider = _make_spider(CrawlConfig(max_pages=1))
         resp = _make_response(body=_RICH_HTML)
         await _drive_parse(spider, resp)
         assert spider._success_count == 1
@@ -235,7 +234,7 @@ async def test_second_call_blocked_after_reaching_max(self):
         assert len(success) == 1
 
     async def test_reaching_limit_closes_scrapy_engine(self):
-        spider = _make_spider(CrawlConfig(max_pages=1, fallback_playwright=False))
+        spider = _make_spider(CrawlConfig(max_pages=1))
         spider.crawler = MagicMock()
         spider.crawler.engine = MagicMock()
         resp = _make_response(body=_RICH_HTML)
@@ -248,7 +247,7 @@ async def test_reaching_limit_closes_scrapy_engine(self):
         assert len(spider.collector) == 1
 
     async def test_skipped_on_non_html_response(self):
-        spider = _make_spider(CrawlConfig(fallback_playwright=False))
+        spider = _make_spider(CrawlConfig())
         resp = _make_response(
             body=b"binary",
             headers={"content-type": b"application/octet-stream"},
@@ -261,7 +260,7 @@ async def test_skipped_on_non_html_response(self):
     async def test_failed_on_validator_rejecting_entry(self):
         def blocker(_):
             raise ValueError("blocked by SSRF")
-        spider = _make_spider(CrawlConfig(request_validator=blocker, fallback_playwright=False))
+        spider = _make_spider(CrawlConfig(request_validator=blocker))
         resp = _make_response(body=_RICH_HTML)
         await _drive_parse(spider, resp)
         assert spider._success_count == 0
diff --git a/tests/parse/parsers/test_html.py b/tests/parse/parsers/test_html.py
index ef87b8ac9b..213c2697c4 100644
--- a/tests/parse/parsers/test_html.py
+++ b/tests/parse/parsers/test_html.py
@@ -19,9 +19,9 @@ def test_strips_empty_anchor_spans(self):
         assert "<span" not in cleaned
         assert "<a name" not in cleaned
 
-    def test_strips_spa_empty_pattern(self):
-        md = "real content. You need to enable JavaScript to run this app."
-        assert "JavaScript" not in self.parser._clean_markdown(md)
+    def test_keeps_spa_notice_text(self):
+        md = "You need to enable JavaScript to run this app."
+        assert self.parser._clean_markdown(md) == md
 
     def test_collapses_blank_lines(self):
         cleaned = self.parser._clean_markdown("a\n\n\n\n\nb")
@@ -31,6 +31,31 @@ def test_trims_leading_trailing_whitespace(self):
         assert self.parser._clean_markdown("   \n\nhello\n  ") == "hello"
 
 
+class TestHTMLParserNoscriptFallback:
+    def setup_method(self):
+        self.parser = HTMLParser()
+
+    def test_spa_shell_keeps_noscript_notice(self):
+        html = (
+            "<html><head><title></title></head><body>"
+            "<noscript>You need to enable JavaScript to run this app.</noscript>"
+            '<div id="root"></div></body></html>'
+        )
+        md = self.parser._html_to_markdown(html, base_url="https://example.com/")
+        assert "You need to enable JavaScript to run this app." in md
+
+    def test_noscript_fallback_skipped_when_body_extractable(self):
+        html = (
+            "<html><head><title>Doc</title></head><body><article>"
+            "<p>" + ("Real body content that trafilatura can extract. " * 20) + "</p>"
+            "</article>"
+            "<noscript>enable javascript</noscript></body></html>"
+        )
+        md = self.parser._html_to_markdown(html, base_url="https://example.com/")
+        assert "Real body content" in md
+        assert "enable javascript" not in md
+
+
 class TestHTMLParserTitleExtraction:
     def setup_method(self):
         self.parser = HTMLParser()
diff --git a/tests/parse/test_html_parser_utils.py b/tests/parse/test_html_parser_utils.py
deleted file mode 100644
index ca2373e7b6..0000000000
--- a/tests/parse/test_html_parser_utils.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import pytest
-from openviking.parse.parsers.html import HTMLParser
-
-
-class TestHTMLParserRawUrlConversion:
-    """Test suite for HTMLParser._convert_to_raw_url method."""
-
-    def setup_method(self):
-        self.parser = HTMLParser()
-
-    def test_github_blob_conversion(self):
-        blob_url = "https://github.com/volcengine/OpenViking/blob/main/docs/design.md"
-        expected = "https://raw.githubusercontent.com/volcengine/OpenViking/main/docs/design.md"
-        assert self.parser._convert_to_raw_url(blob_url) == expected
-
-        blob_deep = "https://github.com/user/repo/blob/feature/branch/src/components/Button.tsx"
-        expected_deep = (
-            "https://raw.githubusercontent.com/user/repo/feature/branch/src/components/Button.tsx"
-        )
-        assert self.parser._convert_to_raw_url(blob_deep) == expected_deep
-
-    def test_github_non_blob_urls(self):
-        repo_root = "https://github.com/volcengine/OpenViking"
-        assert self.parser._convert_to_raw_url(repo_root) == repo_root
-
-        issue_url = "https://github.com/volcengine/OpenViking/issues/1"
-        assert self.parser._convert_to_raw_url(issue_url) == issue_url
-
-        raw_url = "https://raw.githubusercontent.com/volcengine/OpenViking/main/README.md"
-        assert self.parser._convert_to_raw_url(raw_url) == raw_url
-
-    def test_gitlab_blob_conversion(self):
-        blob_url = "https://gitlab.com/gitlab-org/gitlab/-/blob/master/README.md"
-        expected = "https://gitlab.com/gitlab-org/gitlab/-/raw/master/README.md"
-        assert self.parser._convert_to_raw_url(blob_url) == expected
-
-        blob_deep = "https://gitlab.com/group/project/-/blob/dev/src/main.rs"
-        expected_deep = "https://gitlab.com/group/project/-/raw/dev/src/main.rs"
-        assert self.parser._convert_to_raw_url(blob_deep) == expected_deep
-
-    def test_gitlab_non_blob_urls(self):
-        root = "https://gitlab.com/gitlab-org/gitlab"
-        assert self.parser._convert_to_raw_url(root) == root
-
-        issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/123"
-        assert self.parser._convert_to_raw_url(issue) == issue
-
-    def test_other_domains(self):
-        url = "https://example.com/blob/main/file.txt"
-        assert self.parser._convert_to_raw_url(url) == url
-
-        bitbucket = "https://bitbucket.org/user/repo/src/master/README.md"
-        assert self.parser._convert_to_raw_url(bitbucket) == bitbucket
diff --git a/tests/unit/test_accessors_http.py b/tests/unit/test_accessors_http.py
index 47ae0e27c5..329ab312e1 100644
--- a/tests/unit/test_accessors_http.py
+++ b/tests/unit/test_accessors_http.py
@@ -16,6 +16,7 @@ def _mock_config():
         code=SimpleNamespace(
             github_domains=["github.com", "www.github.com"],
             gitlab_domains=["gitlab.com", "www.gitlab.com"],
+            github_raw_domain="raw.githubusercontent.com",
             azure_devops_domains=[
                 "dev.azure.com",
                 "ssh.dev.azure.com",
@@ -288,3 +289,63 @@ def test_accessor_priority_order(self) -> None:
         assert len(accessors) == 2
         assert accessors[0].__class__.__name__ == "GitAccessor"
         assert accessors[1].__class__.__name__ == "HTTPAccessor"
+
+
+class TestHTTPAccessorRawUrlConversion:
+    """Tests for HTTPAccessor._convert_to_raw_url (GitHub/GitLab blob -> raw)."""
+
+    @pytest.fixture(autouse=True)
+    def _patch_config(self):
+        with patch(
+            "openviking_cli.utils.config.open_viking_config.OpenVikingConfigSingleton.get_instance",
+            side_effect=_mock_config,
+        ):
+            yield
+
+    @pytest.fixture
+    def accessor(self) -> HTTPAccessor:
+        return HTTPAccessor()
+
+    def test_github_blob_conversion(self, accessor: HTTPAccessor) -> None:
+        blob_url = "https://github.com/volcengine/OpenViking/blob/main/docs/design.md"
+        expected = "https://raw.githubusercontent.com/volcengine/OpenViking/main/docs/design.md"
+        assert accessor._convert_to_raw_url(blob_url) == expected
+
+        blob_deep = "https://github.com/user/repo/blob/feature/branch/src/components/Button.tsx"
+        expected_deep = (
+            "https://raw.githubusercontent.com/user/repo/feature/branch/src/components/Button.tsx"
+        )
+        assert accessor._convert_to_raw_url(blob_deep) == expected_deep
+
+    def test_github_non_blob_urls(self, accessor: HTTPAccessor) -> None:
+        repo_root = "https://github.com/volcengine/OpenViking"
+        assert accessor._convert_to_raw_url(repo_root) == repo_root
+
+        issue_url = "https://github.com/volcengine/OpenViking/issues/1"
+        assert accessor._convert_to_raw_url(issue_url) == issue_url
+
+        raw_url = "https://raw.githubusercontent.com/volcengine/OpenViking/main/README.md"
+        assert accessor._convert_to_raw_url(raw_url) == raw_url
+
+    def test_gitlab_blob_conversion(self, accessor: HTTPAccessor) -> None:
+        blob_url = "https://gitlab.com/gitlab-org/gitlab/-/blob/master/README.md"
+        expected = "https://gitlab.com/gitlab-org/gitlab/-/raw/master/README.md"
+        assert accessor._convert_to_raw_url(blob_url) == expected
+
+        blob_deep = "https://gitlab.com/group/project/-/blob/dev/src/main.rs"
+        expected_deep = "https://gitlab.com/group/project/-/raw/dev/src/main.rs"
+        assert accessor._convert_to_raw_url(blob_deep) == expected_deep
+
+    def test_gitlab_non_blob_urls(self, accessor: HTTPAccessor) -> None:
+        root = "https://gitlab.com/gitlab-org/gitlab"
+        assert accessor._convert_to_raw_url(root) == root
+
+        issue = "https://gitlab.com/gitlab-org/gitlab/-/issues/123"
+        assert accessor._convert_to_raw_url(issue) == issue
+
+    def test_other_domains(self, accessor: HTTPAccessor) -> None:
+        url = "https://example.com/blob/main/file.txt"
+        assert accessor._convert_to_raw_url(url) == url
+
+        bitbucket = "https://bitbucket.org/user/repo/src/master/README.md"
+        assert accessor._convert_to_raw_url(bitbucket) == bitbucket
diff --git a/tests/unit/test_web_importer.py b/tests/unit/test_web_importer.py
index 4210771085..76601a475e 100644
--- a/tests/unit/test_web_importer.py
+++ b/tests/unit/test_web_importer.py
@@ -133,7 +133,6 @@ async def crawl(self, root_url):
                     total_downloads=0,
                     total_failed=0,
                     total_skipped=0,
-                    fallback_rendered=0,
                 )
 
         monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
@@ -177,7 +176,6 @@ async def crawl(self, root_url):
                     total_downloads=0,
                     total_failed=0,
                     total_skipped=0,
-                    fallback_rendered=0,
                 )
 
         monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
@@ -188,7 +186,7 @@ async def crawl(self, root_url):
                 options=WebImportOptions(),
             )
 
-    async def test_entry_failure_surfaces_renderer_hint(self, monkeypatch):
+    async def test_entry_failure_surfaces_error_detail(self, monkeypatch):
         class FakeCrawler:
             def __init__(self, config):
                 self.config = config
@@ -202,7 +200,7 @@ async def crawl(self, root_url):
                             depth=0,
                             status="failed",
                             html=None,
-                            error="Playwright fallback was needed, but the Python package is not installed.",
+                            error="blocked by SSRF",
                         )
                     ],
                     downloads=[],
@@ -210,22 +208,17 @@ async def crawl(self, root_url):
                     total_downloads=0,
                     total_failed=1,
                     total_skipped=0,
-                    fallback_rendered=0,
                 )
 
         monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
 
-        with pytest.raises(RuntimeError, match="Playwright fallback was needed"):
+        with pytest.raises(RuntimeError, match="blocked by SSRF"):
             await WebImporter().import_to_directory(
                 root_url="https://example.com",
                 options=WebImportOptions(),
             )
 
-    async def test_child_render_hint_surfaces_in_meta(self, monkeypatch):
-        from openviking.parse.accessors.web_crawler.playwright_renderer import (
-            PLAYWRIGHT_PACKAGE_INSTALL_HINT,
-        )
-
+    async def test_entry_failure_robots_txt_is_humanized(self, monkeypatch):
         class FakeCrawler:
             def __init__(self, config):
                 self.config = config
@@ -237,40 +230,29 @@ async def crawl(self, root_url):
                             url=root_url,
                             final_url=root_url,
                             depth=0,
-                            status="success",
-                            html="<html><body><h1>Home</h1></body></html>",
-                            error=None,
-                        ),
-                        SimpleNamespace(
-                            url=f"{root_url}/spa",
-                            final_url=f"{root_url}/spa",
-                            depth=1,
                             status="failed",
                             html=None,
-                            error=PLAYWRIGHT_PACKAGE_INSTALL_HINT,
-                        ),
+                            error="Forbidden by robots.txt",
+                        )
                     ],
                     downloads=[],
-                    total_crawled=1,
+                    total_crawled=0,
                     total_downloads=0,
                     total_failed=1,
                     total_skipped=0,
-                    fallback_rendered=0,
                 )
 
         monkeypatch.setattr("openviking.parse.accessors.web_importer.ScrapyWebCrawler", FakeCrawler)
 
-        result = await WebImporter().import_to_directory(
-            root_url="https://example.com",
-            options=WebImportOptions(depth=1, max_pages=5),
-        )
-        try:
-            assert result.meta["render_hints"] == [PLAYWRIGHT_PACKAGE_INSTALL_HINT]
-            assert result.meta["page_count"] == 1
-        finally:
-            import shutil
-
-            shutil.rmtree(result.path.parent, ignore_errors=True)
+        with pytest.raises(RuntimeError) as exc_info:
+            await WebImporter().import_to_directory(
+                root_url="https://mp.weixin.qq.com/s/abc",
+                options=WebImportOptions(),
+            )
+        message = str(exc_info.value)
+        assert "robots.txt" in message
+        assert "Forbidden by robots.txt" not in message
+        assert "Cannot crawl this page" in message
 
     async def test_import_to_directory_downloads_child_links(self, monkeypatch, tmp_path):
         class FakeCrawler:
@@ -295,7 +277,6 @@ async def crawl(self, root_url):
                     total_downloads=1,
                     total_failed=0,
                     total_skipped=0,
-                    fallback_rendered=0,
                 )
 
         download_file = tmp_path / "download.txt"

From 39d9cabd0393a6b33c6fe3f53e613efa6e7e8586 Mon Sep 17 00:00:00 2001
From: sponge225 <1670519171@qq.com>
Date: Mon, 6 Jul 2026 17:45:34 +0800
Subject: [PATCH 2/2] refactor(web-importer): simplify robots.txt-blocked
 import message

Replace the verbose robots.txt explanation with a short compliance hint
that omits the URL and points users to local-file import instead.
---
 openviking/parse/accessors/web_importer.py | 8 ++++----
 tests/unit/test_web_importer.py            | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/openviking/parse/accessors/web_importer.py b/openviking/parse/accessors/web_importer.py
index c838b3616d..06a0995872 100644
--- a/openviking/parse/accessors/web_importer.py
+++ b/openviking/parse/accessors/web_importer.py
@@ -228,13 +228,13 @@ def _entry_failure_message(root_url: str, detail: str) -> str:
 
     The most common cause is the site's robots.txt disallowing crawlers, which
     Scrapy reports with the terse "Forbidden by robots.txt". Turn that into a
-    plain-language explanation instead.
+    short compliance-oriented hint instead.
     """
     if detail and "robots.txt" in detail.lower():
         return (
-            f"Cannot crawl this page: {root_url}\n"
-            "Reason: the site's robots.txt disallows crawlers for this URL, and "
-            "OpenViking honors robots.txt by default, so the crawl was stopped."
+            "This URL cannot be imported for compliance reasons "
+            "(the site disallows crawlers). Please save the page and import it "
+            "as a local file instead."
         )
     message = f"Failed to fetch entry page: {root_url}"
     if detail:
diff --git a/tests/unit/test_web_importer.py b/tests/unit/test_web_importer.py
index 76601a475e..4fb20579b7 100644
--- a/tests/unit/test_web_importer.py
+++ b/tests/unit/test_web_importer.py
@@ -250,9 +250,9 @@ async def crawl(self, root_url):
                 options=WebImportOptions(),
             )
         message = str(exc_info.value)
-        assert "robots.txt" in message
         assert "Forbidden by robots.txt" not in message
-        assert "Cannot crawl this page" in message
+        assert "compliance" in message
+        assert "local file" in message
 
     async def test_import_to_directory_downloads_child_links(self, monkeypatch, tmp_path):
         class FakeCrawler: