feat(mcp): return clean text by default for LLMs

fetch and stealth_fetch now expose text_only (default True), reusing
the existing client.fetch(text_only=...) path. Agents get clean visible
page text instead of raw HTML, ~5-10x fewer tokens. Pass text_only=False
for HTML. README and site copy lead with the LLM/agent story.

Bump 0.5.7 -> 0.5.8.

Author: kacigaya

README.md

@@ -5,8 +5,8 @@
 <h1 align="center">WebSkrap</h1>
 
 <p align="center">
-   <strong>Async-first Python scraping framework built on Playwright.</strong><br>
-   <em>It provides coherent browser profiles, persistent sessions, resource routing, and Patchright-powered stealth for data collection workflows that need realistic browser behavior.</em>
+   <strong>Async-first Python scraping framework built on Playwright — and a first-class web tool for LLMs and agents.</strong><br>
+   <em>Coherent browser profiles, persistent sessions, resource routing, and Patchright-powered stealth for data collection workflows that need realistic browser behavior. Ships an MCP server so Claude, Codex, and any MCP agent can fetch live pages as clean, token-efficient text.</em>
 </p>
 
 WebSkrap does not include CAPTCHA solving, login-wall bypassing, credential bypassing, or access-control circumvention. Use it only on targets you are allowed to access.
@@ -396,9 +396,18 @@ stderr is not a TTY.
 
 ## MCP server
 
-WebSkrap ships an optional Model Context Protocol server so MCP clients (Claude
-Desktop, Claude Code, ...) can drive scraping directly. It exposes three tools
-over stdio: `fetch`, `stealth_fetch`, and `doctor`.
+WebSkrap ships a Model Context Protocol server so MCP clients (Claude Desktop,
+Claude Code, Codex, ...) can drive a real browser directly. It exposes three
+tools over stdio: `fetch`, `stealth_fetch`, and `doctor`.
+
+Built for LLMs: `fetch` and `stealth_fetch` return **clean visible page text by
+default** — no HTML tags, scripts, or style noise — so agents spend tokens on
+content, not markup (typically 5-10x fewer tokens than raw HTML). Pass
+`text_only=False` when you actually need the HTML. `stealth_fetch` gives agents
+the same CDP-leak-free Patchright path the CLI uses, so anti-bot pages that
+block naive scrapers still load. Every result carries `status`, `final_url`,
+`title`, `text_length`, and truncation flags so the model knows exactly what it
+got.
 
 ```bash
 pip install webskrap

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "webskrap"
-version = "0.5.7"
+version = "0.5.8"
 description = "A Playwright-based Python scraping framework with coherent browser profiles and session controls."
 readme = "README.md"
 requires-python = ">=3.11"

src/webskrap/mcp_server.py

@@ -36,16 +36,21 @@ async def fetch(
     resource_policy: str = "all",
     timeout_ms: float = 30_000,
     max_chars: int = 20_000,
+    text_only: bool = True,
 ) -> dict[str, Any]:
     """Fetch a URL with a standard Playwright browser and return page data.
 
+    Returns clean visible page text by default (LLM-friendly, no HTML tags).
+    Set text_only=False to get the raw HTML instead.
+
     Args:
         url: The URL to load.
         profile: Bundled profile (desktop-chrome, desktop-edge, mobile-chrome).
         wait_until: commit, domcontentloaded, load, or networkidle.
         resource_policy: all, lite (block images/fonts/media), or documents.
         timeout_ms: Navigation timeout in milliseconds.
-        max_chars: Maximum characters of page HTML to return.
+        max_chars: Maximum characters of page text to return.
+        text_only: Return clean visible text (default) instead of raw HTML.
     """
     config = SessionConfig(
         navigation_timeout_ms=timeout_ms,
@@ -58,6 +63,7 @@ async def fetch(
             config=config,
             wait_until=parse_wait_until(wait_until),
             timeout_ms=timeout_ms,
+            text_only=text_only,
         )
     return shape_fetch_result(result, max_chars)
 
@@ -75,11 +81,14 @@ async def stealth_fetch(
     webrtc_ip_handling_policy: str | None = None,
     timeout_ms: float = 90_000,
     max_chars: int = 20_000,
+    text_only: bool = True,
 ) -> dict[str, Any]:
     """Fetch a URL with the Patchright stealth driver (CDP-leak-free).
 
-    Requires Patchright's browser download: webskrap install. Prefer
-    headless=False with channel="chrome" for the strictest anti-bot path.
+    Returns clean visible page text by default (LLM-friendly, no HTML tags).
+    Set text_only=False to get the raw HTML instead. Requires Patchright's
+    browser download: webskrap install. Prefer headless=False with
+    channel="chrome" for the strictest anti-bot path.
 
     Args:
         url: The URL to load.
@@ -93,7 +102,8 @@ async def stealth_fetch(
         webrtc_ip_handling_policy: Chromium WebRTC ICE policy, e.g.
             disable_non_proxied_udp.
         timeout_ms: Navigation timeout in milliseconds.
-        max_chars: Maximum characters of page HTML to return.
+        max_chars: Maximum characters of page text to return.
+        text_only: Return clean visible text (default) instead of raw HTML.
     """
     config = SessionConfig(
         driver="patchright",
@@ -112,6 +122,7 @@ async def stealth_fetch(
             profile=get_profile(profile),
             config=config,
             timeout_ms=timeout_ms,
+            text_only=text_only,
         )
     return shape_fetch_result(result, max_chars)
 

tests/test_mcp.py

@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import asyncio
+from typing import Any
+
+from webskrap import mcp_server
+from webskrap.models import FetchResult
+
+
+class _FakeClient:
+    calls: list[dict[str, Any]] = []
+
+    async def __aenter__(self) -> _FakeClient:
+        return self
+
+    async def __aexit__(self, exc_type: object, exc: object, traceback: object) -> None:
+        return None
+
+    async def fetch(self, url: str, **kwargs: Any) -> FetchResult:
+        self.calls.append({"url": url, **kwargs})
+        text = "Readable body" if kwargs.get("text_only") else "<html>abcdef</html>"
+        return FetchResult(
+            url=url,
+            final_url=f"{url}/final",
+            status=200,
+            ok=True,
+            headers={"content-type": "text/html"},
+            text=text,
+            title="Example",
+            cookies=[],
+            timings={"elapsed_ms": 12.34},
+        )
+
+
+def _fake_client(monkeypatch: Any) -> None:
+    _FakeClient.calls = []
+    monkeypatch.setattr(mcp_server, "WebSkrapClient", _FakeClient)
+
+
+def test_fetch_defaults_to_clean_text(monkeypatch: Any) -> None:
+    _fake_client(monkeypatch)
+
+    result = asyncio.run(mcp_server.fetch("https://example.test"))
+
+    assert _FakeClient.calls[0]["text_only"] is True
+    assert result["text"] == "Readable body"
+
+
+def test_fetch_text_only_false_returns_html(monkeypatch: Any) -> None:
+    _fake_client(monkeypatch)
+
+    result = asyncio.run(mcp_server.fetch("https://example.test", text_only=False))
+
+    assert _FakeClient.calls[0]["text_only"] is False
+    assert result["text"] == "<html>abcdef</html>"
+
+
+def test_stealth_fetch_defaults_to_clean_text(monkeypatch: Any) -> None:
+    _fake_client(monkeypatch)
+
+    result = asyncio.run(mcp_server.stealth_fetch("https://example.test"))
+
+    assert _FakeClient.calls[0]["text_only"] is True
+    assert result["text"] == "Readable body"

web/content/user-guide/mcp.md

@@ -1,8 +1,15 @@
 # MCP server
 
-WebSkrap ships an optional [Model Context Protocol](https://modelcontextprotocol.io)
-server. MCP clients such as Claude Desktop and Claude Code can call it to drive
-scraping directly. It runs over stdio and exposes three tools.
+WebSkrap ships a [Model Context Protocol](https://modelcontextprotocol.io)
+server. MCP clients such as Claude Desktop, Claude Code, and Codex can call it to
+drive a real browser directly. It runs over stdio and exposes three tools.
+
+**Built for LLMs.** `fetch` and `stealth_fetch` return clean visible page text
+by default — no HTML tags, scripts, or CSS noise — so the model spends tokens on
+content, not markup (typically 5-10x fewer tokens than raw HTML). `stealth_fetch`
+gives agents the same CDP-leak-free Patchright path the CLI uses, so anti-bot
+pages that block naive scrapers still load. Pass `text_only=false` when you
+actually need the HTML.
 
 ## Install
 
@@ -32,8 +39,9 @@ python -m webskrap.mcp_server
 | `doctor` | Check that Playwright and Chromium can launch. |
 
 Both fetch tools return `status`, `final_url`, `title`, `ok`, `headers`, and the
-page HTML in `text` (capped by `max_chars`, with `text_length` and
-`text_truncated` reporting the full size).
+page content in `text` (capped by `max_chars`, with `text_length` and
+`text_truncated` reporting the full size). By default `text` is clean visible
+text; set `text_only` to `false` to get raw HTML.
 
 ## Tool arguments
 
@@ -46,7 +54,8 @@ page HTML in `text` (capped by `max_chars`, with `text_length` and
 | `wait_until` | `domcontentloaded` | `commit`, `domcontentloaded`, `load`, or `networkidle`. |
 | `resource_policy` | `all` | `all`, `lite`, or `documents`. |
 | `timeout_ms` | `30000` | Navigation timeout. |
-| `max_chars` | `20000` | Maximum returned HTML characters. |
+| `max_chars` | `20000` | Maximum returned text characters. |
+| `text_only` | `true` | Return clean visible text; set `false` for raw HTML. |
 
 Example arguments:
 

web/src/app/(site)/page.tsx

@@ -45,12 +45,12 @@ const FEATURES = [
   {
     title: "MCP server",
     description:
-      "Expose fetch and stealth_fetch tools to agents without extra Python packages.",
+      "fetch and stealth_fetch tools for Claude, Codex, and any MCP agent — returning clean text, not tag soup.",
   },
   {
-    title: "LLM-friendly CLI",
+    title: "Built for LLMs",
     description:
-      "Use JSON, bounded output, stdout, and text-only fetches from the terminal.",
+      "Clean-text output by default, 5-10x fewer tokens than raw HTML, plus JSON and bounded CLI output.",
   },
 ];
 
@@ -112,9 +112,10 @@ export default function Home() {
           Scrape the web like a real browser
         </h1>
         <p className="mt-6 max-w-2xl text-lg text-muted-foreground">
-          WebSkrap is an async-first Python scraping framework built on Playwright. Coherent
-          browser profiles, persistent sessions, resource routing, Patchright-powered stealth, and
-          machine-readable CLI output for data collection that needs realistic behavior.
+          WebSkrap is an async-first Python scraping framework built on Playwright — and a
+          first-class web tool for LLMs and agents. Coherent browser profiles, persistent sessions,
+          Patchright-powered stealth, and an MCP server that hands agents live pages as clean,
+          token-efficient text.
         </p>
         <div className="mt-10 flex flex-col gap-3 sm:flex-row">
           <Button size="xl" render={<Link href={DOCS_URL} />}>