Degrade gracefully on blocked page: SERP-only audit + clean note, no 502

Author: dev-pd

backend/app/api/run.py

@@ -32,13 +32,17 @@ class RunRequest(BaseModel):
 class RunResponse(BaseModel):
     a: AgentRunResult
     b: AgentRunResult
+    note: str | None = (
+        None  # set when the page body couldn't be fetched (SERP-only run)
+    )
 
 
 @router.post("", response_model=RunResponse)
 async def run(req: RunRequest) -> RunResponse:
     try:
-        # Fail fast if the page can't be fetched — don't burn LLM tokens first.
-        await FetchPageTool(replay=False).run(FetchPageInput(url=req.url))
+        # Probe the page once. A bot-block no longer fails the run — fetch_page degrades
+        # to a SERP-only audit and tells the user via `note`.
+        probe = await FetchPageTool(replay=False).run(FetchPageInput(url=req.url))
         a = await run_agent_a(
             req.url,
             req.keyword,
@@ -57,24 +61,17 @@ async def run(req: RunRequest) -> RunResponse:
         )
         # …then consolidate the diary into the playbook so the NEXT run sees it.
         await run_janitor()
-    except httpx.HTTPStatusError as exc:
-        code = exc.response.status_code
-        raise HTTPException(
-            status_code=502,
-            detail=(
-                f"Couldn't fetch {req.url} (HTTP {code}). The site likely blocks "
-                "automated requests (bot protection). Try a different page."
-            ),
-        ) from exc
     except httpx.HTTPError as exc:
+        # A page bot-block degrades gracefully (see fetch_page); this only fires for
+        # other live HTTP failures, e.g. the SERP call.
         raise HTTPException(
             status_code=502,
-            detail=f"Couldn't reach {req.url}: {exc.__class__.__name__}. Try a different page.",
+            detail=f"Live request failed: {exc.__class__.__name__}. Try again or a different page.",
         ) from exc
     except ReplayMissError as exc:
         raise HTTPException(status_code=409, detail=str(exc)) from exc
     except RuntimeError as exc:
         # e.g. live SERP needs SERP_API_KEY
         raise HTTPException(status_code=502, detail=str(exc)) from exc
 
-    return RunResponse(a=a, b=b)
+    return RunResponse(a=a, b=b, note=None if probe.fetch_ok else probe.fetch_note)

backend/app/models/page.py

@@ -20,6 +20,11 @@ class Page(BaseModel):
     meta_description: str | None = None
     headings: list[Heading] = Field(default_factory=list)
     body_text: str = ""
+    # False when the body couldn't be fetched (e.g. bot-blocked 403). On-page checks
+    # are skipped in that case so we never hallucinate defects from an empty page; the
+    # SERP-driven stages (keyword, rank, gap) still run.
+    fetch_ok: bool = True
+    fetch_note: str = ""
 
     @property
     def word_count(self) -> int:

backend/app/tools/fetch_page.py

@@ -57,11 +57,33 @@ async def run(self, payload: FetchPageInput) -> Page:
             "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "Accept-Language": "en-US,en;q=0.9",
         }
-        async with httpx.AsyncClient(
-            follow_redirects=True, timeout=20.0, headers=headers
-        ) as client:
-            response = await client.get(payload.url)
-            response.raise_for_status()
+        try:
+            async with httpx.AsyncClient(
+                follow_redirects=True, timeout=20.0, headers=headers
+            ) as client:
+                response = await client.get(payload.url)
+                response.raise_for_status()
+        except httpx.HTTPStatusError as exc:
+            # Bot-blocked or missing page: degrade rather than fail the whole audit.
+            # Not saved — we never want a failure cached as the page snapshot.
+            return Page(
+                url=payload.url,
+                fetch_ok=False,
+                fetch_note=(
+                    f"Couldn't read the page body (HTTP {exc.response.status_code} — the "
+                    "site blocks automated requests). On-page checks skipped; SERP-based "
+                    "analysis still ran."
+                ),
+            )
+        except httpx.HTTPError as exc:
+            return Page(
+                url=payload.url,
+                fetch_ok=False,
+                fetch_note=(
+                    f"Couldn't reach the page ({exc.__class__.__name__}). On-page checks "
+                    "skipped; SERP-based analysis still ran."
+                ),
+            )
         page = extract_page(payload.url, response.text)
         await self._pages.save(page, payload.version)
         return page

backend/app/tools/seo_checks.py

@@ -121,7 +121,13 @@ def check_has_comparison_table(page: Page, keyword: str) -> CheckResult:
 def run_checks(
     page: Page, keyword: str, names: list[str] | None = None
 ) -> list[CheckResult]:
-    """Run the named checks (or all of them) against a page."""
+    """Run the named checks (or all of them) against a page.
+
+    If the page body couldn't be fetched, return nothing: on-page checks on an empty
+    page would invent defects (missing title, thin content) that aren't real.
+    """
+    if not page.fetch_ok:
+        return []
     selected = names or list(ALL_CHECKS)
     return [ALL_CHECKS[name](page, keyword) for name in selected]
 

frontend/src/App.css

@@ -75,6 +75,16 @@
   margin: 0.25rem 0 0;
 }
 
+.notice {
+  border: 1px solid #f5d9a8;
+  background: #fdf6e8;
+  border-radius: 8px;
+  padding: 0.6rem 1rem;
+  margin: 0.5rem 0 1rem;
+  color: #7a5a12;
+  font-size: 0.9rem;
+}
+
 .summary {
   border: 1px solid var(--border);
   border-radius: 8px;

frontend/src/App.tsx

@@ -69,6 +69,10 @@ function AuditPage() {
         </div>
       )}
 
+      {run.status === 'done' && run.result.note && (
+        <div className="notice">{run.result.note}</div>
+      )}
+
       {run.status === 'done' && (
         <main className="split">
           <AgentColumn view={run.result.a} />

frontend/src/types.ts

@@ -35,6 +35,7 @@ export interface AgentRunResult {
 export interface RunResponse {
   a: AgentRunResult
   b: AgentRunResult
+  note?: string | null // set when the page body couldn't be fetched (SERP-only run)
 }
 
 export interface PlaybookEntry {