Real-page audit: live POST /api/run, UA fix, UI pivot, memory endpoint, conf tooltip

Author: dev-pd

backend/app/api/memory.py

@@ -1,5 +1,13 @@
-"""Memory (playbook/diary) endpoints. Wired in Phase 3+; empty router for now."""
+"""Memory endpoints — a peek at what Agent B has learned."""
 
 from fastapi import APIRouter
 
+from app.models.memory import PlaybookEntry
+from app.repositories.playbook import PlaybookRepository
+
 router = APIRouter(prefix="/memory", tags=["memory"])
+
+
+@router.get("/playbook", response_model=list[PlaybookEntry])
+async def playbook() -> list[PlaybookEntry]:
+    return await PlaybookRepository().all()

backend/app/api/run.py

@@ -1,5 +1,47 @@
-"""Agent-run endpoints. Wired in Phase 1+; empty router for now."""
+"""Live agent-run endpoint — audit a real URL with A and B.
+
+Runs both agents live (real page fetch + live SERP + Anthropic) on any URL +
+keyword the user enters, and records the results so a later replay is free. There
+is no answer key for an arbitrary real page, so this returns recommendations and
+cost, not scores — the scored A-vs-B proof lives on the planted benchmark.
+"""
+
+from __future__ import annotations
 
 from fastapi import APIRouter
+from pydantic import BaseModel
+
+from app.agents.agent_a import run_agent_a
+from app.agents.agent_b import run_agent_b
+from app.models import AgentRunResult
+from app.tools.registry import build_default_registry
 
 router = APIRouter(prefix="/run", tags=["run"])
+
+
+class RunRequest(BaseModel):
+    url: str
+    keyword: str
+
+
+class RunResponse(BaseModel):
+    a: AgentRunResult
+    b: AgentRunResult
+
+
+@router.post("", response_model=RunResponse)
+async def run(req: RunRequest) -> RunResponse:
+    a = await run_agent_a(
+        req.url,
+        req.keyword,
+        registry=build_default_registry(replay=False),
+        replay=False,
+    )
+    b = await run_agent_b(
+        req.url,
+        req.keyword,
+        registry=build_default_registry(replay=False),
+        replay=False,
+        memory_on=True,
+    )
+    return RunResponse(a=a, b=b)

backend/app/tools/fetch_page.py

@@ -48,7 +48,17 @@ async def run(self, payload: FetchPageInput) -> Page:
                 )
             return page
 
-        async with httpx.AsyncClient(follow_redirects=True, timeout=20.0) as client:
+        # A browser-like User-Agent: many real sites 403 a request without one.
+        headers = {
+            "User-Agent": (
+                "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
+                "(KHTML, like Gecko) Chrome/124.0 Safari/537.36"
+            ),
+            "Accept": "text/html,application/xhtml+xml",
+        }
+        async with httpx.AsyncClient(
+            follow_redirects=True, timeout=20.0, headers=headers
+        ) as client:
             response = await client.get(payload.url)
             response.raise_for_status()
         page = extract_page(payload.url, response.text)

backend/app/tools/registry.py

@@ -3,6 +3,9 @@
 Both agents are built from the identical registry — same scraper, same SERP
 tool, same SEO checks. Agent B differs only by memory and proactivity, never by
 having different tools.
+
+`replay` controls the data layer: None defers to `REPLAY_MODE` (replay from
+Mongo), False forces a live fetch (real page + live SERP), True forces replay.
 """
 
 from __future__ import annotations
@@ -13,11 +16,12 @@
 from app.tools.seo_checks import SeoChecksTool
 
 
-def build_default_registry() -> ToolRegistry:
+def build_default_registry(replay: bool | None = None) -> ToolRegistry:
+    fetch = FetchPageTool(replay=replay)
     return ToolRegistry(
         [
-            FetchPageTool(),
-            SearchSerpTool(),
-            SeoChecksTool(),
+            fetch,
+            SearchSerpTool(replay=replay),
+            SeoChecksTool(fetch_tool=fetch),
         ]
     )

frontend/src/App.tsx

@@ -1,188 +1,118 @@
-import { Suspense, lazy, useEffect, useMemo, useState, type FormEvent } from 'react'
+import { useEffect, useState, type FormEvent } from 'react'
 import './App.css'
-import { fetchReport } from './api'
+import { fetchPlaybook, runAgents } from './api'
 import { AgentColumn } from './components/AgentColumn'
-import { AlertBanner, MemoryPanel } from './components/MemoryPanel'
-import type { Report, ReportCase } from './types'
+import { MemoryPanel } from './components/MemoryPanel'
+import type { PlaybookEntry, RunResponse } from './types'
 
-// Recharts is heavy — lazy-load the chart so it stays out of the main bundle.
-const LearningCurve = lazy(() =>
-  import('./components/LearningCurve').then((m) => ({ default: m.LearningCurve })),
-)
+type View = 'audit' | 'memory'
+type RunState =
+  | { status: 'idle' }
+  | { status: 'running' }
+  | { status: 'done'; result: RunResponse }
+  | { status: 'error'; message: string }
 
-type View = 'compare' | 'learning'
-type Load = { status: 'loading' } | { status: 'ok'; report: Report } | { status: 'error'; message: string }
+function AuditPage() {
+  const [url, setUrl] = useState('')
+  const [keyword, setKeyword] = useState('')
+  const [run, setRun] = useState<RunState>({ status: 'idle' })
 
-// The case where B most beats A on coverage (so the win shows first), else the first.
-function winningCase(report: Report): ReportCase | undefined {
-  const sorted = [...report.cases].sort(
-    (x, y) =>
-      y.b.scores.coverage - y.a.scores.coverage - (x.b.scores.coverage - x.a.scores.coverage),
-  )
-  return sorted[0] ?? report.cases[0]
-}
-
-function AggregateSummary({ report }: { report: Report }) {
-  const a = report.aggregate.A
-  const b = report.aggregate.B
-  const bWins = b.coverage > a.coverage
-  const metrics: Array<[string, number, number]> = [
-    ['coverage', a.coverage, b.coverage],
-    ['correctness', a.correctness, b.correctness],
-    ['novelty', a.novelty, b.novelty],
-    ['efficiency', a.efficiency, b.efficiency],
-  ]
-  return (
-    <div className="summary">
-      <div className="summary-head">
-        Overall (held-out eval): {bWins ? 'Agent B beats Agent A' : 'A vs B'}
-      </div>
-      <table className="summary-table">
-        <thead>
-          <tr>
-            <th>metric</th>
-            <th>A</th>
-            <th>B</th>
-          </tr>
-        </thead>
-        <tbody>
-          {metrics.map(([name, av, bv]) => (
-            <tr key={name}>
-              <td>{name}</td>
-              <td>{av.toFixed(2)}</td>
-              <td className={bv > av ? 'win' : ''}>{bv.toFixed(2)}</td>
-            </tr>
-          ))}
-        </tbody>
-      </table>
-    </div>
-  )
-}
-
-function ComparePage({ report }: { report: Report }) {
-  const initial = winningCase(report)
-  const [url, setUrl] = useState(initial?.url ?? '')
-  const [keyword, setKeyword] = useState(initial?.keyword ?? '')
-  const [selectedId, setSelectedId] = useState(initial?.id ?? '')
-  const [running, setRunning] = useState(false)
-  const [notFound, setNotFound] = useState(false)
-
-  const selected: ReportCase | undefined = useMemo(
-    () => report.cases.find((c) => c.id === selectedId),
-    [report, selectedId],
-  )
-
-  function handleSubmit(event: FormEvent<HTMLFormElement>) {
+  async function handleSubmit(event: FormEvent<HTMLFormElement>) {
     event.preventDefault()
-    const match = report.cases.find(
-      (c) => c.keyword.toLowerCase() === keyword.trim().toLowerCase() || c.url === url.trim(),
-    )
-    if (!match) {
-      setNotFound(true)
-      return
+    if (!url.trim() || !keyword.trim()) return
+    setRun({ status: 'running' })
+    try {
+      const result = await runAgents(url.trim(), keyword.trim())
+      setRun({ status: 'done', result })
+    } catch (error) {
+      setRun({ status: 'error', message: error instanceof Error ? error.message : 'unknown error' })
     }
-    setNotFound(false)
-    setRunning(true)
-    setSelectedId(match.id)
-    // brief spinner to convey "running A and B"
-    window.setTimeout(() => setRunning(false), 350)
   }
 
   return (
     <>
       <form className="run-form" onSubmit={handleSubmit}>
         <input
-          type="text"
-          placeholder="Page URL"
+          type="url"
+          placeholder="Real page URL (e.g. https://example.com/blog/post)"
           value={url}
           onChange={(e) => setUrl(e.target.value)}
-          list="known-urls"
+          required
         />
         <input
           type="text"
           placeholder="Target keyword"
           value={keyword}
           onChange={(e) => setKeyword(e.target.value)}
-          list="known-keywords"
+          required
         />
-        <button type="submit">Run A vs B</button>
-        <datalist id="known-urls">
-          {report.cases.map((c) => (
-            <option key={c.id} value={c.url} />
-          ))}
-        </datalist>
-        <datalist id="known-keywords">
-          {report.cases.map((c) => (
-            <option key={c.id} value={c.keyword} />
-          ))}
-        </datalist>
+        <button type="submit" disabled={run.status === 'running'}>
+          {run.status === 'running' ? 'Running…' : 'Run A vs B (live)'}
+        </button>
       </form>
-      {notFound && (
-        <p className="hint">
-          Recorded demo cases only — pick a keyword/URL from the suggestions (live runs need an API key).
-        </p>
-      )}
+      <p className="hint">
+        Live run on the real page — fetches it, pulls the live SERP, and runs both agents. Takes
+        ~1–2 minutes.
+      </p>
 
-      <AggregateSummary report={report} />
+      {run.status === 'error' && <p className="hint error">{run.message}</p>}
 
-      {running || !selected ? (
+      {run.status === 'running' && (
         <div className="split">
-          <div className="agent-column spinner-box">running Agent A…</div>
+          <div className="agent-column spinner-box">running Agent A on the live page…</div>
           <div className="agent-column spinner-box">running Agent B…</div>
         </div>
-      ) : (
+      )}
+
+      {run.status === 'done' && (
         <main className="split">
-          <AgentColumn agent="A" view={selected.a} />
-          <AgentColumn agent="B" view={selected.b} />
+          <AgentColumn view={run.result.a} />
+          <AgentColumn view={run.result.b} />
         </main>
       )}
     </>
   )
 }
 
-export default function App() {
-  const [view, setView] = useState<View>('compare')
-  const [load, setLoad] = useState<Load>({ status: 'loading' })
+function MemoryPage() {
+  const [playbook, setPlaybook] = useState<PlaybookEntry[] | null>(null)
+  const [error, setError] = useState<string | null>(null)
 
   useEffect(() => {
     let active = true
-    fetchReport()
-      .then((report) => active && setLoad({ status: 'ok', report }))
-      .catch((error: unknown) => {
-        const message = error instanceof Error ? error.message : 'unknown error'
-        if (active) setLoad({ status: 'error', message })
-      })
+    fetchPlaybook()
+      .then((p) => active && setPlaybook(p))
+      .catch((e: unknown) => active && setError(e instanceof Error ? e.message : 'error'))
     return () => {
       active = false
     }
   }, [])
 
+  if (error) return <p className="hint error">{error}</p>
+  if (!playbook) return <p className="hint">loading playbook…</p>
+  return (
+    <main className="learning">
+      <MemoryPanel playbook={playbook} />
+    </main>
+  )
+}
+
+export default function App() {
+  const [view, setView] = useState<View>('audit')
   return (
     <div className="app">
       <header className="app-header">
         <h1>Learning SEO Agent</h1>
         <nav className="nav">
-          <button className={view === 'compare' ? 'active' : ''} onClick={() => setView('compare')}>
-            Compare
+          <button className={view === 'audit' ? 'active' : ''} onClick={() => setView('audit')}>
+            Audit (live)
           </button>
-          <button className={view === 'learning' ? 'active' : ''} onClick={() => setView('learning')}>
-            Learning &amp; Memory
+          <button className={view === 'memory' ? 'active' : ''} onClick={() => setView('memory')}>
+            B's Memory
           </button>
         </nav>
       </header>
-
-      {load.status === 'loading' && <p className="hint">Running the agents… (first load replays the comparison)</p>}
-      {load.status === 'error' && <p className="hint error">Could not load report: {load.message}</p>}
-      {load.status === 'ok' && view === 'compare' && <ComparePage report={load.report} />}
-      {load.status === 'ok' && view === 'learning' && (
-        <main className="learning">
-          <AlertBanner alert={load.report.alert} />
-          <Suspense fallback={<p className="hint">loading chart…</p>}>
-            <LearningCurve cases={load.report.cases} />
-          </Suspense>
-          <MemoryPanel playbook={load.report.playbook} />
-        </main>
-      )}
+      {view === 'audit' ? <AuditPage /> : <MemoryPage />}
     </div>
   )
 }