diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf6dd2d..128b3f2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,9 @@ jobs: - name: Execute unit test suite run: python -m unittest discover -s tests -p "test_*.py" -v + - name: Typecheck + run: make typecheck + - name: Verify OpenAPI snapshot run: make openapi-verify diff --git a/AGENTS.md b/AGENTS.md index 3b6930f..42c023d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -5,11 +5,103 @@ - Docker-first and Docker-only unless user asks otherwise. - Keep repo focused: stable Botasaurus scrape API wrapper, not generic framework. +## Project Layout + +``` +app/ + main.py # create_app() factory; module-level `app` for uvicorn + constants.py # SERVICE_NAME and other shared literals + config.py # Settings + nested SentrySettings; single env source of truth + exceptions.py # domain exceptions (e.g. RequestIdCollisionError) + logging_config.py # setup_logging + get_logger(); single owner of logger name + api/ + deps.py # FastAPI Depends: settings, engine, executor, ScrapeService + errors.py # 422 + 500 handlers → scrape error envelope + openapi.py # route OpenAPI metadata (configure_openapi at app creation) + openapi_examples.py # OpenAPI examples built from Pydantic model instances + routes/ # thin HTTP handlers (health, scrape) + domain/ + scrape_service.py # request-id resolution, URL guardrails, threadpool execution, status mapping + engine/ + orchestrator.py # ScraperEngine.execute + session.py # ScrapeSession lifecycle + budget.py # wall-clock budget math shared across tiers (elapsed_ms, step budgets) + request_tier.py # HTTP/curl_cffi path + browser_tier.py # Chromium path + strategies.py # NavigationMode resolution, driver helpers + driver_capabilities.py # DriverProtocol + call_if_available adapter + envelope.py # success/error builders, UTF-8 HTML normalization + schemas/ + enums.py # ExecutionMode, NavigationMode, ErrorCategory, ... + request.py # ScrapeRequest and validators + response.py # ScrapeSuccess, ScrapeError, HealthResponse, ... + infra/ # telemetry, progress, metadata, xhr, runtime cleanup, sentry + security/ # UrlGuard SSRF guardrails +scripts/ + bench_scrape.py # TestClient wall-time bench for POST /scrape (request tier) +tests/ + api/ # HTTP contract, request schema, 504 timeout envelope tests + domain/ # ScrapeService unit tests (timeout error mapping) + engine/ # ScraperEngine units, isolation regressions, timeout progress + infra/ # challenge, metadata, xhr, progress, sentry, telemetry, request-id, cleanup + security/ # UrlGuard tests + support/ + http.py # test_client() context manager + dependency_overrides helper + fakes.py # shared FakeDriver, FakeRequest, fake_request_cls, ... + factories.py # scrape_request(), example_url() + test_bench_regression.py # lightweight guard that bench script completes (root: guards scripts/) +``` + +Layer rules: + +| Layer | May import | Must not import | +| --- | --- | --- | +| `api/routes` | `domain`, `api/deps`, `schemas.*` | `engine` internals, Botasaurus | +| `domain` | `engine`, `security`, `schemas.*`, `infra` | FastAPI, Botasaurus | +| `engine` | `infra`, `security`, `schemas.*`, `config` | FastAPI | +| `infra` | Botasaurus, CDP (lazy at use sites) | FastAPI, routes | + +Conventions: + +- Use `tests/support/http.test_client()` for HTTP tests; it runs lifespan and manages `dependency_overrides`. No ad-hoc `TestClient(create_app())` in test modules. +- Loggers come from `app.logging_config.get_logger()`; do not call `logging.getLogger` with a literal name. +- Wall-clock/timeout math (elapsed, remaining, step budgets) lives in `app/engine/budget.py`; tiers must not re-derive it. +- Config: add env vars to `Settings` in `config.py`; call `reset_settings_cache()` in tests that patch env. +- Wire types live in `app/schemas/` submodules; import directly (`from app.schemas.request import ScrapeRequest`). No long-lived re-export shim. +- Domain logic stays out of route handlers and Pydantic shells. +- Typed exceptions over string-matching (`RequestIdCollisionError`, not `RuntimeError` message checks). +- `NavigationMode` end-to-end in engine code; no raw strategy strings outside enum conversion boundaries. +- Optional Botasaurus driver methods go through `driver_capabilities.call_if_available` / `resolve_callable` only; do not ad-hoc `getattr(driver, ...)`. +- OpenAPI route examples come from `openapi_examples.py` model instances, not hand-typed dicts. +- Route modules expose `create_router()` factories included from `create_app()` **after** `configure_openapi(settings)`, so timeout-dependent response metadata is not frozen on first import. +- Botasaurus/CDP imports are lazy inside tier entrypoints (`run_request_tier`, `run_browser_tier`, XhrCollector methods), not at app import time. + +## Singleton + Settings Threading + +- **One** `ScraperEngine` and **one** `ThreadPoolExecutor` are created in `create_app()` lifespan and stored on `app.state`. +- `get_engine` / `get_executor` / `SettingsDep` read from `request.app.state` (not per-request construction). +- Lifespan shutdown calls `executor.shutdown(wait=False, cancel_futures=True)` on the real pool instance. +- Do **not** freeze settings at import time in schemas or OpenAPI modules. `configure_openapi(settings)` runs during `create_app()`; `clamp_wait_timeout_seconds` reads live `get_settings()`. +- `ScraperEngine` and tier functions require an injected `Settings` parameter; no `get_settings()` fallback in the hot path. + +## Isolation Invariants + +| Resource | Lifetime | Rule | +| --- | --- | --- | +| `ScraperEngine` | process (app.state) | shared | +| `ThreadPoolExecutor` | process (app.state) | shared, sized by `SCRAPE_MAX_WORKERS` | +| `_active_request_ids` | in-process memory | shared; collision guard | +| runtime dir `/tmp/scrape/` | per request | isolated; deleted in `finally` | +| browser profile | per request | isolated; no reuse | +| Botasaurus Driver | per request | isolated; closed in `finally` | + +Multi-worker uvicorn breaks in-process collision detection unless request ids are sticky to a worker. Default to single-worker for isolation semantics. + ## Contract (Do Not Break) - Endpoints: `GET /health`, `POST /scrape`. - `openapi.yaml` is generated from `app.openapi()` via `make openapi`. Do not hand-edit. `make openapi-verify` is part of `make check`. Spectral (`make spectral`) lints the snapshot; do not add a post-processor that mutates the dump. -- Wire types live in `app/schemas.py`. Engine imports them. Routes `model_dump()` once into `JSONResponse`. +- Wire types live in `app/schemas/`. Engine imports them. Routes call `ScrapeService.process()` and serialize via `json_response()`. - OpenAPI `info.version` is `2.0.0`. Schema names: `ScrapeSuccess` (200) and `ScrapeError` (400/403/422/502/504). No `ScrapeResponse` alias. - Success `/scrape` fields: `url`, `final_url`, `status_code`, `headers`, `html`, `metadata_error`, `xhr_responses`, `diagnostics`. - When `html` is present, document `headers` `content-type` is `text/html; charset=utf-8` and `html` is UTF-8-normalized. @@ -36,7 +128,8 @@ - close browser driver - delete request runtime dir - remove in-memory active request id -- Keep request-id collision/invariant guard (`_active_request_ids`) intact. +- Before each scrape, prune orphaned runtime dirs not tied to an active request id; ENOSPC on profile creation retries after another prune pass. Optional `SCRAPE_RUNTIME_MIN_FREE_BYTES` (default 256MiB) logs when the runtime filesystem is low. +- Keep request-id collision/invariant guard (`_active_request_ids`) intact; raises `RequestIdCollisionError`. - `driver.requests.get` metadata is best-effort; metadata failure must not fail HTML success. - Keep strategy engine behavior: - `auto` mode attempt order: `google_get` -> `google_get_bypass` -> `get` @@ -46,6 +139,21 @@ - keep `/usr/bin/google-chrome` symlink to Chromium for compatibility - If browser install logic changes, re-verify binary path and Botasaurus startup. +## Performance + +- Baseline bench: `PYTHONPATH=. .venv/bin/python3 scripts/bench_scrape.py --runs 10` (TestClient, `execution_mode=request`). +- Record p50 wall time when changing hot paths; avoid regressions vs prior baseline. +- On low-RAM hosts (Docker `--memory=768m`, 1–2 vCPU), tune `SCRAPE_MAX_WORKERS` to `1` or `2`; higher values increase queue wait and swap without improving wall time. +- Browser tier skips XHR harvest before Cloudflare bypass; one consolidated `collect_page_state` pass runs after bypass. + +## Types + +- `make typecheck` runs **`pyright` strict** on `app tests` and is part of `make check`. +- Vendor seams: local stubs in `typings/` (`stubPath` in `pyproject.toml`); CDP shapes in `app/infra/cdp_types.py`. +- Engine driver seams use `DriverProtocol` / `CdpTabProtocol` in `driver_capabilities.py`; cast vendor `Driver` at construction when needed. +- Tests construct `ScrapeRequest` via `tests/support/factories.py` (`scrape_request`, `example_url`); fakes implement protocol shapes in `tests/support/fakes.py`. +- Residual policy and file-level test directives: `docs/typing-residuals.md`. + ## Safety - Keep SSRF guardrails: localhost/domain checks and blocked IP classes (loopback/private/link-local/multicast/reserved/unspecified). @@ -53,7 +161,7 @@ ## Done Criteria -- Run `make check` before finish. +- Run `make check` before finish (lint, test, typecheck, openapi-verify). - When Pydantic models or route response metadata change, run `make openapi` and commit the snapshot with the code change. - When API contract, Docker behavior, or error semantics change, also run `make smoke`. - `make smoke` must cover build, boot, `/health`, `/scrape` happy path, strategy override, retry path, isolation check, localhost guardrail. diff --git a/Makefile b/Makefile index c55d75b..34bf8c0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: test build serve health scrape-example smoke lint lintfix check ready openapi openapi-verify spectral +.PHONY: test build serve health scrape-example smoke lint lintfix check ready openapi openapi-verify spectral typecheck .DEFAULT_GOAL := check @@ -35,13 +35,16 @@ openapi-verify: $(PYTHON) scripts/export_openapi.py --out $$tmp && \ diff -u $(OPENAPI_FILE) $$tmp -check: lint test openapi-verify +check: lint test typecheck openapi-verify ready: check test: $(PYTHON) -m unittest discover -s tests +typecheck: + $(PYTHON) -m pyright app tests + build: docker build -t $(IMAGE) . diff --git a/README.md b/README.md index bf51938..d24b74a 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,7 @@ Exception: - Browser profile/session artifacts are request-scoped only. - No cache/profile/driver reuse across requests. - Cleanup is enforced in `finally`: driver close + runtime directory delete + request-id in-memory state scrub. +- Before each scrape, orphaned runtime dirs (not tied to an active request id) are pruned; ENOSPC triggers an extra prune-and-retry. Mount `/tmp/scrape` on tmpfs in production (see html2rss-web `docker-compose.yml`). ## Environment Variables @@ -321,20 +322,21 @@ Use a **separate Sentry project** from html2rss-web (`BOTASAURUS_SENTRY_DSN` → | Variable | Default | Description | | :--- | :--- | :--- | -| `SENTRY_DSN` | _(unset)_ | Project DSN. | -| `SENTRY_ENVIRONMENT` | `production` | Deployment tag (`ENVIRONMENT` fallback). | -| `SENTRY_RELEASE` | _(unset)_ | Release tag on events. | -| `SENTRY_TRACES_SAMPLE_RATE` | `0.0` | APM traces (off by default; enable later if needed). | -| `SENTRY_PROFILES_SAMPLE_RATE` | `0.0` | Profiling sample rate. | -| `SENTRY_SEND_DEFAULT_PII` | `false` | Send default PII when `true`. | +| `SENTRY_DSN` | _(unset)_ | Project DSN (`Settings.sentry.dsn`). | +| `SENTRY_ENVIRONMENT` | `production` | Deployment tag (`Settings.sentry.environment`; `ENVIRONMENT` fallback). | +| `SENTRY_RELEASE` | _(unset)_ | Release tag on events (`Settings.sentry.release`). | +| `SENTRY_TRACES_SAMPLE_RATE` | `0.0` | APM traces (`Settings.sentry.traces_sample_rate`; off by default). | +| `SENTRY_PROFILES_SAMPLE_RATE` | `0.0` | Profiling sample rate (`Settings.sentry.profiles_sample_rate`). | +| `SENTRY_SEND_DEFAULT_PII` | `false` | Send default PII when `true` (`Settings.sentry.send_default_pii`). | **Signal routing:** `navigation_error` and `timeout` → grouped Sentry Issues. `challenge_block` → `scrape.challenge_block` metric only; engine stdout keeps the detailed log line. Traces stay off unless you raise `SENTRY_TRACES_SAMPLE_RATE`. | Variable | Default | Description | | :--- | :--- | :--- | -| `SCRAPE_MAX_WORKERS` | `4` | Threadpool worker limit for sync browser execution. | +| `SCRAPE_MAX_WORKERS` | `4` | Threadpool worker limit for sync browser execution. On low-RAM hosts (for example Docker with `--memory=768m` or a 1–2 vCPU VM), use `1` or `2` to limit concurrent Chromium boots; higher values increase queue wait and swap pressure without improving wall time. | | `SCRAPE_TIMEOUT_SECONDS` | `45` | Handler wall-clock budget in seconds (queue, browser boot, and work). | | `SCRAPE_WORK_TIMEOUT_SECONDS` | `30` | Post-boot navigate, selector wait, and scroll budget in seconds. | +| `SCRAPE_RUNTIME_MIN_FREE_BYTES` | `268435456` (256 MiB) | Prune orphan runtime dirs when free space drops below this threshold. | ## Example Calls diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..53fce10 --- /dev/null +++ b/app/__init__.py @@ -0,0 +1 @@ +"""Botasaurus scrape API application package.""" diff --git a/app/api/__init__.py b/app/api/__init__.py new file mode 100644 index 0000000..5f2c474 --- /dev/null +++ b/app/api/__init__.py @@ -0,0 +1 @@ +"""HTTP route registration.""" diff --git a/app/api/deps.py b/app/api/deps.py new file mode 100644 index 0000000..54dd8f5 --- /dev/null +++ b/app/api/deps.py @@ -0,0 +1,44 @@ +"""FastAPI dependency providers.""" + +from __future__ import annotations + +from concurrent.futures import ThreadPoolExecutor +from typing import Annotated + +from fastapi import Depends, Request + +from app.config import Settings +from app.domain.scrape_service import ScrapeService +from app.engine import ScraperEngine + + +def get_app_settings(request: Request) -> Settings: + return request.app.state.settings + + +SettingsDep = Annotated[Settings, Depends(get_app_settings)] + + +def get_executor(request: Request) -> ThreadPoolExecutor: + return request.app.state.executor + + +ExecutorDep = Annotated[ThreadPoolExecutor, Depends(get_executor)] + + +def get_engine(request: Request) -> ScraperEngine: + return request.app.state.engine + + +EngineDep = Annotated[ScraperEngine, Depends(get_engine)] + + +def get_scrape_service( + settings: SettingsDep, + engine: EngineDep, + executor: ExecutorDep, +) -> ScrapeService: + return ScrapeService(settings=settings, engine=engine, executor=executor) + + +ScrapeServiceDep = Annotated[ScrapeService, Depends(get_scrape_service)] diff --git a/app/api/errors.py b/app/api/errors.py new file mode 100644 index 0000000..23f8a0c --- /dev/null +++ b/app/api/errors.py @@ -0,0 +1,111 @@ +"""HTTP exception handlers returning scrape error envelopes.""" + +from __future__ import annotations + +from typing import Any, TypedDict, cast +from urllib.parse import urlparse + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from app.infra.request_id import resolve_request_id +from app.logging_config import get_logger +from app.schemas.response import ScrapeError, ScrapeSuccess, validation_error + +logger = get_logger() + +_NON_FIELD_LOC = {"body", "query", "path", "header"} + + +class ValidationErrorItem(TypedDict, total=False): + loc: tuple[Any, ...] | list[Any] + msg: str + type: str + input: Any + + +def schema_field_from_loc(loc: tuple[Any, ...] | list[Any]) -> str: + for part in loc: + if part not in _NON_FIELD_LOC: + return str(part) + return str(loc[-1]) if loc else "unknown" + + +def first_schema_field(errors: list[ValidationErrorItem]) -> str: + if not errors: + return "unknown" + return schema_field_from_loc(errors[0].get("loc") or ()) + + +def url_from_validation_body(body: Any) -> str: + if isinstance(body, dict): + url_value = cast(dict[str, Any], body).get("url") + if url_value is not None: + return str(url_value) + return "" + + +def validation_error_message(errors: list[ValidationErrorItem]) -> str: + if not errors: + return "Request schema validation failed" + parts: list[str] = [] + for err in errors: + loc = err.get("loc") or () + field = schema_field_from_loc(loc) + message = str(err.get("msg") or "invalid") + parts.append(f"{field}: {message}") + return "; ".join(parts) + + +def json_response( + body: ScrapeSuccess | ScrapeError, *, status_code: int +) -> JSONResponse: + return JSONResponse(status_code=status_code, content=body.model_dump(mode="json")) + + +async def request_schema_validation_handler( + request: Request, exc: RequestValidationError +) -> JSONResponse: + errors: list[ValidationErrorItem] = list(exc.errors()) # type: ignore[arg-type] + url = url_from_validation_body(exc.body) + field = first_schema_field(errors) + request_id, _ = resolve_request_id( + request.headers.get("X-Request-Id"), + host=urlparse(url).hostname if url else None, + ) + logger.info( + "request_schema_422 host=%s field=%s", + urlparse(url).hostname if url else None, + field, + ) + return json_response( + validation_error( + url, + validation_error_message(errors), + request_id=request_id, + ), + status_code=422, + ) + + +async def unhandled_exception_handler(request: Request, exc: Exception) -> JSONResponse: + del exc + logger.exception("unhandled_exception path=%s", request.url.path) + request_id, _ = resolve_request_id(request.headers.get("X-Request-Id")) + return json_response( + validation_error( + "", + "Internal server error", + request_id=request_id, + ), + status_code=500, + ) + + +def register_exception_handlers(app: FastAPI) -> None: + app.add_exception_handler( + RequestValidationError, + cast(Any, request_schema_validation_handler), + ) + app.add_exception_handler(Exception, unhandled_exception_handler) diff --git a/app/api/openapi.py b/app/api/openapi.py new file mode 100644 index 0000000..f803f15 --- /dev/null +++ b/app/api/openapi.py @@ -0,0 +1,187 @@ +"""OpenAPI metadata shared by route modules.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, cast + +from app.api.openapi_examples import ( + HEALTH_EXAMPLE, + SCRAPE_ERROR_EXAMPLE, + SCRAPE_SUCCESS_EXAMPLE, +) +from app.config import Settings +from app.schemas.response import ScrapeError + + +@dataclass(frozen=True, slots=True) +class OpenApiMetadata: + api_description: str + json_error_example: dict[str, dict[str, dict[str, Any]]] + scrape_error_responses: dict[int | str, dict[str, Any]] + health_responses: dict[int | str, dict[str, Any]] + scrape_success_response: dict[int | str, dict[str, Any]] + openapi_tags: list[dict[str, str]] + servers: list[dict[str, str]] + contact: dict[str, str] + license_info: dict[str, str] + + +def build_openapi_metadata(settings: Settings) -> OpenApiMetadata: + work_timeout = settings.scrape_work_timeout_seconds + api_description = f""" +Docker-first scrape API that uses Botasaurus to fetch rendered HTML. + +- `GET /health` — liveness and detected Botasaurus version +- `POST /scrape` — scrape a public `http`/`https` URL + +`wait_timeout_seconds` values outside `[1, {work_timeout}]` +are **clamped** into that range so scrape still runs; they are not rejected +with 422. + +When `html` is present it is UTF-8-normalized and `headers` `content-type` is +`text/html; charset=utf-8`. + +Localhost, private, link-local, multicast, reserved, and unspecified +destinations are blocked (403). Schema validation failures use this API's +scrape error envelope, not FastAPI `detail`. +""" + json_error_example = {"application/json": {"example": SCRAPE_ERROR_EXAMPLE}} + scrape_error_responses = { + 400: { + "model": ScrapeError, + "description": "URL rejected by validation (scheme, host, or unresolvable target).", + "content": json_error_example, + }, + 403: { + "model": ScrapeError, + "description": "URL blocked by SSRF guardrails (localhost, private, or reserved destination).", + "content": json_error_example, + }, + 422: { + "model": ScrapeError, + "description": "Request schema validation failed. Body is the scrape error envelope, not FastAPI `detail`.", + "content": json_error_example, + }, + 502: { + "model": ScrapeError, + "description": "Scrape execution failure or challenge block after the final attempt.", + "content": { + "application/json": { + "example": { + **SCRAPE_ERROR_EXAMPLE, + "error": "Bot challenge detected (Just a moment...)", + "error_category": "challenge_block", + "diagnostics": { + **SCRAPE_ERROR_EXAMPLE["diagnostics"], + "attempts": 3, + "strategy_used": "get", + "render_ms": 1500, + "execution_tier": "browser_driver", + "challenge": { + "blocked": True, + "detected": True, + "marker": "Just a moment...", + }, + }, + } + } + }, + }, + 504: { + "model": ScrapeError, + "description": "Scrape timed out before a result was produced.", + "content": { + "application/json": { + "example": { + **SCRAPE_ERROR_EXAMPLE, + "error": ( + f"Scrape timed out after {settings.scrape_timeout_seconds} " + "seconds (phase=work)" + ), + "error_category": "timeout", + "diagnostics": { + **SCRAPE_ERROR_EXAMPLE["diagnostics"], + "attempts": 1, + "strategy_used": "get", + "render_ms": 45012, + "execution_tier": "browser_driver", + "timeout_phase": "work", + }, + } + } + }, + }, + } + return OpenApiMetadata( + api_description=api_description, + json_error_example=json_error_example, + scrape_error_responses=cast( + dict[int | str, dict[str, Any]], scrape_error_responses + ), + health_responses={ + 200: { + "description": "Service is up.", + "content": {"application/json": {"example": HEALTH_EXAMPLE}}, + } + }, + scrape_success_response={ + 200: { + "description": "Rendered HTML plus diagnostics. `html` is UTF-8-normalized.", + "content": {"application/json": {"example": SCRAPE_SUCCESS_EXAMPLE}}, + } + }, + openapi_tags=[ + { + "name": "health", + "description": "Liveness probe and detected Botasaurus package version.", + }, + { + "name": "scrape", + "description": "Render a public URL and return UTF-8 HTML plus diagnostics.", + }, + ], + servers=[ + { + "url": "http://localhost:4010", + "description": "Local Docker (make serve)", + } + ], + contact={ + "name": "html2rss", + "url": "https://github.com/html2rss/botasaurus-scrape-api/issues", + }, + license_info={ + "name": "MIT", + "url": "https://opensource.org/licenses/MIT", + }, + ) + + +_registry: OpenApiMetadata | None = None + + +def configure_openapi(settings: Settings) -> OpenApiMetadata: + global _registry + _registry = build_openapi_metadata(settings) + return _registry + + +def get_openapi_metadata() -> OpenApiMetadata: + if _registry is None: + from app.config import get_settings + + return configure_openapi(get_settings()) + return _registry + + +def get_scrape_error_responses() -> dict[int | str, dict[str, Any]]: + return get_openapi_metadata().scrape_error_responses + + +def get_scrape_success_response() -> dict[int | str, dict[str, Any]]: + return get_openapi_metadata().scrape_success_response + + +def get_health_responses() -> dict[int | str, dict[str, Any]]: + return get_openapi_metadata().health_responses diff --git a/app/api/openapi_examples.py b/app/api/openapi_examples.py new file mode 100644 index 0000000..91431eb --- /dev/null +++ b/app/api/openapi_examples.py @@ -0,0 +1,66 @@ +"""OpenAPI response examples built from Pydantic model instances.""" + +from __future__ import annotations + +from typing import Any + +from app.constants import SERVICE_NAME +from app.schemas.enums import ErrorCategory, ExecutionTier +from app.schemas.response import ( + ChallengeSignal, + HealthResponse, + ScrapeDiagnostics, + ScrapeError, + ScrapeSuccess, +) + +OpenApiExampleDict = dict[str, Any] + + +def build_scrape_success_example() -> OpenApiExampleDict: + return ScrapeSuccess( + url="https://example.com", + final_url="https://example.com/", + status_code=200, + headers={"content-type": "text/html; charset=utf-8"}, + html="Example Domain", + metadata_error=None, + xhr_responses=[], + diagnostics=ScrapeDiagnostics( + request_id="b01ef2f8-f641-4e75-8ef2-0b73f7b4f372", + attempts=1, + strategy_used=None, + render_ms=154, + execution_tier=ExecutionTier.HTTP_REQUEST, + challenge=ChallengeSignal(blocked=False, detected=False, marker=None), + ), + ).model_dump(mode="json") + + +def build_scrape_error_example() -> OpenApiExampleDict: + return ScrapeError( + url="https://example.com", + error="Target URL is blocked", + error_category=ErrorCategory.VALIDATION, + diagnostics=ScrapeDiagnostics( + request_id="b01ef2f8-f641-4e75-8ef2-0b73f7b4f372", + attempts=0, + strategy_used=None, + render_ms=0, + execution_tier=None, + challenge=None, + ), + ).model_dump(mode="json") + + +def build_health_example() -> OpenApiExampleDict: + return HealthResponse( + status="ok", + service=SERVICE_NAME, + botasaurus_version="4.0.91", + ).model_dump(mode="json") + + +SCRAPE_SUCCESS_EXAMPLE = build_scrape_success_example() +SCRAPE_ERROR_EXAMPLE = build_scrape_error_example() +HEALTH_EXAMPLE = build_health_example() diff --git a/app/api/routes/__init__.py b/app/api/routes/__init__.py new file mode 100644 index 0000000..276674e --- /dev/null +++ b/app/api/routes/__init__.py @@ -0,0 +1 @@ +"""Versioned HTTP route modules.""" diff --git a/app/api/routes/health.py b/app/api/routes/health.py new file mode 100644 index 0000000..0fe920b --- /dev/null +++ b/app/api/routes/health.py @@ -0,0 +1,42 @@ +"""Health probe routes.""" + +from __future__ import annotations + +from importlib.metadata import PackageNotFoundError, version + +from fastapi import APIRouter + +from app.api.openapi import get_health_responses +from app.constants import SERVICE_NAME +from app.schemas.response import HealthResponse + + +def _health() -> HealthResponse: + try: + botasaurus_version = version("botasaurus") + except PackageNotFoundError: + botasaurus_version = "unknown" + + return HealthResponse( + status="ok", + service=SERVICE_NAME, + botasaurus_version=botasaurus_version, + ) + + +def create_router() -> APIRouter: + """Build the health router after OpenAPI metadata is configured.""" + router = APIRouter(tags=["health"]) + router.add_api_route( + "/health", + _health, + methods=["GET"], + response_model=HealthResponse, + operation_id="get-health", + summary="Health", + description=( + "Return liveness status, service name, and the installed Botasaurus version." + ), + responses=get_health_responses(), + ) + return router diff --git a/app/api/routes/scrape.py b/app/api/routes/scrape.py new file mode 100644 index 0000000..9bd2440 --- /dev/null +++ b/app/api/routes/scrape.py @@ -0,0 +1,40 @@ +"""Scrape endpoint routes.""" + +from __future__ import annotations + +from fastapi import APIRouter, Header +from fastapi.responses import JSONResponse + +from app.api.deps import ScrapeServiceDep +from app.api.errors import json_response +from app.api.openapi import get_scrape_error_responses, get_scrape_success_response +from app.schemas.request import ScrapeRequest +from app.schemas.response import ScrapeSuccess + + +async def _scrape( + payload: ScrapeRequest, + service: ScrapeServiceDep, + x_request_id: str | None = Header(None, alias="X-Request-Id"), +) -> JSONResponse: + outcome = await service.process(payload, inbound_request_id=x_request_id) + return json_response(outcome.body, status_code=outcome.status_code) + + +def create_router() -> APIRouter: + """Build the scrape router after OpenAPI metadata is configured.""" + router = APIRouter(tags=["scrape"]) + router.add_api_route( + "/scrape", + _scrape, + methods=["POST"], + response_model=ScrapeSuccess, + responses={**get_scrape_success_response(), **get_scrape_error_responses()}, + operation_id="scrape-url", + summary="Scrape a URL", + description=( + "Fetch rendered HTML for a public http(s) URL. Invalid or blocked " + "targets return `ScrapeError`. `wait_timeout_seconds` is clamped, not 422." + ), + ) + return router diff --git a/app/config.py b/app/config.py new file mode 100644 index 0000000..f59c0c9 --- /dev/null +++ b/app/config.py @@ -0,0 +1,93 @@ +"""Centralized runtime configuration loaded from environment variables.""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from pydantic import Field, field_validator, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class SentrySettings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore") + + dsn: str = Field(default="", validation_alias="SENTRY_DSN") + environment: str = Field(default="", validation_alias="SENTRY_ENVIRONMENT") + release: str = Field(default="", validation_alias="SENTRY_RELEASE") + traces_sample_rate: float = Field( + default=0.0, validation_alias="SENTRY_TRACES_SAMPLE_RATE" + ) + profiles_sample_rate: float = Field( + default=0.0, validation_alias="SENTRY_PROFILES_SAMPLE_RATE" + ) + send_default_pii: bool = Field( + default=False, validation_alias="SENTRY_SEND_DEFAULT_PII" + ) + + @field_validator( + "traces_sample_rate", + "profiles_sample_rate", + mode="before", + ) + @classmethod + def parse_sample_rate(cls, value: object) -> float: + # Env values are best-effort: invalid floats disable sampling + # instead of failing service startup. + try: + parsed = float(str(value).strip()) if value is not None else 0.0 + except ValueError: + return 0.0 + return max(0.0, min(1.0, parsed)) + + @field_validator("send_default_pii", mode="before") + @classmethod + def parse_bool(cls, value: object) -> bool: + if isinstance(value, bool): + return value + return str(value or "").strip().lower() in {"true", "1", "yes", "on"} + + def effective_environment(self, deployment_environment: str) -> str: + return (self.environment or deployment_environment or "production").strip() + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", extra="ignore") + + scrape_timeout_seconds: int = Field( + default=45, validation_alias="SCRAPE_TIMEOUT_SECONDS" + ) + scrape_work_timeout_seconds: int = Field( + default=30, validation_alias="SCRAPE_WORK_TIMEOUT_SECONDS" + ) + scrape_max_workers: int = Field(default=4, validation_alias="SCRAPE_MAX_WORKERS") + scrape_runtime_min_free_bytes: int = Field( + default=256 * 1024 * 1024, + validation_alias="SCRAPE_RUNTIME_MIN_FREE_BYTES", + ) + runtime_root: Path = Field(default=Path("/tmp/scrape")) + environment: str = Field(default="production", validation_alias="ENVIRONMENT") + sentry: SentrySettings = Field(default_factory=SentrySettings) + + @model_validator(mode="after") + def validate_timeout_relationship(self) -> Settings: + if self.scrape_work_timeout_seconds > self.scrape_timeout_seconds: + raise ValueError( + "SCRAPE_WORK_TIMEOUT_SECONDS cannot exceed SCRAPE_TIMEOUT_SECONDS: " + f"work={self.scrape_work_timeout_seconds} " + f"total={self.scrape_timeout_seconds}" + ) + return self + + @property + def default_wait_timeout_seconds(self) -> int: + return min(15, self.scrape_work_timeout_seconds) + + +@lru_cache +def get_settings() -> Settings: + return Settings() + + +def reset_settings_cache() -> None: + get_settings.cache_clear() diff --git a/app/constants.py b/app/constants.py new file mode 100644 index 0000000..e6ccd09 --- /dev/null +++ b/app/constants.py @@ -0,0 +1,5 @@ +"""Shared application constants.""" + +from __future__ import annotations + +SERVICE_NAME = "botasaurus-scrape-api" diff --git a/app/domain/__init__.py b/app/domain/__init__.py new file mode 100644 index 0000000..24bec66 --- /dev/null +++ b/app/domain/__init__.py @@ -0,0 +1 @@ +"""Application services orchestrating validation, execution, and response mapping.""" diff --git a/app/domain/scrape_service.py b/app/domain/scrape_service.py new file mode 100644 index 0000000..1049a57 --- /dev/null +++ b/app/domain/scrape_service.py @@ -0,0 +1,212 @@ +"""Scrape request orchestration between HTTP boundary and execution engine.""" + +from __future__ import annotations + +import asyncio +import time +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass +from functools import partial +from urllib.parse import urlparse + +from app.config import Settings +from app.engine import ScraperEngine +from app.engine.budget import elapsed_ms +from app.exceptions import RequestIdCollisionError +from app.infra.ops_telemetry import emit_terminal_telemetry +from app.infra.request_id import resolve_request_id +from app.infra.scrape_progress import ScrapeProgress +from app.logging_config import get_logger +from app.schemas.enums import ErrorCategory, TimeoutPhase +from app.schemas.request import ScrapeRequest +from app.schemas.response import ( + ScrapeDiagnostics, + ScrapeError, + ScrapeSuccess, + validation_error, +) +from app.security import UrlGuard, ValidationResult + +logger = get_logger() + + +@dataclass(frozen=True, slots=True) +class ScrapeOutcome: + body: ScrapeSuccess | ScrapeError + status_code: int + + +class ScrapeService: + """Owns request-id resolution, URL guardrails, threadpool execution, status mapping, and telemetry.""" + + def __init__( + self, + *, + settings: Settings, + engine: ScraperEngine, + executor: ThreadPoolExecutor, + ) -> None: + self.settings = settings + self.engine = engine + self.executor = executor + + async def process( + self, + payload: ScrapeRequest, + *, + inbound_request_id: str | None = None, + ) -> ScrapeOutcome: + """Resolve the request id, enforce SSRF guardrails, then execute.""" + target_url = str(payload.url) + request_id, _ = resolve_request_id( + inbound_request_id, host=urlparse(target_url).hostname + ) + blocked = self._guard_outcome(payload, target_url, request_id=request_id) + if blocked is not None: + return blocked + return await self._run(payload, request_id=request_id) + + def _guard_outcome( + self, + payload: ScrapeRequest, + target_url: str, + *, + request_id: str, + ) -> ScrapeOutcome | None: + target_validation = UrlGuard.validate(target_url) + if not target_validation.is_allowed: + return self._validation_outcome( + target_url, + target_validation, + request_id=request_id, + default_message="Target URL is blocked", + ) + if payload.proxy: + proxy_validation = UrlGuard.validate_proxy(str(payload.proxy)) + if not proxy_validation.is_allowed: + return self._validation_outcome( + target_url, + proxy_validation, + request_id=request_id, + default_message="Proxy URL is invalid or blocked", + ) + return None + + @staticmethod + def _validation_outcome( + url: str, + validation: ValidationResult, + *, + request_id: str, + default_message: str, + ) -> ScrapeOutcome: + return ScrapeOutcome( + body=validation_error( + url, + validation.error_message or default_message, + request_id=request_id, + ), + status_code=validation.status_code, + ) + + @staticmethod + def build_timeout_error( + url: str, + *, + request_id: str, + started_monotonic: float, + progress: ScrapeProgress, + timeout_seconds: int, + ) -> ScrapeError: + snap = progress.snapshot() + phase = snap.phase + render_ms = elapsed_ms(started_monotonic) + return ScrapeError( + url=url, + error=( + f"Scrape timed out after {timeout_seconds} seconds (phase={phase.value})" + ), + error_category=ErrorCategory.TIMEOUT, + diagnostics=ScrapeDiagnostics( + request_id=request_id, + attempts=snap.attempts, + strategy_used=snap.strategy_used, + render_ms=render_ms, + execution_tier=snap.execution_tier, + timeout_phase=phase, + ), + ) + + async def _run( + self, + payload: ScrapeRequest, + *, + request_id: str, + ) -> ScrapeOutcome: + target_url = str(payload.url) + started_monotonic = time.monotonic() + deadline_monotonic = started_monotonic + self.settings.scrape_timeout_seconds + progress = ScrapeProgress() + + try: + loop = asyncio.get_running_loop() + result = await asyncio.wait_for( + loop.run_in_executor( + self.executor, + partial( + self.engine.execute, + payload, + deadline_monotonic, + request_id=request_id, + progress=progress, + ), + ), + timeout=self.settings.scrape_timeout_seconds, + ) + except RequestIdCollisionError: + collision_result = ScrapeError( + url=target_url, + error="Request id collision detected", + error_category=ErrorCategory.NAVIGATION_ERROR, + diagnostics=ScrapeDiagnostics( + request_id=request_id, + attempts=0, + render_ms=0, + ), + ) + emit_terminal_telemetry(collision_result, http_status=502) + return ScrapeOutcome(body=collision_result, status_code=502) + except TimeoutError: + timeout_result = self.build_timeout_error( + target_url, + request_id=request_id, + started_monotonic=started_monotonic, + progress=progress, + timeout_seconds=self.settings.scrape_timeout_seconds, + ) + phase = timeout_result.diagnostics.timeout_phase or TimeoutPhase.QUEUE + logger.warning( + "scrape_timeout host=%s mode=%s timeout_seconds=%d phase=%s attempts=%d", + urlparse(target_url).hostname, + payload.navigation_mode, + self.settings.scrape_timeout_seconds, + phase.value, + timeout_result.diagnostics.attempts, + ) + emit_terminal_telemetry(timeout_result, http_status=504) + return ScrapeOutcome(body=timeout_result, status_code=504) + + status_code = 200 if isinstance(result, ScrapeSuccess) else 502 + if isinstance(result, ScrapeError): + emit_terminal_telemetry(result, http_status=status_code) + logger.info( + "scrape_complete request_id=%s host=%s mode=%s tier=%s attempts=%s status=%d error_category=%s", + result.diagnostics.request_id, + urlparse(target_url).hostname, + payload.navigation_mode, + result.diagnostics.execution_tier, + result.diagnostics.attempts, + status_code, + result.error_category if isinstance(result, ScrapeError) else None, + ) + return ScrapeOutcome(body=result, status_code=status_code) diff --git a/app/engine.py b/app/engine.py deleted file mode 100644 index 70cda03..0000000 --- a/app/engine.py +++ /dev/null @@ -1,756 +0,0 @@ -# app/engine.py -from __future__ import annotations - -import logging -import shutil -import threading -import time -import uuid -from pathlib import Path -from typing import Any -from urllib.parse import urlparse - -from botasaurus.browser import Driver -from botasaurus.request import Request - -from app.detector import ChallengeAssessment, ChallengeDetector -from app.metadata import MetadataExtractor -from app.schemas import ( - DEFAULT_SCRAPE_TIMEOUT_SECONDS, - DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS, - ChallengeSignal, - ErrorCategory, - ExecutionMode, - ExecutionTier, - NavigationMode, - ScrapeDiagnostics, - ScrapeError, - ScrapeRequest, - ScrapeSuccess, - TimeoutPhase, - XhrResponse, -) -from app.scrape_progress import ScrapeProgress -from app.xhr_collector import XhrCollector - -logger = logging.getLogger("botasaurus_scrape_api") - - -def _remaining_total_seconds(started_monotonic: float) -> int: - return max( - 1, - int(DEFAULT_SCRAPE_TIMEOUT_SECONDS - (time.monotonic() - started_monotonic)), - ) - - -def _remaining_work_seconds(browser_ready_monotonic: float) -> int: - return max( - 1, - int( - DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS - - (time.monotonic() - browser_ready_monotonic) - ), - ) - - -def _browser_step_budget_seconds( - started_monotonic: float, browser_ready_monotonic: float -) -> int: - return min( - _remaining_total_seconds(started_monotonic), - _remaining_work_seconds(browser_ready_monotonic), - ) - - -_RUNTIME_ROOT = Path("/tmp/scrape") - -_TRACKER_URL_PATTERNS: list[str] = [ - "*google-analytics.com*", - "*googletagmanager.com*", - "*facebook.net*", - "*doubleclick.net*", - "*sentry.io*", - "*hotjar.com*", - "*clarity.ms*", - "*datadoghq-browser-agent.com*", - "*segment.io*", - "*analytics.js*", - "*.woff", - "*.woff2", - "*.ttf", -] - - -HTML_DOCUMENT_CONTENT_TYPE = "text/html; charset=utf-8" - - -def utf8_normalize_html(html: str) -> str: - if not html: - return html - if not isinstance(html, str): - html = str(html) - try: - html = html.encode("latin-1").decode("utf-8") - except (UnicodeEncodeError, UnicodeDecodeError): # fmt: skip - pass - return html.encode("utf-8", errors="replace").decode("utf-8") - - -def html_document_headers( - html: str, headers: dict[str, str] | None -) -> tuple[str, dict[str, str] | None]: - if not html: - return html, headers - normalized = utf8_normalize_html(html) - out: dict[str, str] = {} - for key, value in (headers or {}).items(): - if str(key).lower() == "content-type": - continue - out[str(key)] = str(value) - out["content-type"] = HTML_DOCUMENT_CONTENT_TYPE - return normalized, out - - -def _diagnostics( - *, - request_id: str, - attempts: int = 0, - strategy_used: NavigationMode | None = None, - render_ms: int = 0, - execution_tier: ExecutionTier | None = None, - assessment: ChallengeAssessment | None = None, - timeout_phase: TimeoutPhase | None = None, -) -> ScrapeDiagnostics: - challenge = None - if assessment is not None: - challenge = ChallengeSignal( - blocked=assessment.blocked_detected, - detected=assessment.challenge_detected, - marker=assessment.detected_marker, - ) - return ScrapeDiagnostics( - request_id=request_id, - attempts=attempts, - strategy_used=strategy_used, - render_ms=render_ms, - execution_tier=execution_tier, - challenge=challenge, - timeout_phase=timeout_phase, - ) - - -def _success( - url: str, - *, - request_id: str, - html: str, - attempts: int, - render_ms: int, - execution_tier: ExecutionTier, - strategy_used: NavigationMode | None = None, - final_url: str | None = None, - status_code: int | None = 200, - headers: dict[str, str] | None = None, - metadata_error: str | None = None, - assessment: ChallengeAssessment | None = None, - xhr_responses: list[dict[str, Any]] | list[XhrResponse] | None = None, -) -> ScrapeSuccess: - html, headers = html_document_headers(html, headers) - return ScrapeSuccess( - url=url, - final_url=final_url or url, - status_code=status_code, - headers=headers, - html=html, - metadata_error=metadata_error, - xhr_responses=xhr_responses or [], - diagnostics=_diagnostics( - request_id=request_id, - attempts=attempts, - strategy_used=strategy_used, - render_ms=render_ms, - execution_tier=execution_tier, - assessment=assessment, - ), - ) - - -def _error( - url: str, - message: str, - *, - request_id: str, - error_category: ErrorCategory, - attempts: int = 0, - strategy_used: NavigationMode | None = None, - render_ms: int = 0, - execution_tier: ExecutionTier | None = None, - assessment: ChallengeAssessment | None = None, - timeout_phase: TimeoutPhase | None = None, -) -> ScrapeError: - return ScrapeError( - url=url, - error=message, - error_category=error_category, - diagnostics=_diagnostics( - request_id=request_id, - attempts=attempts, - strategy_used=strategy_used, - render_ms=render_ms, - execution_tier=execution_tier, - assessment=assessment, - timeout_phase=timeout_phase, - ), - ) - - -# Semantic alias: terminal outcomes use the same envelope builder. -_terminal_error = _error - - -class ScrapeSession: - """Encapsulates per-request concurrency registration and filesystem isolation.""" - - def __init__(self, engine: ScraperEngine, request_id: str) -> None: - self.engine = engine - self.request_id = request_id - self.runtime_dir = engine.runtime_root / request_id - self.profile_dir = self.runtime_dir / "profile" - self.driver: Driver | None = None - - def __enter__(self) -> ScrapeSession: - self.engine.register_request_id(self.request_id) - return self - - def prepare_profile_dirs(self) -> None: - self.runtime_dir.mkdir(parents=True, exist_ok=False) - self.profile_dir.mkdir(parents=True, exist_ok=False) - - def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: - try: - if self.driver is not None: - try: - self.driver.close() - except Exception: - # Best-effort driver shutdown during cleanup - pass - finally: - shutil.rmtree(self.runtime_dir, ignore_errors=True) - self.engine.unregister_request_id(self.request_id) - - -class ScraperEngine: - """Deep module orchestrating anti-detect HTTP and browser execution tiers.""" - - def __init__(self, runtime_root: Path = _RUNTIME_ROOT) -> None: - self.runtime_root = runtime_root - self._active_request_ids: set[str] = set() - self._active_request_ids_lock = threading.Lock() - - def register_request_id(self, request_id: str) -> None: - with self._active_request_ids_lock: - if request_id in self._active_request_ids: - raise RuntimeError("request id collision detected") - self._active_request_ids.add(request_id) - - def unregister_request_id(self, request_id: str) -> None: - with self._active_request_ids_lock: - self._active_request_ids.discard(request_id) - - @classmethod - def resolve_strategies(cls, mode: NavigationMode, max_retries: int) -> list[str]: - max_attempts = 1 + max_retries - if mode == "auto": - ordered = ["google_get", "google_get_bypass", "get"] - return ordered[: min(len(ordered), max_attempts)] - return [mode] * max_attempts - - @classmethod - def navigate( - cls, driver: Driver, target_url: str, strategy: str, timeout_seconds: int - ) -> None: - if strategy == "organic_get": - method = getattr( - driver, "organic_get", getattr(driver, "google_get", driver.get) - ) - elif strategy.startswith("google_get"): - method = getattr(driver, "google_get", driver.get) - else: - method = driver.get - - kwargs: dict[str, Any] = {} - if strategy == "google_get_bypass": - kwargs["bypass_cloudflare"] = True - try: - method(target_url, timeout=timeout_seconds, **kwargs) - except TypeError: - method(target_url, **kwargs) - - @classmethod - def _configure_driver( - cls, - driver: Driver, - payload: ScrapeRequest, - target_url: str, - collector: XhrCollector | None = None, - ) -> None: - if hasattr(driver, "_tab"): - if collector is not None: - try: - collector.install(driver._tab) - except Exception: - # Best-effort XHR capture; HTML scrape must still proceed - pass - - if payload.block_trackers: - try: - driver._tab.block_urls(_TRACKER_URL_PATTERNS) - except Exception: - # Optional CDP URL blocker feature - pass - - if payload.cookies: - for c_name, c_val in payload.cookies.items(): - try: - driver.add_cookies( - [{"name": str(c_name), "value": str(c_val), "url": target_url}] - ) - except Exception: - # Best-effort cookie initialization - pass - - if payload.headers and hasattr(driver, "_tab"): - try: - driver._tab.set_extra_http_headers(payload.headers) - except Exception: - # Optional CDP extra HTTP headers feature - pass - - @classmethod - def wait_for_readiness( - cls, - driver: Driver, - *, - selector: str | None, - timeout_seconds: int, - ) -> None: - if selector: - driver.wait_for_element(selector, wait=timeout_seconds) - return - - sleep_random_fn = getattr(driver, "sleep_random", None) - if callable(sleep_random_fn): - try: - sleep_random_fn(0.5, 1.2) - return - except Exception: - # Fall back to standard sleep if driver sleep_random fails - pass - driver.sleep(1) - - @classmethod - def apply_scrolling(cls, driver: Driver) -> None: - scroll_bottom_fn = getattr(driver, "scroll_to_bottom", None) - scroll_fn = getattr(driver, "scroll", None) - run_js_fn = getattr(driver, "run_js", None) - - if callable(scroll_bottom_fn): - try: - scroll_bottom_fn() - except Exception: - # Fall back to alternative scrolling if scroll_to_bottom fails - pass - elif callable(scroll_fn): - try: - scroll_fn() - except Exception: - # Fall back to JS scroll if scroll fails - pass - elif callable(run_js_fn): - try: - run_js_fn("window.scrollTo(0, document.body.scrollHeight);") - except Exception: - # Fall back to execute_script if run_js fails - pass - elif hasattr(driver, "execute_script"): - try: - driver.execute_script("window.scrollTo(0, document.body.scrollHeight);") - except Exception: - # Best-effort JS scroll execution - pass - - sleep_random_fn = getattr(driver, "sleep_random", None) - if callable(sleep_random_fn): - try: - sleep_random_fn(0.4, 0.9) - return - except Exception: - # Fall back to standard sleep if sleep_random fails - pass - - try: - driver.sleep(0.5) - except Exception: - # Best-effort post-scroll timing wait - pass - - @staticmethod - def _harvest_xhr(collector: XhrCollector, driver: Driver) -> list[dict[str, Any]]: - tab = getattr(driver, "_tab", None) - if tab is None: - return collector.results() - try: - return collector.harvest(tab) - except Exception as exc: - logger.debug("xhr_harvest_failed error=%s", str(exc)) - return collector.results() - - def run_request_tier( - self, - payload: ScrapeRequest, - request_id: str, - started_monotonic: float, - progress: ScrapeProgress, - ) -> ScrapeSuccess | ScrapeError | None: - target_url = str(payload.url) - remaining_budget = _remaining_total_seconds(started_monotonic) - progress.mark( - TimeoutPhase.WORK, - attempts=1, - execution_tier=ExecutionTier.HTTP_REQUEST, - ) - - req_headers = dict(payload.headers) if payload.headers else {} - proxies = ( - {"http": payload.proxy, "https": payload.proxy} if payload.proxy else None - ) - - req = Request() - try: - resp = req.get( - target_url, - headers=req_headers if req_headers else None, - cookies=payload.cookies, - user_agent=payload.effective_user_agent, - proxies=proxies, - timeout=remaining_budget, - browser="chrome", - allow_redirects=True, - ) - - html = resp.text or "" - status_code = int(resp.status_code) if resp.status_code is not None else 200 - headers_dict = ( - {str(k): str(v) for k, v in resp.headers.items()} - if getattr(resp, "headers", None) - else None - ) - final_url = str(resp.url) if getattr(resp, "url", None) else target_url - - assessment = ChallengeDetector.detect(html, status_code) - render_ms = int((time.monotonic() - started_monotonic) * 1000) - - is_clean_success = ( - assessment.is_clean - and (200 <= status_code < 300) - and len(html.strip()) > 0 - and not payload.wait_for_selector - and not payload.scroll - ) - - if payload.execution_mode == ExecutionMode.AUTO and not is_clean_success: - logger.info( - "request_tier_escalating request_id=%s host=%s status=%d blocked=%s challenge=%s", - request_id, - urlparse(target_url).hostname, - status_code, - assessment.blocked_detected, - assessment.challenge_detected, - ) - return None - - if assessment.blocked_detected: - return _terminal_error( - target_url, - "Challenge block detected", - request_id=request_id, - error_category=ErrorCategory.CHALLENGE_BLOCK, - attempts=1, - render_ms=render_ms, - execution_tier=ExecutionTier.HTTP_REQUEST, - assessment=assessment, - ) - - return _success( - target_url, - request_id=request_id, - html=html, - final_url=final_url, - status_code=status_code, - headers=headers_dict, - attempts=1, - render_ms=render_ms, - execution_tier=ExecutionTier.HTTP_REQUEST, - assessment=assessment, - ) - finally: - try: - req.close() - except Exception: - # Best-effort HTTP client cleanup - pass - - def run_browser_tier( - self, - payload: ScrapeRequest, - session: ScrapeSession, - started_monotonic: float, - progress: ScrapeProgress, - ) -> ScrapeSuccess | ScrapeError: - target_url = str(payload.url) - request_id = session.request_id - progress.mark( - TimeoutPhase.BOOT, - execution_tier=ExecutionTier.BROWSER_DRIVER, - ) - session.prepare_profile_dirs() - - strategies = self.resolve_strategies( - payload.navigation_mode, payload.max_retries - ) - attempts = 0 - collector = XhrCollector(target_url) - driver_window_size = ( - [payload.window_size.width, payload.window_size.height] - if payload.window_size - else None - ) - - session.driver = Driver( - headless=payload.headless, - enable_xvfb_virtual_display=not payload.headless, - proxy=payload.proxy, - profile=str(session.profile_dir), - tiny_profile=True, - block_images=payload.block_images, - block_images_and_css=payload.block_images_and_css, - wait_for_complete_page_load=payload.wait_for_complete_page_load, - user_agent=payload.effective_user_agent, - window_size=driver_window_size, - lang=payload.lang, - remove_default_browser_check_argument=True, - ) - self._configure_driver(session.driver, payload, target_url, collector=collector) - browser_ready_monotonic = time.monotonic() - progress.mark( - TimeoutPhase.WORK, - execution_tier=ExecutionTier.BROWSER_DRIVER, - ) - - for attempt_index, strategy in enumerate(strategies, start=1): - attempts = attempt_index - progress.mark( - TimeoutPhase.WORK, - attempts=attempts, - strategy_used=NavigationMode(strategy), - execution_tier=ExecutionTier.BROWSER_DRIVER, - ) - try: - step_budget = _browser_step_budget_seconds( - started_monotonic, browser_ready_monotonic - ) - self.navigate(session.driver, target_url, strategy, step_budget) - self.wait_for_readiness( - session.driver, - selector=payload.wait_for_selector, - timeout_seconds=min(payload.wait_timeout_seconds, step_budget), - ) - - if payload.scroll: - self.apply_scrolling(session.driver) - - xhr_responses = self._harvest_xhr(collector, session.driver) - - html = session.driver.page_html or "" - meta = MetadataExtractor.fetch(session.driver, target_url) - assessment = ChallengeDetector.detect( - html, meta.status_code, driver=session.driver - ) - - if assessment.challenge_detected or assessment.blocked_detected: - bypass_fn = getattr(session.driver, "bypass_cloudflare", None) - if callable(bypass_fn): - try: - bypass_fn() - html = session.driver.page_html or "" - meta = MetadataExtractor.fetch(session.driver, target_url) - assessment = ChallengeDetector.detect( - html, meta.status_code, driver=session.driver - ) - xhr_responses = self._harvest_xhr(collector, session.driver) - except Exception as exc: - logger.debug( - "bypass_cloudflare_attempt_failed error=%s", str(exc) - ) - - if assessment.challenge_detected or assessment.blocked_detected: - logger.warning( - "scrape_challenge_detected request_id=%s host=%s strategy=%s attempt=%d marker=%s", - request_id, - urlparse(target_url).hostname, - strategy, - attempt_index, - assessment.detected_marker, - ) - if attempt_index < len(strategies): - # Drop interstitial JSON from the failed attempt so it - # cannot pollute the next strategy's xhr_responses/cap. - collector.reset() - continue - - render_ms = int((time.monotonic() - started_monotonic) * 1000) - return _terminal_error( - target_url, - f"Bot challenge detected ({assessment.detected_marker or 'unknown'})", - request_id=request_id, - error_category=ErrorCategory.CHALLENGE_BLOCK, - attempts=attempts, - strategy_used=NavigationMode(strategy), - render_ms=render_ms, - execution_tier=ExecutionTier.BROWSER_DRIVER, - assessment=assessment, - ) - - render_ms = int((time.monotonic() - started_monotonic) * 1000) - return _success( - target_url, - request_id=request_id, - html=html, - final_url=meta.final_url, - status_code=meta.status_code, - headers=meta.headers, - metadata_error=meta.metadata_error, - attempts=attempts, - strategy_used=NavigationMode(strategy), - render_ms=render_ms, - execution_tier=ExecutionTier.BROWSER_DRIVER, - assessment=assessment, - xhr_responses=xhr_responses, - ) - except Exception as exc: - logger.warning( - "scrape_attempt_failed request_id=%s host=%s mode=%s strategy=%s attempt=%d error=%s", - request_id, - urlparse(target_url).hostname, - payload.navigation_mode, - strategy, - attempt_index, - str(exc), - ) - if attempt_index < len(strategies): - collector.reset() - continue - - render_ms = int((time.monotonic() - started_monotonic) * 1000) - is_timeout = "timeout" in str(exc).lower() - category = ( - ErrorCategory.TIMEOUT - if is_timeout - else ErrorCategory.NAVIGATION_ERROR - ) - return _terminal_error( - target_url, - str(exc), - request_id=request_id, - attempts=attempts, - strategy_used=NavigationMode(strategy), - render_ms=render_ms, - error_category=category, - execution_tier=ExecutionTier.BROWSER_DRIVER, - timeout_phase=TimeoutPhase.WORK if is_timeout else None, - ) - - render_ms = int((time.monotonic() - started_monotonic) * 1000) - return _terminal_error( - target_url, - "Scrape failed after all strategy attempts", - request_id=request_id, - attempts=attempts, - strategy_used=NavigationMode(strategies[-1]) if strategies else None, - render_ms=render_ms, - error_category=ErrorCategory.NAVIGATION_ERROR, - execution_tier=ExecutionTier.BROWSER_DRIVER, - ) - - def execute( - self, - payload: ScrapeRequest, - deadline_monotonic: float | None = None, - *, - request_id: str | None = None, - progress: ScrapeProgress | None = None, - ) -> ScrapeSuccess | ScrapeError: - target_url = str(payload.url) - resolved_request_id = request_id or str(uuid.uuid4()) - started_monotonic = time.monotonic() - progress = progress or ScrapeProgress() - - if deadline_monotonic and started_monotonic >= deadline_monotonic: - progress.mark(TimeoutPhase.QUEUE) - return _terminal_error( - target_url, - "Scrape timed out in threadpool queue before execution started", - request_id=resolved_request_id, - error_category=ErrorCategory.TIMEOUT, - timeout_phase=TimeoutPhase.QUEUE, - ) - - with ScrapeSession(self, resolved_request_id) as session: - should_try_request_tier = ( - payload.execution_mode == ExecutionMode.REQUEST - or ( - payload.execution_mode == ExecutionMode.AUTO - and payload.navigation_mode == NavigationMode.AUTO - and not payload.wait_for_selector - and not payload.scroll - ) - ) - - if should_try_request_tier: - try: - request_result = self.run_request_tier( - payload, - resolved_request_id, - started_monotonic, - progress=progress, - ) - if request_result is not None: - return request_result - except Exception as exc: - logger.info( - "request_tier_failed request_id=%s host=%s error=%s", - resolved_request_id, - urlparse(target_url).hostname, - str(exc), - ) - if payload.execution_mode == ExecutionMode.REQUEST: - render_ms = int((time.monotonic() - started_monotonic) * 1000) - is_timeout = "timeout" in str(exc).lower() - return _terminal_error( - target_url, - str(exc), - request_id=resolved_request_id, - attempts=1, - render_ms=render_ms, - error_category=( - ErrorCategory.TIMEOUT - if is_timeout - else ErrorCategory.NAVIGATION_ERROR - ), - execution_tier=ExecutionTier.HTTP_REQUEST, - timeout_phase=TimeoutPhase.WORK if is_timeout else None, - ) - - return self.run_browser_tier( - payload, session, started_monotonic, progress=progress - ) diff --git a/app/engine/__init__.py b/app/engine/__init__.py new file mode 100644 index 0000000..72a436d --- /dev/null +++ b/app/engine/__init__.py @@ -0,0 +1,12 @@ +"""Scrape engine public surface.""" + +from app.engine.envelope import html_document_headers, utf8_normalize_html +from app.engine.orchestrator import ScraperEngine +from app.engine.session import ScrapeSession + +__all__ = [ + "ScrapeSession", + "ScraperEngine", + "html_document_headers", + "utf8_normalize_html", +] diff --git a/app/engine/browser_tier.py b/app/engine/browser_tier.py new file mode 100644 index 0000000..392d9ae --- /dev/null +++ b/app/engine/browser_tier.py @@ -0,0 +1,258 @@ +"""Chromium browser execution tier.""" + +from __future__ import annotations + +import errno +import time +from typing import cast +from urllib.parse import urlparse + +from app.config import Settings +from app.engine.budget import ( + browser_step_budget_seconds, + elapsed_ms, + is_timeout_exception, +) +from app.engine.driver_capabilities import DriverProtocol, call_if_available +from app.engine.envelope import build_error, build_success +from app.engine.session import ScrapeSession +from app.engine.strategies import ( + apply_scrolling, + configure_driver, + harvest_xhr, + navigate, + resolve_strategies, + wait_for_readiness, +) +from app.infra.detector import ChallengeAssessment, ChallengeDetector +from app.infra.metadata import MetadataExtractor, MetadataResult +from app.infra.scrape_progress import ScrapeProgress +from app.infra.xhr_collector import XhrCollector +from app.logging_config import get_logger +from app.schemas.enums import ErrorCategory, ExecutionTier, TimeoutPhase +from app.schemas.request import ScrapeRequest +from app.schemas.response import ScrapeError, ScrapeSuccess, XhrResponse + +logger = get_logger() + + +def _page_state( + driver: DriverProtocol, target_url: str +) -> tuple[str, MetadataResult, ChallengeAssessment]: + html = driver.page_html or "" + meta = MetadataExtractor.fetch(driver, target_url) + assessment = ChallengeDetector.detect(html, meta.status_code, driver=driver) + return html, meta, assessment + + +def settle_page_state( + driver: DriverProtocol, + target_url: str, + collector: XhrCollector, +) -> tuple[str, MetadataResult, ChallengeAssessment, list[XhrResponse]]: + """Collect final page state, attempting one Cloudflare bypass on challenges. + + XHR harvest runs exactly once per attempt, after any bypass, so the + consolidated pass captures post-bypass sub-resources. + """ + html, meta, assessment = _page_state(driver, target_url) + if assessment.is_clean: + return html, meta, assessment, harvest_xhr(collector, driver) + + call_if_available(driver, "bypass_cloudflare") + xhr_responses = harvest_xhr(collector, driver) + html, meta, assessment = _page_state(driver, target_url) + return html, meta, assessment, xhr_responses + + +def run_browser_tier( + payload: ScrapeRequest, + session: ScrapeSession, + started_monotonic: float, + progress: ScrapeProgress, + *, + settings: Settings, +) -> ScrapeSuccess | ScrapeError: + from botasaurus.browser import Driver + + target_url = str(payload.url) + request_id = session.request_id + progress.mark( + TimeoutPhase.BOOT, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ) + try: + session.prepare_profile_dirs() + except OSError as exc: + if exc.errno == errno.ENOSPC: + detail = "Scrape runtime storage full" + else: + detail = "Scrape runtime storage unavailable" + return build_error( + target_url, + f"{detail}: {exc}", + request_id=request_id, + attempts=0, + render_ms=elapsed_ms(started_monotonic), + error_category=ErrorCategory.NAVIGATION_ERROR, + execution_tier=ExecutionTier.BROWSER_DRIVER, + timeout_phase=TimeoutPhase.BOOT, + ) + + strategies = resolve_strategies(payload.navigation_mode, payload.max_retries) + attempts = 0 + collector = XhrCollector(target_url) + driver_window_size = ( + [payload.window_size.width, payload.window_size.height] + if payload.window_size + else None + ) + + try: + driver = cast( + DriverProtocol, + Driver( + headless=payload.headless, + enable_xvfb_virtual_display=not payload.headless, + proxy=payload.proxy, + profile=str(session.profile_dir), + tiny_profile=True, + block_images=payload.block_images, + block_images_and_css=payload.block_images_and_css, + wait_for_complete_page_load=payload.wait_for_complete_page_load, + user_agent=payload.effective_user_agent, + window_size=driver_window_size, + lang=payload.lang, + remove_default_browser_check_argument=True, + ), + ) + except Exception as exc: + is_timeout = is_timeout_exception(exc) + return build_error( + target_url, + str(exc), + request_id=request_id, + attempts=0, + render_ms=elapsed_ms(started_monotonic), + error_category=( + ErrorCategory.TIMEOUT if is_timeout else ErrorCategory.NAVIGATION_ERROR + ), + execution_tier=ExecutionTier.BROWSER_DRIVER, + timeout_phase=TimeoutPhase.BOOT, + ) + + session.driver = driver + configure_driver(driver, payload, target_url, collector=collector) + browser_ready_monotonic = time.monotonic() + progress.mark( + TimeoutPhase.WORK, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ) + + for attempt_index, strategy in enumerate(strategies, start=1): + attempts = attempt_index + progress.mark( + TimeoutPhase.WORK, + attempts=attempts, + strategy_used=strategy, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ) + try: + step_budget = browser_step_budget_seconds( + settings, started_monotonic, browser_ready_monotonic + ) + navigate(driver, target_url, strategy, step_budget) + wait_for_readiness( + driver, + selector=payload.wait_for_selector, + timeout_seconds=min(payload.wait_timeout_seconds, step_budget), + ) + + if payload.scroll: + apply_scrolling(driver) + + html, meta, assessment, xhr_responses = settle_page_state( + driver, target_url, collector + ) + + if not assessment.is_clean: + logger.warning( + "scrape_challenge_detected request_id=%s host=%s strategy=%s attempt=%d marker=%s", + request_id, + urlparse(target_url).hostname, + strategy.value, + attempt_index, + assessment.detected_marker, + ) + if attempt_index < len(strategies): + collector.reset() + continue + + return build_error( + target_url, + f"Bot challenge detected ({assessment.detected_marker or 'unknown'})", + request_id=request_id, + error_category=ErrorCategory.CHALLENGE_BLOCK, + attempts=attempts, + strategy_used=strategy, + render_ms=elapsed_ms(started_monotonic), + execution_tier=ExecutionTier.BROWSER_DRIVER, + assessment=assessment, + ) + + return build_success( + target_url, + request_id=request_id, + html=html, + final_url=meta.final_url, + status_code=meta.status_code, + headers=meta.headers, + metadata_error=meta.metadata_error, + attempts=attempts, + strategy_used=strategy, + render_ms=elapsed_ms(started_monotonic), + execution_tier=ExecutionTier.BROWSER_DRIVER, + assessment=assessment, + xhr_responses=xhr_responses, + ) + except Exception as exc: + logger.warning( + "scrape_attempt_failed request_id=%s host=%s mode=%s strategy=%s attempt=%d error=%s", + request_id, + urlparse(target_url).hostname, + payload.navigation_mode, + strategy.value, + attempt_index, + str(exc), + ) + if attempt_index < len(strategies): + collector.reset() + continue + + is_timeout = is_timeout_exception(exc) + return build_error( + target_url, + str(exc), + request_id=request_id, + attempts=attempts, + strategy_used=strategy, + render_ms=elapsed_ms(started_monotonic), + error_category=( + ErrorCategory.TIMEOUT + if is_timeout + else ErrorCategory.NAVIGATION_ERROR + ), + execution_tier=ExecutionTier.BROWSER_DRIVER, + timeout_phase=TimeoutPhase.WORK if is_timeout else None, + ) + + return build_error( + target_url, + "Scrape failed after all strategy attempts", + request_id=request_id, + attempts=attempts, + strategy_used=strategies[-1] if strategies else None, + render_ms=elapsed_ms(started_monotonic), + error_category=ErrorCategory.NAVIGATION_ERROR, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ) diff --git a/app/engine/budget.py b/app/engine/budget.py new file mode 100644 index 0000000..9f977c5 --- /dev/null +++ b/app/engine/budget.py @@ -0,0 +1,41 @@ +"""Single owner of scrape wall-clock budget math shared across tiers.""" + +from __future__ import annotations + +import time + +from app.config import Settings + + +def elapsed_ms(started_monotonic: float) -> int: + return int((time.monotonic() - started_monotonic) * 1000) + + +def remaining_total_seconds(settings: Settings, started_monotonic: float) -> int: + return max( + 1, + int(settings.scrape_timeout_seconds - (time.monotonic() - started_monotonic)), + ) + + +def remaining_work_seconds(settings: Settings, browser_ready_monotonic: float) -> int: + return max( + 1, + int( + settings.scrape_work_timeout_seconds + - (time.monotonic() - browser_ready_monotonic) + ), + ) + + +def browser_step_budget_seconds( + settings: Settings, started_monotonic: float, browser_ready_monotonic: float +) -> int: + return min( + remaining_total_seconds(settings, started_monotonic), + remaining_work_seconds(settings, browser_ready_monotonic), + ) + + +def is_timeout_exception(exc: Exception) -> bool: + return "timeout" in str(exc).lower() diff --git a/app/engine/driver_capabilities.py b/app/engine/driver_capabilities.py new file mode 100644 index 0000000..75d5882 --- /dev/null +++ b/app/engine/driver_capabilities.py @@ -0,0 +1,116 @@ +"""Typed optional-driver method adapter for Botasaurus Driver seams.""" + +from __future__ import annotations + +from typing import Any, Protocol, cast, runtime_checkable + +from app.logging_config import get_logger + +logger = get_logger() + + +@runtime_checkable +class DriverTabProtocol(Protocol): + def block_urls(self, patterns: list[str]) -> None: ... + + def set_extra_http_headers(self, headers: dict[str, str]) -> None: ... + + +@runtime_checkable +class CdpTabProtocol(Protocol): + def send(self, cdp_obj: Any) -> Any: ... + + def after_response_received(self, handler: Any, /) -> None: ... + + def add_handler(self, event_type: type[Any], handler: Any, /) -> None: ... + + +@runtime_checkable +class DriverRequestResponseProtocol(Protocol): + status_code: int | None + headers: dict[str, str] | object | None + + +@runtime_checkable +class DriverRequestProtocol(Protocol): + url: str | None + response: DriverRequestResponseProtocol | None + + +@runtime_checkable +class DriverProtocol(Protocol): + page_html: str | None + current_url: str | None + requests: list[DriverRequestProtocol] | tuple[DriverRequestProtocol, ...] | object + + def get(self, url: str, /, **kwargs: Any) -> Any: ... + + def google_get(self, url: str, /, **kwargs: Any) -> Any: ... + + def organic_get(self, url: str, /, **kwargs: Any) -> Any: ... + + def wait_for_element(self, selector: str, /, **kwargs: Any) -> Any: ... + + def sleep(self, seconds: float, /) -> None: ... + + def sleep_random(self, minimum: float, maximum: float, /) -> None: ... + + def scroll_to_bottom(self) -> None: ... + + def scroll(self) -> None: ... + + def run_js(self, script: str, /) -> Any: ... + + def execute_script(self, script: str, /) -> Any: ... + + def add_cookies(self, cookies: list[dict[str, str]]) -> None: ... + + def bypass_cloudflare(self) -> None: ... + + def close(self) -> None: ... + + def get_log(self, log_type: str) -> list[dict[str, str]]: ... + + @property + def _tab(self) -> CdpTabProtocol: ... + + +def resolve_callable( + driver: DriverProtocol | DriverTabProtocol, *names: str +) -> Any | None: + """Return the first callable attribute among ``names``, else ``None``.""" + for name in names: + method = getattr(driver, name, None) + if callable(method): + return method + return None + + +def call_if_available[T]( + driver: DriverProtocol, + name: str, + /, + *args: Any, + default: T = None, # type: ignore[assignment] + **kwargs: Any, +) -> T: + method = resolve_callable(driver, name) + if method is None: + return default + try: + return cast(T, method(*args, **kwargs)) + except Exception as exc: + logger.debug("driver_capability_failed method=%s error=%s", name, exc) + return default + + +def call_quietly( + driver: DriverProtocol | DriverTabProtocol, name: str, /, *args: Any, **kwargs: Any +) -> None: + method = resolve_callable(driver, name) + if method is None: + return + try: + method(*args, **kwargs) + except Exception as exc: + logger.debug("driver_capability_failed method=%s error=%s", name, exc) diff --git a/app/engine/envelope.py b/app/engine/envelope.py new file mode 100644 index 0000000..a2ed8dd --- /dev/null +++ b/app/engine/envelope.py @@ -0,0 +1,134 @@ +"""Success and error envelope builders plus HTML normalization.""" + +from __future__ import annotations + +from app.infra.detector import ChallengeAssessment +from app.schemas.enums import ErrorCategory, ExecutionTier, NavigationMode, TimeoutPhase +from app.schemas.response import ( + ChallengeSignal, + ScrapeDiagnostics, + ScrapeError, + ScrapeSuccess, + XhrResponse, +) + +HTML_DOCUMENT_CONTENT_TYPE = "text/html; charset=utf-8" + + +def utf8_normalize_html(html: str) -> str: + if not html: + return html + try: + html = html.encode("latin-1").decode("utf-8") + # fmt: skip keeps parenthesized except (dev venv predates PEP 758 syntax) + except (UnicodeEncodeError, UnicodeDecodeError): # fmt: skip + pass + return html.encode("utf-8", errors="replace").decode("utf-8") + + +def html_document_headers( + html: str, headers: dict[str, str] | None +) -> tuple[str, dict[str, str] | None]: + if not html: + return html, headers + normalized = utf8_normalize_html(html) + out: dict[str, str] = {} + for key, value in (headers or {}).items(): + if str(key).lower() == "content-type": + continue + out[str(key)] = str(value) + out["content-type"] = HTML_DOCUMENT_CONTENT_TYPE + return normalized, out + + +def build_diagnostics( + *, + request_id: str, + attempts: int = 0, + strategy_used: NavigationMode | None = None, + render_ms: int = 0, + execution_tier: ExecutionTier | None = None, + assessment: ChallengeAssessment | None = None, + timeout_phase: TimeoutPhase | None = None, +) -> ScrapeDiagnostics: + challenge = None + if assessment is not None: + challenge = ChallengeSignal( + blocked=assessment.blocked_detected, + detected=assessment.challenge_detected, + marker=assessment.detected_marker, + ) + return ScrapeDiagnostics( + request_id=request_id, + attempts=attempts, + strategy_used=strategy_used, + render_ms=render_ms, + execution_tier=execution_tier, + challenge=challenge, + timeout_phase=timeout_phase, + ) + + +def build_success( + url: str, + *, + request_id: str, + html: str, + attempts: int, + render_ms: int, + execution_tier: ExecutionTier, + strategy_used: NavigationMode | None = None, + final_url: str | None = None, + status_code: int | None = 200, + headers: dict[str, str] | None = None, + metadata_error: str | None = None, + assessment: ChallengeAssessment | None = None, + xhr_responses: list[XhrResponse] | None = None, +) -> ScrapeSuccess: + html, headers = html_document_headers(html, headers) + return ScrapeSuccess( + url=url, + final_url=final_url or url, + status_code=status_code, + headers=headers, + html=html, + metadata_error=metadata_error, + xhr_responses=xhr_responses or [], + diagnostics=build_diagnostics( + request_id=request_id, + attempts=attempts, + strategy_used=strategy_used, + render_ms=render_ms, + execution_tier=execution_tier, + assessment=assessment, + ), + ) + + +def build_error( + url: str, + message: str, + *, + request_id: str, + error_category: ErrorCategory, + attempts: int = 0, + strategy_used: NavigationMode | None = None, + render_ms: int = 0, + execution_tier: ExecutionTier | None = None, + assessment: ChallengeAssessment | None = None, + timeout_phase: TimeoutPhase | None = None, +) -> ScrapeError: + return ScrapeError( + url=url, + error=message, + error_category=error_category, + diagnostics=build_diagnostics( + request_id=request_id, + attempts=attempts, + strategy_used=strategy_used, + render_ms=render_ms, + execution_tier=execution_tier, + assessment=assessment, + timeout_phase=timeout_phase, + ), + ) diff --git a/app/engine/orchestrator.py b/app/engine/orchestrator.py new file mode 100644 index 0000000..c617851 --- /dev/null +++ b/app/engine/orchestrator.py @@ -0,0 +1,163 @@ +"""Scrape execution orchestrator across HTTP and browser tiers.""" + +from __future__ import annotations + +import threading +import time +import uuid +from pathlib import Path +from urllib.parse import urlparse + +from app.config import Settings +from app.engine.browser_tier import run_browser_tier +from app.engine.budget import elapsed_ms, is_timeout_exception +from app.engine.envelope import build_error +from app.engine.request_tier import run_request_tier +from app.engine.session import ScrapeSession +from app.exceptions import RequestIdCollisionError +from app.infra.runtime_cleanup import ( + prune_orphan_runtime_dirs, + runtime_root_low_on_space, +) +from app.infra.scrape_progress import ScrapeProgress +from app.logging_config import get_logger +from app.schemas.enums import ( + ErrorCategory, + ExecutionMode, + ExecutionTier, + NavigationMode, + TimeoutPhase, +) +from app.schemas.request import ScrapeRequest +from app.schemas.response import ScrapeError, ScrapeSuccess + +logger = get_logger() + + +class ScraperEngine: + """Deep module orchestrating anti-detect HTTP and browser execution tiers.""" + + def __init__( + self, + *, + settings: Settings, + runtime_root: Path | None = None, + ) -> None: + self.settings = settings + self.runtime_root = runtime_root or self.settings.runtime_root + self._active_request_ids: set[str] = set() + self._active_request_ids_lock = threading.Lock() + + def register_request_id(self, request_id: str) -> None: + with self._active_request_ids_lock: + if request_id in self._active_request_ids: + raise RequestIdCollisionError(request_id) + self._active_request_ids.add(request_id) + + def unregister_request_id(self, request_id: str) -> None: + with self._active_request_ids_lock: + self._active_request_ids.discard(request_id) + + def prune_runtime_dirs(self) -> int: + # Hold the lock for the full prune so a newly registered request cannot + # create its runtime dir and then be deleted from a stale snapshot. + with self._active_request_ids_lock: + return prune_orphan_runtime_dirs( + self.runtime_root, set(self._active_request_ids) + ) + + def prepare_runtime_for_request(self) -> None: + if runtime_root_low_on_space( + self.runtime_root, + min_free_bytes=self.settings.scrape_runtime_min_free_bytes, + ): + logger.info("runtime_root_low_on_space path=%s", self.runtime_root) + self.prune_runtime_dirs() + self.runtime_root.mkdir(parents=True, exist_ok=True) + + def execute( + self, + payload: ScrapeRequest, + deadline_monotonic: float | None = None, + *, + request_id: str | None = None, + progress: ScrapeProgress | None = None, + ) -> ScrapeSuccess | ScrapeError: + target_url = str(payload.url) + resolved_request_id = request_id or str(uuid.uuid4()) + now = time.monotonic() + # When the API supplies a deadline (computed at submission), budget math + # must use that submission start — not the post-queue worker clock — or + # a long queue wait grants a second full timeout. + if deadline_monotonic is not None: + started_monotonic = ( + deadline_monotonic - self.settings.scrape_timeout_seconds + ) + else: + started_monotonic = now + progress = progress or ScrapeProgress() + + if deadline_monotonic is not None and now >= deadline_monotonic: + progress.mark(TimeoutPhase.QUEUE) + return build_error( + target_url, + "Scrape timed out in threadpool queue before execution started", + request_id=resolved_request_id, + error_category=ErrorCategory.TIMEOUT, + timeout_phase=TimeoutPhase.QUEUE, + ) + + with ScrapeSession(self, resolved_request_id) as session: + should_try_request_tier = ( + payload.execution_mode == ExecutionMode.REQUEST + or ( + payload.execution_mode == ExecutionMode.AUTO + and payload.navigation_mode == NavigationMode.AUTO + and not payload.wait_for_selector + and not payload.scroll + ) + ) + + if should_try_request_tier: + try: + request_result = run_request_tier( + payload, + resolved_request_id, + started_monotonic, + progress, + settings=self.settings, + ) + if request_result is not None: + return request_result + except Exception as exc: + logger.info( + "request_tier_failed request_id=%s host=%s error=%s", + resolved_request_id, + urlparse(target_url).hostname, + str(exc), + ) + if payload.execution_mode == ExecutionMode.REQUEST: + render_ms = elapsed_ms(started_monotonic) + is_timeout = is_timeout_exception(exc) + return build_error( + target_url, + str(exc), + request_id=resolved_request_id, + attempts=1, + render_ms=render_ms, + error_category=( + ErrorCategory.TIMEOUT + if is_timeout + else ErrorCategory.NAVIGATION_ERROR + ), + execution_tier=ExecutionTier.HTTP_REQUEST, + timeout_phase=TimeoutPhase.WORK if is_timeout else None, + ) + + return run_browser_tier( + payload, + session, + started_monotonic, + progress, + settings=self.settings, + ) diff --git a/app/engine/request_tier.py b/app/engine/request_tier.py new file mode 100644 index 0000000..05d4248 --- /dev/null +++ b/app/engine/request_tier.py @@ -0,0 +1,116 @@ +"""Anti-detect HTTP request execution tier.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING +from urllib.parse import urlparse + +from app.config import Settings +from app.engine.budget import elapsed_ms, remaining_total_seconds +from app.engine.envelope import build_error, build_success +from app.infra.detector import ChallengeDetector +from app.infra.scrape_progress import ScrapeProgress +from app.logging_config import get_logger +from app.schemas.enums import ErrorCategory, ExecutionMode, ExecutionTier, TimeoutPhase +from app.schemas.request import ScrapeRequest +from app.schemas.response import ScrapeError, ScrapeSuccess + +logger = get_logger() + +if TYPE_CHECKING: + from botasaurus.request import HttpResponse + + +def run_request_tier( + payload: ScrapeRequest, + request_id: str, + started_monotonic: float, + progress: ScrapeProgress, + *, + settings: Settings, +) -> ScrapeSuccess | ScrapeError | None: + from botasaurus.request import Request + + target_url = str(payload.url) + remaining_budget = remaining_total_seconds(settings, started_monotonic) + progress.mark( + TimeoutPhase.WORK, + attempts=1, + execution_tier=ExecutionTier.HTTP_REQUEST, + ) + + req_headers = dict(payload.headers) if payload.headers else {} + proxies = {"http": payload.proxy, "https": payload.proxy} if payload.proxy else None + + req = Request() + resp: HttpResponse | None = None + try: + resp = req.get( + target_url, + headers=req_headers if req_headers else None, + cookies=payload.cookies, + user_agent=payload.effective_user_agent, + proxies=proxies, + timeout=remaining_budget, + browser="chrome", + allow_redirects=True, + ) + + html = resp.text or "" + status_code = int(resp.status_code) if resp.status_code is not None else 200 + headers_dict = ( + {str(k): str(v) for k, v in resp.headers.items()} if resp.headers else None + ) + final_url = resp.url or target_url + + assessment = ChallengeDetector.detect(html, status_code) + render_ms = elapsed_ms(started_monotonic) + + is_clean_success = ( + assessment.is_clean + and (200 <= status_code < 300) + and len(html.strip()) > 0 + and not payload.wait_for_selector + and not payload.scroll + ) + + if payload.execution_mode == ExecutionMode.AUTO and not is_clean_success: + logger.info( + "request_tier_escalating request_id=%s host=%s status=%d blocked=%s challenge=%s", + request_id, + urlparse(target_url).hostname, + status_code, + assessment.blocked_detected, + assessment.challenge_detected, + ) + return None + + if assessment.blocked_detected: + return build_error( + target_url, + "Challenge block detected", + request_id=request_id, + error_category=ErrorCategory.CHALLENGE_BLOCK, + attempts=1, + render_ms=render_ms, + execution_tier=ExecutionTier.HTTP_REQUEST, + assessment=assessment, + ) + + return build_success( + target_url, + request_id=request_id, + html=html, + final_url=final_url, + status_code=status_code, + headers=headers_dict, + attempts=1, + render_ms=render_ms, + execution_tier=ExecutionTier.HTTP_REQUEST, + assessment=assessment, + ) + finally: + try: + req.close() + except Exception: + pass diff --git a/app/engine/session.py b/app/engine/session.py new file mode 100644 index 0000000..178bb90 --- /dev/null +++ b/app/engine/session.py @@ -0,0 +1,56 @@ +"""Per-request browser session lifecycle and filesystem isolation.""" + +from __future__ import annotations + +import errno +import shutil +from typing import TYPE_CHECKING, Any + +from app.engine.driver_capabilities import DriverProtocol, call_quietly + +if TYPE_CHECKING: + from app.engine.orchestrator import ScraperEngine + + +class ScrapeSession: + """Encapsulates per-request concurrency registration and filesystem isolation.""" + + def __init__(self, engine: ScraperEngine, request_id: str) -> None: + self.engine = engine + self.request_id = request_id + self.runtime_dir = engine.runtime_root / request_id + self.profile_dir = self.runtime_dir / "profile" + self.driver: DriverProtocol | None = None + + def __enter__(self) -> ScrapeSession: + self.engine.register_request_id(self.request_id) + try: + self.engine.prepare_runtime_for_request() + except Exception: + self.engine.unregister_request_id(self.request_id) + raise + return self + + def prepare_profile_dirs(self) -> None: + try: + self._make_dirs() + except OSError as exc: + if exc.errno != errno.ENOSPC: + raise + # Drop any partially created dirs so the retry can recreate them + # with exist_ok=False after the prune pass frees space. + shutil.rmtree(self.runtime_dir, ignore_errors=True) + self.engine.prune_runtime_dirs() + self._make_dirs() + + def _make_dirs(self) -> None: + self.runtime_dir.mkdir(parents=True, exist_ok=False) + self.profile_dir.mkdir(parents=True, exist_ok=False) + + def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None: + try: + if self.driver is not None: + call_quietly(self.driver, "close") + finally: + shutil.rmtree(self.runtime_dir, ignore_errors=True) + self.engine.unregister_request_id(self.request_id) diff --git a/app/engine/strategies.py b/app/engine/strategies.py new file mode 100644 index 0000000..1dd7888 --- /dev/null +++ b/app/engine/strategies.py @@ -0,0 +1,161 @@ +"""Browser navigation strategies and driver interaction helpers.""" + +from __future__ import annotations + +from typing import Any + +from app.engine.driver_capabilities import ( + DriverProtocol, + call_if_available, + call_quietly, + resolve_callable, +) +from app.infra.xhr_collector import XhrCollector +from app.logging_config import get_logger +from app.schemas.enums import NavigationMode +from app.schemas.request import ScrapeRequest +from app.schemas.response import XhrResponse + +logger = get_logger() + +_CAPABILITY_MISS = object() + +TRACKER_URL_PATTERNS: list[str] = [ + "*google-analytics.com*", + "*googletagmanager.com*", + "*facebook.net*", + "*doubleclick.net*", + "*sentry.io*", + "*hotjar.com*", + "*clarity.ms*", + "*datadoghq-browser-agent.com*", + "*segment.io*", + "*analytics.js*", + "*.woff", + "*.woff2", + "*.ttf", +] + +_AUTO_STRATEGIES: tuple[NavigationMode, ...] = ( + NavigationMode.GOOGLE_GET, + NavigationMode.GOOGLE_GET_BYPASS, + NavigationMode.GET, +) + + +def resolve_strategies(mode: NavigationMode, max_retries: int) -> list[NavigationMode]: + max_attempts = 1 + max_retries + if mode == NavigationMode.AUTO: + return list(_AUTO_STRATEGIES[: min(len(_AUTO_STRATEGIES), max_attempts)]) + return [mode] * max_attempts + + +def _driver_method(driver: DriverProtocol, *names: str): + return resolve_callable(driver, *names) or driver.get + + +def navigate( + driver: DriverProtocol, + target_url: str, + strategy: NavigationMode, + timeout_seconds: int, +) -> None: + if strategy == NavigationMode.ORGANIC_GET: + method = _driver_method(driver, "organic_get", "google_get") + elif strategy.value.startswith("google_get"): + method = _driver_method(driver, "google_get") + else: + method = driver.get + + kwargs: dict[str, Any] = {} + if strategy == NavigationMode.GOOGLE_GET_BYPASS: + kwargs["bypass_cloudflare"] = True + try: + method(target_url, timeout=timeout_seconds, **kwargs) + except TypeError: + method(target_url, **kwargs) + + +def configure_driver( + driver: DriverProtocol, + payload: ScrapeRequest, + target_url: str, + collector: XhrCollector | None = None, +) -> None: + tab = getattr(driver, "_tab", None) + if tab is not None: + if collector is not None: + try: + collector.install(tab) + except Exception: + pass + + if payload.block_trackers: + call_quietly(tab, "block_urls", TRACKER_URL_PATTERNS) + + if payload.cookies: + for c_name, c_val in payload.cookies.items(): + call_quietly( + driver, + "add_cookies", + [{"name": str(c_name), "value": str(c_val), "url": target_url}], + ) + + if payload.headers and tab is not None: + call_quietly(tab, "set_extra_http_headers", payload.headers) + + +def wait_for_readiness( + driver: DriverProtocol, + *, + selector: str | None, + timeout_seconds: int, +) -> None: + if selector: + driver.wait_for_element(selector, wait=timeout_seconds) + return + + if ( + call_if_available(driver, "sleep_random", 0.5, 1.2, default=_CAPABILITY_MISS) + is not _CAPABILITY_MISS + ): + return + driver.sleep(1) + + +def apply_scrolling(driver: DriverProtocol) -> None: + if ( + call_if_available(driver, "scroll_to_bottom", default=_CAPABILITY_MISS) + is _CAPABILITY_MISS + and call_if_available(driver, "scroll", default=_CAPABILITY_MISS) + is _CAPABILITY_MISS + and call_if_available( + driver, + "run_js", + "window.scrollTo(0, document.body.scrollHeight);", + default=_CAPABILITY_MISS, + ) + is _CAPABILITY_MISS + ): + call_quietly( + driver, + "execute_script", + "window.scrollTo(0, document.body.scrollHeight);", + ) + + if ( + call_if_available(driver, "sleep_random", 0.4, 0.9, default=_CAPABILITY_MISS) + is _CAPABILITY_MISS + ): + call_quietly(driver, "sleep", 0.5) + + +def harvest_xhr(collector: XhrCollector, driver: DriverProtocol) -> list[XhrResponse]: + tab = getattr(driver, "_tab", None) + if tab is None: + return collector.results() + try: + return collector.harvest(tab) + except Exception as exc: + logger.debug("xhr_harvest_failed error=%s", str(exc)) + return collector.results() diff --git a/app/exceptions.py b/app/exceptions.py new file mode 100644 index 0000000..f704f71 --- /dev/null +++ b/app/exceptions.py @@ -0,0 +1,15 @@ +"""Application-specific exceptions with stable semantics across layers.""" + +from __future__ import annotations + + +class BotasaurusScrapeError(Exception): + """Base class for domain errors that map to scrape envelopes.""" + + +class RequestIdCollisionError(BotasaurusScrapeError): + """Raised when an inbound request id is already active.""" + + def __init__(self, request_id: str) -> None: + self.request_id = request_id + super().__init__("request id collision detected") diff --git a/app/infra/__init__.py b/app/infra/__init__.py new file mode 100644 index 0000000..002785c --- /dev/null +++ b/app/infra/__init__.py @@ -0,0 +1 @@ +"""Infrastructure adapters: telemetry, progress, metadata, and runtime helpers.""" diff --git a/app/infra/cdp_types.py b/app/infra/cdp_types.py new file mode 100644 index 0000000..b64e5cd --- /dev/null +++ b/app/infra/cdp_types.py @@ -0,0 +1,37 @@ +"""TypedDict shapes for CDP log entries and XHR capture state.""" + +from __future__ import annotations + +from typing import TypedDict + + +class CdpNetworkResponse(TypedDict, total=False): + status: int + headers: dict[str, str] + url: str + type: str + + +class CdpResponseReceivedParams(TypedDict, total=False): + type: str + response: CdpNetworkResponse + + +class CdpResponseReceivedMessage(TypedDict, total=False): + method: str + params: CdpResponseReceivedParams + + +class CdpLogMessageEnvelope(TypedDict, total=False): + message: CdpResponseReceivedMessage + + +class CdpPerformanceLogEntry(TypedDict): + message: str + + +class PendingXhrMeta(TypedDict): + url: str + status_code: int + headers: dict[str, str] + request_id: str | int diff --git a/app/detector.py b/app/infra/detector.py similarity index 90% rename from app/detector.py rename to app/infra/detector.py index cf3bfd1..9f06031 100644 --- a/app/detector.py +++ b/app/infra/detector.py @@ -1,8 +1,8 @@ -# app/detector.py +"""Anti-bot challenge detection from HTML and driver signals.""" + from __future__ import annotations from dataclasses import dataclass -from typing import Any _CHALLENGE_MARKERS: tuple[str, ...] = ( "challenge-error-text", @@ -26,7 +26,6 @@ class ChallengeAssessment: blocked_detected: bool challenge_detected: bool detected_marker: str | None = None - error_category: str | None = None @property def is_clean(self) -> bool: @@ -41,7 +40,7 @@ def detect( cls, html: str, status_code: int | None = None, - driver: Any = None, + driver: object | None = None, ) -> ChallengeAssessment: lower_html = html.lower() matched_marker: str | None = None @@ -75,13 +74,8 @@ def detect( status_code in {401, 403, 429} if status_code is not None else False ) - error_category = ( - "challenge_block" if (challenge_detected or blocked_detected) else None - ) - return ChallengeAssessment( blocked_detected=blocked_detected, challenge_detected=challenge_detected, detected_marker=matched_marker, - error_category=error_category, ) diff --git a/app/metadata.py b/app/infra/metadata.py similarity index 66% rename from app/metadata.py rename to app/infra/metadata.py index 9568385..debd9dc 100644 --- a/app/metadata.py +++ b/app/infra/metadata.py @@ -1,12 +1,20 @@ -# app/metadata.py +"""Best-effort HTTP metadata extraction from browser driver state.""" + from __future__ import annotations import json -import logging from dataclasses import dataclass -from typing import Any +from typing import cast + +from app.engine.driver_capabilities import DriverProtocol +from app.infra.cdp_types import ( + CdpLogMessageEnvelope, + CdpPerformanceLogEntry, + CdpResponseReceivedMessage, +) +from app.logging_config import get_logger -logger = logging.getLogger("botasaurus_scrape_api") +logger = get_logger() @dataclass(frozen=True, slots=True) @@ -22,14 +30,16 @@ class MetadataExtractor: @classmethod def extract_from_requests( - cls, driver: Any, target_url: str + cls, driver: DriverProtocol ) -> tuple[int | None, dict[str, str] | None, str | None]: reqs = getattr(driver, "requests", None) if not isinstance(reqs, (list, tuple)): return None, None, None - for req in reversed(reqs): + for req in reversed(cast(list[object], reqs)): resp = getattr(req, "response", None) - status_code = getattr(resp, "status_code", None) + status_code = ( + getattr(resp, "status_code", None) if resp is not None else None + ) if status_code is not None: headers = getattr(resp, "headers", None) hdr_dict = ( @@ -41,9 +51,23 @@ def extract_from_requests( return int(status_code), hdr_dict, str(req_url) if req_url else None return None, None, None + @classmethod + def _parse_cdp_log_message(cls, raw_msg: str) -> CdpResponseReceivedMessage | None: + try: + msg_obj = json.loads(raw_msg) + except json.JSONDecodeError: + return None + if not isinstance(msg_obj, dict): + return None + envelope = cast(CdpLogMessageEnvelope, msg_obj) + message = envelope.get("message") + if not isinstance(message, dict): + return None + return message + @classmethod def extract_from_cdp_logs( - cls, driver: Any, target_url: str + cls, driver: DriverProtocol ) -> tuple[int | None, dict[str, str] | None, str | None]: get_log = getattr(driver, "get_log", None) if not callable(get_log): @@ -54,17 +78,14 @@ def extract_from_cdp_logs( if not isinstance(logs, list): return None, None, None - for entry in reversed(logs): - raw_msg = ( - entry.get("message", "{}") if isinstance(entry, dict) else "{}" - ) - msg_obj = json.loads(raw_msg) if isinstance(raw_msg, str) else raw_msg - msg = msg_obj.get("message", {}) if isinstance(msg_obj, dict) else {} - if msg.get("method") != "Network.responseReceived": + for entry in reversed(cast(list[CdpPerformanceLogEntry], logs)): + raw_msg = entry.get("message", "{}") + msg = cls._parse_cdp_log_message(raw_msg) + if msg is None or msg.get("method") != "Network.responseReceived": continue - params = msg.get("params", {}) - resp = params.get("response", {}) + params = msg.get("params") or {} + resp = params.get("response") or {} res_type = params.get("type") or resp.get("type") if res_type and res_type not in ("Document", "Other"): continue @@ -85,12 +106,12 @@ def extract_from_cdp_logs( return None, None, None @classmethod - def fetch(cls, driver: Any, target_url: str) -> MetadataResult: - final_url = getattr(driver, "current_url", None) or target_url + def fetch(cls, driver: DriverProtocol, target_url: str) -> MetadataResult: + final_url = driver.current_url or target_url try: for extractor in (cls.extract_from_requests, cls.extract_from_cdp_logs): - status_code, headers, passive_url = extractor(driver, target_url) + status_code, headers, passive_url = extractor(driver) if status_code is not None: return MetadataResult( status_code=status_code, diff --git a/app/ops_telemetry.py b/app/infra/ops_telemetry.py similarity index 71% rename from app/ops_telemetry.py rename to app/infra/ops_telemetry.py index 9bfaa56..7216ef9 100644 --- a/app/ops_telemetry.py +++ b/app/infra/ops_telemetry.py @@ -1,12 +1,25 @@ -# app/ops_telemetry.py from __future__ import annotations +from typing import Protocol, cast from urllib.parse import urlparse -from app.schemas import ErrorCategory, ScrapeError -from app.sentry import sentry_is_ready +from app.constants import SERVICE_NAME +from app.logging_config import get_logger +from app.schemas.enums import ErrorCategory +from app.schemas.response import ScrapeError + +logger = get_logger() + + +class SentryScope(Protocol): + """Structural seam for the sentry_sdk scope used by terminal telemetry.""" + + fingerprint: list[str] | None + + def set_tag(self, key: str, value: str) -> None: ... + + def set_context(self, key: str, value: dict[str, object]) -> None: ... -SERVICE_NAME = "botasaurus-scrape-api" _P0_CATEGORIES = frozenset( { @@ -24,33 +37,37 @@ def _issue_fingerprint(error_category: str, host: str | None) -> list[str]: return [SERVICE_NAME, error_category, host or "unknown"] -def _apply_scrape_tags(scope: object, result: ScrapeError, *, http_status: int) -> None: +def _apply_scrape_tags( + scope: SentryScope, result: ScrapeError, *, http_status: int +) -> None: host = _hostname(result.url) category = result.error_category.value diagnostics = result.diagnostics - scope.set_tag("service", SERVICE_NAME) # type: ignore[attr-defined] - scope.set_tag("error_category", category) # type: ignore[attr-defined] + scope.set_tag("service", SERVICE_NAME) + scope.set_tag("error_category", category) if host: - scope.set_tag("host", host) # type: ignore[attr-defined] - scope.set_tag("http_status", str(http_status)) # type: ignore[attr-defined] - scope.set_tag("render_ms", str(diagnostics.render_ms)) # type: ignore[attr-defined] + scope.set_tag("host", host) + scope.set_tag("http_status", str(http_status)) + scope.set_tag("render_ms", str(diagnostics.render_ms)) scrape_context: dict[str, object] = { "request_id": diagnostics.request_id, "attempts": diagnostics.attempts, } if phase := diagnostics.timeout_phase: - scope.set_tag("timeout_phase", phase.value) # type: ignore[attr-defined] + scope.set_tag("timeout_phase", phase.value) scrape_context["timeout_phase"] = phase.value - scope.set_context("scrape", scrape_context) # type: ignore[attr-defined] + scope.set_context("scrape", scrape_context) if diagnostics.strategy_used is not None: - scope.set_tag("strategy_used", diagnostics.strategy_used.value) # type: ignore[attr-defined] + scope.set_tag("strategy_used", diagnostics.strategy_used.value) if diagnostics.execution_tier is not None: - scope.set_tag("execution_tier", diagnostics.execution_tier.value) # type: ignore[attr-defined] + scope.set_tag("execution_tier", diagnostics.execution_tier.value) def report_terminal_outcome(result: ScrapeError, *, http_status: int) -> None: """Emit P0 operational scrape failures to Sentry as grouped Issues.""" + from app.infra.sentry import sentry_is_ready + if not sentry_is_ready(): return if result.error_category not in _P0_CATEGORIES: @@ -63,7 +80,8 @@ def report_terminal_outcome(result: ScrapeError, *, http_status: int) -> None: phase = result.diagnostics.timeout_phase category_label = f"{category}/{phase.value}" if phase else category - with sentry_sdk.new_scope() as scope: + with sentry_sdk.new_scope() as raw_scope: + scope = cast(SentryScope, raw_scope) scope.fingerprint = _issue_fingerprint(category, host) _apply_scrape_tags(scope, result, http_status=http_status) sentry_sdk.capture_message( @@ -74,6 +92,8 @@ def report_terminal_outcome(result: ScrapeError, *, http_status: int) -> None: def record_challenge_block(result: ScrapeError) -> None: """Increment challenge_block product signal metric; stdout logging stays in engine.""" + from app.infra.sentry import sentry_is_ready + if not sentry_is_ready(): return if result.error_category != ErrorCategory.CHALLENGE_BLOCK: diff --git a/app/request_id.py b/app/infra/request_id.py similarity index 84% rename from app/request_id.py rename to app/infra/request_id.py index 87055e9..2aee64a 100644 --- a/app/request_id.py +++ b/app/infra/request_id.py @@ -1,11 +1,13 @@ -# app/request_id.py +"""Request id resolution from headers with fallback generation.""" + from __future__ import annotations -import logging import re import uuid -logger = logging.getLogger("botasaurus_scrape_api") +from app.logging_config import get_logger + +logger = get_logger() _MAX_LENGTH = 128 _SAFE_PATTERN = re.compile(r"^[A-Za-z0-9._-]+$") @@ -21,7 +23,7 @@ def resolve_request_id( @return tuple of resolved id and used_fallback flag """ candidate = inbound.strip() if inbound is not None else None - if _is_valid(candidate): + if candidate is not None and _is_valid(candidate): return candidate, False reason = "absent" if not candidate else "invalid" diff --git a/app/infra/runtime_cleanup.py b/app/infra/runtime_cleanup.py new file mode 100644 index 0000000..a9a9592 --- /dev/null +++ b/app/infra/runtime_cleanup.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import shutil +from pathlib import Path + +from app.logging_config import get_logger + +logger = get_logger() + + +def runtime_root_low_on_space( + runtime_root: Path, + *, + min_free_bytes: int, +) -> bool: + """Return True when free space under runtime_root's filesystem is below min_free_bytes.""" + runtime_root.mkdir(parents=True, exist_ok=True) + return shutil.disk_usage(runtime_root).free < min_free_bytes + + +def prune_orphan_runtime_dirs( + runtime_root: Path, + active_request_ids: set[str], +) -> int: + """Delete runtime dirs that are not tied to an active request id.""" + if not runtime_root.is_dir(): + return 0 + + removed = 0 + for entry in runtime_root.iterdir(): + if not entry.is_dir() or entry.name in active_request_ids: + continue + try: + shutil.rmtree(entry) + removed += 1 + logger.info("runtime_dir_pruned path=%s", entry) + except OSError as exc: + logger.warning("runtime_dir_prune_failed path=%s error=%s", entry, exc) + return removed diff --git a/app/scrape_progress.py b/app/infra/scrape_progress.py similarity index 89% rename from app/scrape_progress.py rename to app/infra/scrape_progress.py index 0101bb1..326a7d0 100644 --- a/app/scrape_progress.py +++ b/app/infra/scrape_progress.py @@ -1,10 +1,11 @@ -# app/scrape_progress.py +"""In-memory scrape progress tracking for timeout phase diagnostics.""" + from __future__ import annotations import threading from dataclasses import dataclass, replace -from app.schemas import ExecutionTier, NavigationMode, TimeoutPhase +from app.schemas.enums import ExecutionTier, NavigationMode, TimeoutPhase @dataclass(frozen=True, slots=True) diff --git a/app/infra/sentry.py b/app/infra/sentry.py new file mode 100644 index 0000000..6446bfd --- /dev/null +++ b/app/infra/sentry.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +from typing import Any, Protocol, TypedDict, cast + +from app.config import SentrySettings, Settings +from app.logging_config import get_logger + +logger = get_logger() + +_initialized = False + + +class SentryEventHint(TypedDict, total=False): + log_record: object + + +class SentryEvent(TypedDict, total=False): + tags: dict[str, str] + logger: str + logentry: dict[str, str] + message: str + + +class SentryBeforeSend(Protocol): + def __call__( + self, event: SentryEvent, hint: SentryEventHint + ) -> SentryEvent | None: ... + + +def sentry_is_ready() -> bool: + """Return True when Sentry init succeeded (init requires a DSN).""" + return _initialized + + +def _before_send(event: SentryEvent, hint: SentryEventHint) -> SentryEvent | None: + tags = event.get("tags") or {} + if tags.get("error_category") == "challenge_block": + return None + + log_record = hint.get("log_record") + logger_name = event.get("logger") + if logger_name == "websocket" or getattr(log_record, "name", None) == "websocket": + return None + + logentry = event.get("logentry") or {} + message = logentry.get("formatted") or event.get("message") or "" + if "Connection to remote host was lost" in str(message): + return None + + return event + + +def setup_sentry(settings: Settings | None = None) -> bool: + """Initialize Sentry when SENTRY_DSN is set. Returns True on success.""" + global _initialized + + from app.config import get_settings + + resolved = settings or get_settings() + return _setup_sentry(resolved.sentry, deployment_environment=resolved.environment) + + +def _setup_sentry(sentry: SentrySettings, *, deployment_environment: str) -> bool: + global _initialized + + dsn = sentry.dsn.strip() + if not dsn: + return False + + try: + import sentry_sdk + from sentry_sdk.integrations.fastapi import FastApiIntegration + from sentry_sdk.integrations.starlette import StarletteIntegration + except ImportError: + logger.warning( + "sentry_sdk_import_failed SENTRY_DSN is set but sentry-sdk package is not available" + ) + return False + + init_kwargs: dict[str, Any] = { + "dsn": dsn, + "environment": sentry.effective_environment(deployment_environment), + "traces_sample_rate": sentry.traces_sample_rate, + "send_default_pii": sentry.send_default_pii, + "integrations": [ + FastApiIntegration(), + StarletteIntegration(), + ], + "before_send": cast(SentryBeforeSend, _before_send), + } + + if sentry.release.strip(): + init_kwargs["release"] = sentry.release.strip() + if sentry.profiles_sample_rate > 0.0: + init_kwargs["profiles_sample_rate"] = sentry.profiles_sample_rate + + sentry_sdk.init(**init_kwargs) + _initialized = True + + logger.info( + "sentry_initialized environment=%s release=%s traces_sample_rate=%.2f", + sentry.effective_environment(deployment_environment), + sentry.release or None, + sentry.traces_sample_rate, + ) + return True + + +def flush_sentry(timeout: float = 2.0) -> None: + if not _initialized: + return + try: + import sentry_sdk + + sentry_sdk.flush(timeout=timeout) + except Exception as exc: + logger.debug("sentry_flush_failed error=%s", str(exc)) diff --git a/app/xhr_collector.py b/app/infra/xhr_collector.py similarity index 60% rename from app/xhr_collector.py rename to app/infra/xhr_collector.py index 8c1b53e..1d07aec 100644 --- a/app/xhr_collector.py +++ b/app/infra/xhr_collector.py @@ -1,5 +1,4 @@ -# app/xhr_collector.py -"""Collect JSON XHR/fetch response bodies via CDP network events. +"""XHR/fetch JSON response capture via CDP network events. Phase 0 finding: ``Network.getResponseBody`` deadlocks when called from inside a ``LoadingFinished`` handler (CDP session re-entrancy). Handlers only record @@ -10,14 +9,15 @@ from __future__ import annotations import base64 -import logging import threading -from typing import Any +from typing import cast -from botasaurus_driver import cdp -from botasaurus_driver.core.custom_storage_cdp import enable_network +from app.engine.driver_capabilities import CdpTabProtocol +from app.infra.cdp_types import PendingXhrMeta +from app.logging_config import get_logger +from app.schemas.response import XhrResponse -logger = logging.getLogger("botasaurus_scrape_api") +logger = get_logger() class XhrCollector: @@ -31,13 +31,16 @@ class XhrCollector: def __init__(self, target_url: str) -> None: self._target_url = str(target_url).rstrip("/") - self._pending: dict[str, dict[str, Any]] = {} + self._pending: dict[str, PendingXhrMeta] = {} self._ready_ids: list[str] = [] - self._collected: list[dict[str, Any]] = [] + self._collected: list[XhrResponse] = [] self._lock = threading.Lock() - def install(self, tab: Any) -> None: + def install(self, tab: CdpTabProtocol) -> None: """Enable the network domain and register handlers before navigation.""" + from botasaurus_driver import cdp + from botasaurus_driver.core.custom_storage_cdp import enable_network + tab.send(enable_network()) tab.after_response_received(self._on_response) tab.add_handler(cdp.network.LoadingFinished, self._on_finished) @@ -50,39 +53,48 @@ def reset(self) -> None: self._collected.clear() @classmethod - def _allowlisted_headers(cls, headers: Any) -> dict[str, str]: + def _allowlisted_headers(cls, headers: object) -> dict[str, str]: """Keep only content-type; drop Set-Cookie and other headers.""" allowed: dict[str, str] = {} - for key, value in dict(headers or {}).items(): + header_map = cast( + dict[object, object], headers if isinstance(headers, dict) else {} + ) + for key, value in header_map.items(): normalized = str(key).lower() if normalized in cls._HEADER_ALLOWLIST: allowed[normalized] = str(value) return allowed - def _on_response(self, request_id: Any, response: Any, _event: Any) -> None: - url = str(response.url) + def _on_response( + self, request_id: object, response: object, _event: object + ) -> None: + url = str(getattr(response, "url", "")) if url.rstrip("/") == self._target_url: return - if "json" not in (response.mime_type or "").lower(): + mime_type = str(getattr(response, "mime_type", "") or "") + if "json" not in mime_type.lower(): return with self._lock: if len(self._collected) + len(self._pending) >= self.MAX_RESPONSES: return - self._pending[str(request_id)] = { - "url": url, - "status_code": int(response.status), - "headers": self._allowlisted_headers(response.headers), - "request_id": request_id, - } - - def _on_finished(self, event: cdp.network.LoadingFinished) -> None: + stored_request_id = ( + request_id if isinstance(request_id, (str, int)) else str(request_id) + ) + self._pending[str(request_id)] = PendingXhrMeta( + url=url, + status_code=int(getattr(response, "status", 0)), + headers=self._allowlisted_headers(getattr(response, "headers", {})), + request_id=stored_request_id, + ) + + def _on_finished(self, event: object) -> None: # Do not call get_response_body here — CDP deadlocks (Phase 0 spike). - rid = str(event.request_id) + rid = str(getattr(event, "request_id", "")) with self._lock: if rid in self._pending and rid not in self._ready_ids: self._ready_ids.append(rid) - def harvest(self, tab: Any) -> list[dict[str, Any]]: + def harvest(self, tab: CdpTabProtocol) -> list[XhrResponse]: """Fetch bodies for finished responses on the caller thread.""" with self._lock: ready = list(self._ready_ids) @@ -91,21 +103,20 @@ def harvest(self, tab: Any) -> list[dict[str, Any]]: (rid, self._pending.pop(rid)) for rid in ready if rid in self._pending ] aggregate_bytes = sum( - len(entry["body"].encode("utf-8")) for entry in self._collected + len(entry.body.encode("utf-8")) for entry in self._collected ) for rid, meta in jobs: - request_id = meta.pop("request_id") - body = self._fetch_body(tab, request_id, rid) + body = self._fetch_body(tab, meta["request_id"], rid) if body is None: continue body_bytes = len(body.encode("utf-8")) - entry = { - "url": meta["url"], - "status_code": meta["status_code"], - "headers": meta["headers"], - "body": body, - } + entry = XhrResponse( + url=meta["url"], + status_code=meta["status_code"], + headers=meta["headers"], + body=body, + ) with self._lock: if len(self._collected) >= self.MAX_RESPONSES: break @@ -116,7 +127,11 @@ def harvest(self, tab: Any) -> list[dict[str, Any]]: return self.results() - def _fetch_body(self, tab: Any, request_id: Any, rid: str) -> str | None: + def _fetch_body( + self, tab: CdpTabProtocol, request_id: str | int, rid: str + ) -> str | None: + from botasaurus_driver import cdp + try: body, b64 = tab.send(cdp.network.get_response_body(request_id)) except Exception as exc: @@ -127,7 +142,9 @@ def _fetch_body(self, tab: Any, request_id: Any, rid: str) -> str | None: if b64: try: - body = base64.b64decode(body).decode("utf-8", errors="replace") + body = base64.b64decode(cast(str, body)).decode( + "utf-8", errors="replace" + ) except Exception as exc: logger.debug( "xhr_body_base64_decode_failed request_id=%s error=%s", rid, exc @@ -140,6 +157,6 @@ def _fetch_body(self, tab: Any, request_id: Any, rid: str) -> str | None: return None return body - def results(self) -> list[dict[str, Any]]: + def results(self) -> list[XhrResponse]: with self._lock: return list(self._collected) diff --git a/app/logging_config.py b/app/logging_config.py new file mode 100644 index 0000000..2adb2e3 --- /dev/null +++ b/app/logging_config.py @@ -0,0 +1,21 @@ +"""Logging bootstrap for the service process.""" + +from __future__ import annotations + +import logging + +LOGGER_NAME = "botasaurus_scrape_api" + + +def setup_logging(level: int = logging.INFO) -> logging.Logger: + logger = logging.getLogger(LOGGER_NAME) + if not logger.handlers: + logging.basicConfig( + level=level, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + ) + return logger + + +def get_logger() -> logging.Logger: + return logging.getLogger(LOGGER_NAME) diff --git a/app/main.py b/app/main.py index d97fea4..ca29de4 100644 --- a/app/main.py +++ b/app/main.py @@ -1,420 +1,56 @@ -# app/main.py +"""FastAPI application factory.""" + from __future__ import annotations -import asyncio -import logging -import os -import time from collections.abc import AsyncGenerator from concurrent.futures import ThreadPoolExecutor from contextlib import asynccontextmanager -from functools import partial -from importlib.metadata import PackageNotFoundError, version -from typing import Any -from urllib.parse import urlparse - -from fastapi import FastAPI, Header, Request -from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse - -from app.engine import DEFAULT_SCRAPE_TIMEOUT_SECONDS, ScraperEngine -from app.ops_telemetry import emit_terminal_telemetry -from app.request_id import resolve_request_id -from app.schemas import ( - DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS, - HEALTH_EXAMPLE, - SCRAPE_ERROR_EXAMPLE, - SCRAPE_SUCCESS_EXAMPLE, - ErrorCategory, - HealthResponse, - ScrapeDiagnostics, - ScrapeError, - ScrapeRequest, - ScrapeSuccess, - TimeoutPhase, - validation_error, -) -from app.scrape_progress import ScrapeProgress -from app.security import UrlGuard -from app.sentry import flush_sentry, setup_sentry - -logger = logging.getLogger("botasaurus_scrape_api") -if not logger.handlers: - logging.basicConfig( - level=logging.INFO, - format="%(asctime)s %(levelname)s %(name)s %(message)s", - ) - -setup_sentry() - -_MAX_WORKERS = int(os.getenv("SCRAPE_MAX_WORKERS", "4")) -_executor = ThreadPoolExecutor(max_workers=max(1, _MAX_WORKERS)) -_engine = ScraperEngine() - - -@asynccontextmanager -async def lifespan(_: FastAPI) -> AsyncGenerator[None]: - yield - _executor.shutdown(wait=False, cancel_futures=True) - flush_sentry() - - -_API_DESCRIPTION = f""" -Docker-first scrape API that uses Botasaurus to fetch rendered HTML. - -- `GET /health` — liveness and detected Botasaurus version -- `POST /scrape` — scrape a public `http`/`https` URL - -`wait_timeout_seconds` values outside `[1, {DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS}]` -are **clamped** into that range so scrape still runs; they are not rejected -with 422. - -When `html` is present it is UTF-8-normalized and `headers` `content-type` is -`text/html; charset=utf-8`. - -Localhost, private, link-local, multicast, reserved, and unspecified -destinations are blocked (403). Schema validation failures use this API's -scrape error envelope, not FastAPI `detail`. -""" - -_JSON_EXAMPLE = {"application/json": {"example": SCRAPE_ERROR_EXAMPLE}} -_SCRAPE_ERROR_RESPONSES = { - 400: { - "model": ScrapeError, - "description": "URL rejected by validation (scheme, host, or unresolvable target).", - "content": _JSON_EXAMPLE, - }, - 403: { - "model": ScrapeError, - "description": "URL blocked by SSRF guardrails (localhost, private, or reserved destination).", - "content": _JSON_EXAMPLE, - }, - 422: { - "model": ScrapeError, - "description": "Request schema validation failed. Body is the scrape error envelope, not FastAPI `detail`.", - "content": _JSON_EXAMPLE, - }, - 502: { - "model": ScrapeError, - "description": "Scrape execution failure or challenge block after the final attempt.", - "content": { - "application/json": { - "example": { - **SCRAPE_ERROR_EXAMPLE, - "error": "Bot challenge detected (Just a moment...)", - "error_category": "challenge_block", - "diagnostics": { - **SCRAPE_ERROR_EXAMPLE["diagnostics"], - "attempts": 3, - "strategy_used": "get", - "render_ms": 1500, - "execution_tier": "browser_driver", - "challenge": { - "blocked": True, - "detected": True, - "marker": "Just a moment...", - }, - }, - } - } - }, - }, - 504: { - "model": ScrapeError, - "description": "Scrape timed out before a result was produced.", - "content": { - "application/json": { - "example": { - **SCRAPE_ERROR_EXAMPLE, - "error": ( - f"Scrape timed out after {DEFAULT_SCRAPE_TIMEOUT_SECONDS} " - "seconds (phase=work)" - ), - "error_category": "timeout", - "diagnostics": { - **SCRAPE_ERROR_EXAMPLE["diagnostics"], - "attempts": 1, - "strategy_used": "get", - "render_ms": 45012, - "execution_tier": "browser_driver", - "timeout_phase": "work", - }, - } - } - }, - }, -} - - -app = FastAPI( - title="Botasaurus Scrape API", - description=_API_DESCRIPTION.strip(), - version="2.0.0", - contact={ - "name": "html2rss", - "url": "https://github.com/html2rss/botasaurus-scrape-api/issues", - }, - license_info={ - "name": "MIT", - "url": "https://opensource.org/licenses/MIT", - }, - servers=[ - { - "url": "http://localhost:4010", - "description": "Local Docker (make serve)", - } - ], - openapi_tags=[ - { - "name": "health", - "description": "Liveness probe and detected Botasaurus package version.", - }, - { - "name": "scrape", - "description": "Render a public URL and return UTF-8 HTML plus diagnostics.", - }, - ], - lifespan=lifespan, -) - -_NON_FIELD_LOC = {"body", "query", "path", "header"} - - -def _schema_field_from_loc(loc: tuple[Any, ...] | list[Any]) -> str: - for part in loc: - if part not in _NON_FIELD_LOC: - return str(part) - return str(loc[-1]) if loc else "unknown" - - -def _first_schema_field(errors: list[Any]) -> str: - if not errors: - return "unknown" - return _schema_field_from_loc(errors[0].get("loc") or ()) +from fastapi import FastAPI -def _url_from_validation_body(body: Any) -> str: - if isinstance(body, dict) and body.get("url") is not None: - return str(body["url"]) - return "" +from app.api.errors import register_exception_handlers +from app.api.openapi import configure_openapi +from app.config import Settings, get_settings +from app.engine import ScraperEngine +from app.infra.sentry import flush_sentry, setup_sentry +from app.logging_config import setup_logging -def _validation_error_message(errors: list[Any]) -> str: - if not errors: - return "Request schema validation failed" - parts: list[str] = [] - for err in errors: - field = _schema_field_from_loc(err.get("loc") or ()) - parts.append(f"{field}: {err.get('msg') or 'invalid'}") - return "; ".join(parts) +def create_app(settings: Settings | None = None) -> FastAPI: + resolved_settings = settings or get_settings() + setup_logging() + setup_sentry(resolved_settings) + openapi_metadata = configure_openapi(resolved_settings) - -def _json(model: ScrapeSuccess | ScrapeError) -> dict[str, Any]: - return model.model_dump(mode="json") - - -def handler_timeout_error( - url: str, - *, - request_id: str, - started_monotonic: float, - progress: ScrapeProgress, - timeout_seconds: int = DEFAULT_SCRAPE_TIMEOUT_SECONDS, -) -> ScrapeError: - """Build a 504 envelope from thread-visible scrape progress after wait_for fires.""" - snap = progress.snapshot() - phase = snap.phase - render_ms = int((time.monotonic() - started_monotonic) * 1000) - return ScrapeError( - url=url, - error=( - f"Scrape timed out after {timeout_seconds} seconds (phase={phase.value})" - ), - error_category=ErrorCategory.TIMEOUT, - diagnostics=ScrapeDiagnostics( - request_id=request_id, - attempts=snap.attempts, - strategy_used=snap.strategy_used, - render_ms=render_ms, - execution_tier=snap.execution_tier, - timeout_phase=phase, - ), - ) - - -@app.exception_handler(RequestValidationError) -async def request_schema_validation_handler( - request: Request, exc: RequestValidationError -) -> JSONResponse: - errors = list(exc.errors()) - url = _url_from_validation_body(exc.body) - field = _first_schema_field(errors) - request_id, _ = resolve_request_id( - request.headers.get("X-Request-Id"), - host=urlparse(url).hostname if url else None, - ) - logger.info( - "request_schema_422 host=%s field=%s", - urlparse(url).hostname if url else None, - field, - ) - return JSONResponse( - status_code=422, - content=_json( - validation_error( - url, - _validation_error_message(errors), - request_id=request_id, - ) - ), - ) - - -@app.get( - "/health", - response_model=HealthResponse, - operation_id="get-health", - tags=["health"], - summary="Health", - description="Return liveness status, service name, and the installed Botasaurus version.", - responses={ - 200: { - "description": "Service is up.", - "content": {"application/json": {"example": HEALTH_EXAMPLE}}, - } - }, -) -def health() -> HealthResponse: - try: - botasaurus_version = version("botasaurus") - except PackageNotFoundError: - botasaurus_version = "unknown" - - return HealthResponse( - status="ok", - service="botasaurus-scrape-api", - botasaurus_version=botasaurus_version, + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncGenerator[None]: + app.state.settings = resolved_settings + app.state.engine = ScraperEngine(settings=resolved_settings) + app.state.executor = ThreadPoolExecutor( + max_workers=max(1, resolved_settings.scrape_max_workers) + ) + yield + app.state.executor.shutdown(wait=False, cancel_futures=True) + flush_sentry() + + from app.api.routes import health, scrape + + app = FastAPI( + title="Botasaurus Scrape API", + description=openapi_metadata.api_description.strip(), + version="2.0.0", + contact=openapi_metadata.contact, + license_info=openapi_metadata.license_info, + servers=openapi_metadata.servers, + openapi_tags=openapi_metadata.openapi_tags, + lifespan=lifespan, ) + register_exception_handlers(app) + app.include_router(health.create_router()) + app.include_router(scrape.create_router()) -@app.post( - "/scrape", - response_model=ScrapeSuccess, - responses={ - 200: { - "description": "Rendered HTML plus diagnostics. `html` is UTF-8-normalized.", - "content": {"application/json": {"example": SCRAPE_SUCCESS_EXAMPLE}}, - }, - **_SCRAPE_ERROR_RESPONSES, - }, - operation_id="scrape-url", - tags=["scrape"], - summary="Scrape a URL", - description=( - "Fetch rendered HTML for a public http(s) URL. Invalid or blocked " - "targets return `ScrapeError`. `wait_timeout_seconds` is clamped, not 422." - ), -) -async def scrape( - payload: ScrapeRequest, - x_request_id: str | None = Header(None, alias="X-Request-Id"), -) -> JSONResponse: - target_url = str(payload.url) - target_host = urlparse(target_url).hostname - request_id, _ = resolve_request_id(x_request_id, host=target_host) - - target_validation = UrlGuard.validate(target_url) - if not target_validation.is_allowed: - return JSONResponse( - status_code=target_validation.status_code, - content=_json( - validation_error( - target_url, - target_validation.error_message or "Target URL is blocked", - request_id=request_id, - ) - ), - ) - - if payload.proxy: - proxy_validation = UrlGuard.validate_proxy(str(payload.proxy)) - if not proxy_validation.is_allowed: - return JSONResponse( - status_code=proxy_validation.status_code, - content=_json( - validation_error( - target_url, - proxy_validation.error_message - or "Proxy URL is invalid or blocked", - request_id=request_id, - ) - ), - ) + return app - started_monotonic = time.monotonic() - deadline_monotonic = started_monotonic + DEFAULT_SCRAPE_TIMEOUT_SECONDS - progress = ScrapeProgress() - try: - loop = asyncio.get_running_loop() - result = await asyncio.wait_for( - loop.run_in_executor( - _executor, - partial( - _engine.execute, - payload, - deadline_monotonic, - request_id=request_id, - progress=progress, - ), - ), - timeout=DEFAULT_SCRAPE_TIMEOUT_SECONDS, - ) - except RuntimeError as exc: - if str(exc) != "request id collision detected": - raise - collision_result = ScrapeError( - url=target_url, - error="Request id collision detected", - error_category=ErrorCategory.NAVIGATION_ERROR, - diagnostics=ScrapeDiagnostics( - request_id=request_id, - attempts=0, - render_ms=0, - ), - ) - emit_terminal_telemetry(collision_result, http_status=502) - return JSONResponse(status_code=502, content=_json(collision_result)) - except TimeoutError: - timeout_result = handler_timeout_error( - target_url, - request_id=request_id, - started_monotonic=started_monotonic, - progress=progress, - ) - phase = timeout_result.diagnostics.timeout_phase or TimeoutPhase.QUEUE - logger.warning( - "scrape_timeout host=%s mode=%s timeout_seconds=%d phase=%s attempts=%d", - urlparse(target_url).hostname, - payload.navigation_mode, - DEFAULT_SCRAPE_TIMEOUT_SECONDS, - phase.value, - timeout_result.diagnostics.attempts, - ) - emit_terminal_telemetry(timeout_result, http_status=504) - return JSONResponse(status_code=504, content=_json(timeout_result)) - - status_code = 200 if isinstance(result, ScrapeSuccess) else 502 - if isinstance(result, ScrapeError): - emit_terminal_telemetry(result, http_status=status_code) - logger.info( - "scrape_complete request_id=%s host=%s mode=%s tier=%s attempts=%s status=%d error_category=%s", - result.diagnostics.request_id, - urlparse(target_url).hostname, - payload.navigation_mode, - result.diagnostics.execution_tier, - result.diagnostics.attempts, - status_code, - result.error_category if isinstance(result, ScrapeError) else None, - ) - return JSONResponse(status_code=status_code, content=_json(result)) +app = create_app() diff --git a/app/schemas.py b/app/schemas.py deleted file mode 100644 index 2bdabea..0000000 --- a/app/schemas.py +++ /dev/null @@ -1,485 +0,0 @@ -import logging -import os -import uuid -from enum import StrEnum -from typing import Any, Literal -from urllib.parse import urlparse - -from pydantic import ( - BaseModel, - ConfigDict, - Field, - HttpUrl, - ValidationInfo, - field_validator, -) - -logger = logging.getLogger("botasaurus_scrape_api") - -DEFAULT_SCRAPE_TIMEOUT_SECONDS = int(os.getenv("SCRAPE_TIMEOUT_SECONDS", "45")) -DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS = int( - os.getenv("SCRAPE_WORK_TIMEOUT_SECONDS", "30") -) -if DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS > DEFAULT_SCRAPE_TIMEOUT_SECONDS: - raise ValueError( - "SCRAPE_WORK_TIMEOUT_SECONDS cannot exceed SCRAPE_TIMEOUT_SECONDS: " - f"work={DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS} total={DEFAULT_SCRAPE_TIMEOUT_SECONDS}" - ) -DEFAULT_WAIT_TIMEOUT_SECONDS = min(15, DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS) - - -class ExecutionMode(StrEnum): - AUTO = "auto" - REQUEST = "request" - BROWSER = "browser" - - -class NavigationMode(StrEnum): - AUTO = "auto" - GET = "get" - GOOGLE_GET = "google_get" - GOOGLE_GET_BYPASS = "google_get_bypass" - ORGANIC_GET = "organic_get" - - -class ExecutionTier(StrEnum): - HTTP_REQUEST = "http_request" - BROWSER_DRIVER = "browser_driver" - - -class ErrorCategory(StrEnum): - TIMEOUT = "timeout" - CHALLENGE_BLOCK = "challenge_block" - NAVIGATION_ERROR = "navigation_error" - METADATA_ERROR = "metadata_error" - VALIDATION = "validation" - - -class TimeoutPhase(StrEnum): - QUEUE = "queue" - BOOT = "boot" - WORK = "work" - - -class WindowSize(BaseModel): - width: int = Field( - ge=1, - description="Browser viewport width in pixels.", - examples=[1920], - ) - height: int = Field( - ge=1, - description="Browser viewport height in pixels.", - examples=[1080], - ) - - -class ChallengeSignal(BaseModel): - blocked: bool = Field( - description="True when HTTP status or HTML markers indicate an anti-bot block.", - examples=[False], - ) - detected: bool = Field( - description="True when a known challenge interstitial or driver bot-detection signal matched.", - examples=[False], - ) - marker: str | None = Field( - default=None, - description="Matched challenge marker, or null when none matched.", - examples=["Just a moment..."], - ) - - -class ScrapeDiagnostics(BaseModel): - request_id: str = Field( - description="Unique id for this scrape attempt, used for tracing and runtime isolation.", - examples=["b01ef2f8-f641-4e75-8ef2-0b73f7b4f372"], - ) - attempts: int = Field( - default=0, - description="Number of navigation or request attempts actually performed.", - examples=[1], - ) - strategy_used: NavigationMode | None = Field( - default=None, - description=( - "Browser navigation strategy used on the final attempt. Null for the " - "HTTP-request tier and for failures before navigation starts." - ), - examples=["google_get"], - ) - render_ms: int = Field( - default=0, - description="Elapsed scrape runtime in milliseconds.", - examples=[154], - ) - execution_tier: ExecutionTier | None = Field( - default=None, - description="Tier that produced this result, or null when execution never started.", - examples=["http_request"], - ) - challenge: ChallengeSignal | None = Field( - default=None, - description="Anti-bot assessment for this attempt, or null when detection did not run.", - examples=[{"blocked": False, "detected": False, "marker": None}], - ) - timeout_phase: TimeoutPhase | None = Field( - default=None, - description=( - "When `error_category` is `timeout`, which stage burned the budget: " - "`queue` (threadpool wait), `boot` (browser/driver start), or `work` " - "(navigate/wait/scroll). Null for non-timeout outcomes." - ), - examples=["work"], - ) - - -class XhrResponse(BaseModel): - url: str = Field( - description="Sub-resource URL whose JSON body was captured.", - examples=["https://api.example.com/items"], - ) - status_code: int = Field( - description="HTTP status code of the captured XHR/fetch response.", - examples=[200], - ) - headers: dict[str, str] = Field( - default_factory=dict, - description="Allowlisted response headers. Only `content-type` is kept.", - examples=[{"content-type": "application/json"}], - ) - body: str = Field( - description="Raw JSON response body as UTF-8 text.", - examples=['{"items":[]}'], - ) - - -class ScrapeRequest(BaseModel): - url: HttpUrl = Field( - description=( - "Absolute http(s) URL to scrape. Localhost and private destinations " - "are rejected by SSRF guardrails." - ), - examples=["https://example.com"], - ) - execution_mode: ExecutionMode = Field( - default=ExecutionMode.AUTO, - description=( - "`auto` tries anti-detect HTTP first and escalates to the browser; " - "`request` stays on HTTP; `browser` always uses Chromium." - ), - examples=["auto"], - ) - navigation_mode: NavigationMode = Field( - default=NavigationMode.AUTO, - description=( - "Browser navigation strategy. `auto` tries `google_get`, then " - "`google_get_bypass`, then `get`." - ), - examples=["auto"], - ) - max_retries: int = Field( - default=2, - ge=0, - le=3, - description="Retries after the first attempt (`attempts = 1 + max_retries`). `auto` is capped at three strategy steps.", - examples=[2], - ) - wait_for_selector: str | None = Field( - default=None, - description="CSS selector to wait for before capture. When set, execution uses the browser tier.", - examples=["h1"], - ) - wait_timeout_seconds: int = Field( - default=DEFAULT_WAIT_TIMEOUT_SECONDS, - description=( - "Selector wait timeout in seconds. Values outside " - f"[1, {DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS}] are clamped into that " - "range so scrape still runs; they are not rejected with 422." - ), - examples=[15], - ) - scroll: bool = Field( - default=False, - description="Scroll to the bottom to trigger lazy-loaded content. Routes to the browser tier when true.", - examples=[False], - ) - block_images: bool = Field( - default=True, - description="Ask the driver to skip image downloads.", - examples=[True], - ) - block_images_and_css: bool = Field( - default=False, - description="Ask the driver to skip image and CSS downloads.", - examples=[False], - ) - block_trackers: bool = Field( - default=True, - description="Block tracking/ad networks and web fonts to speed up rendering.", - examples=[True], - ) - wait_for_complete_page_load: bool = Field( - default=True, - description="Wait for the driver complete-page-load signal before capture.", - examples=[True], - ) - user_agent: str | None = Field( - default=None, - description="Explicit User-Agent. Overrides a User-Agent header when both are set.", - examples=["Mozilla/5.0 (compatible; html2rss)"], - ) - headers: dict[str, str] | None = Field( - default=None, - description="Extra HTTP headers forwarded to the request client or browser session.", - examples=[{"Accept-Language": "en-US,en;q=0.9"}], - ) - cookies: dict[str, str] | None = Field( - default=None, - description="Cookie name/value map forwarded to the request client or browser session.", - examples=[{"session": "abc123"}], - ) - window_size: WindowSize | None = Field( - default=None, - description="Browser viewport size passed to the driver.", - examples=[{"width": 1920, "height": 1080}], - ) - lang: str | None = Field( - default=None, - description="Browser language passed to the driver.", - examples=["en-US"], - ) - headless: bool = Field( - default=False, - description="Run Chromium headless. Default false uses a virtual display in Docker.", - examples=[False], - ) - proxy: str | None = Field( - default=None, - description="Proxy URL passed to the driver. Invalid or blocked proxy URLs are rejected by SSRF guardrails.", - examples=["http://user:pass@proxy.example:8080"], - ) - - model_config = ConfigDict( - json_schema_extra={ - "examples": [ - {"url": "https://example.com"}, - { - "url": "https://example.com", - "execution_mode": "auto", - "navigation_mode": "auto", - "max_retries": 2, - "wait_for_selector": "h1", - "wait_timeout_seconds": 15, - "scroll": True, - "block_images": True, - "block_images_and_css": False, - "block_trackers": True, - "wait_for_complete_page_load": True, - "user_agent": "Mozilla/5.0 (compatible; html2rss)", - "headers": {"Accept-Language": "en-US,en;q=0.9"}, - "cookies": {"session": "abc123"}, - "window_size": {"width": 1920, "height": 1080}, - "lang": "en-US", - "headless": False, - "proxy": "http://user:pass@proxy.example:8080", - }, - ] - } - ) - - @field_validator("wait_timeout_seconds", mode="before") - @classmethod - def clamp_wait_timeout_seconds(cls, value: Any, info: ValidationInfo) -> Any: - if value is None: - return value - try: - numeric = int(value) - except (TypeError, ValueError): # fmt: skip - return value - - clamped = max(1, min(numeric, DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS)) - if clamped != numeric: - url = info.data.get("url") - logger.info( - "request_field_clamped host=%s field=%s from=%s to=%s", - urlparse(str(url)).hostname if url else None, - "wait_timeout_seconds", - numeric, - clamped, - ) - return clamped - - @property - def effective_user_agent(self) -> str | None: - if self.user_agent: - return self.user_agent - if self.headers: - return self.headers.get("User-Agent") or self.headers.get("user-agent") - return None - - -class ScrapeSuccess(BaseModel): - url: str = Field( - description="Requested scrape URL as submitted.", - examples=["https://example.com"], - ) - final_url: str | None = Field( - default=None, - description="Best-effort landing URL after redirects.", - examples=["https://example.com/"], - ) - status_code: int | None = Field( - default=None, - description="Best-effort HTTP status of the main document.", - examples=[200], - ) - headers: dict[str, str] | None = Field( - default=None, - description=( - "Best-effort response headers. When `html` is present, `content-type` " - "is `text/html; charset=utf-8`." - ), - examples=[{"content-type": "text/html; charset=utf-8"}], - ) - html: str = Field( - description=( - "Rendered page HTML, UTF-8-normalized. When present, headers " - "`content-type` is `text/html; charset=utf-8`." - ), - examples=["Example Domain"], - ) - metadata_error: str | None = Field( - default=None, - description="Metadata extraction failure message. HTML success is still returned.", - examples=[None], - ) - xhr_responses: list[XhrResponse] = Field( - default_factory=list, - description=( - "JSON XHR/fetch sub-resource bodies captured on the browser tier " - "(empty on the HTTP-request tier)." - ), - examples=[[]], - ) - diagnostics: ScrapeDiagnostics = Field( - description="Per-request tracing, strategy, timing, and challenge signals.", - examples=[ - { - "request_id": "b01ef2f8-f641-4e75-8ef2-0b73f7b4f372", - "attempts": 1, - "strategy_used": None, - "render_ms": 154, - "execution_tier": "http_request", - "challenge": {"blocked": False, "detected": False, "marker": None}, - } - ], - ) - - -class ScrapeError(BaseModel): - url: str = Field( - description="Requested scrape URL as submitted, or empty when the body had no URL.", - examples=["https://example.com"], - ) - error: str = Field( - description="Human-readable failure message.", - examples=["Target URL is blocked"], - ) - error_category: ErrorCategory = Field( - description=( - "Closed failure class: `timeout`, `challenge_block`, `navigation_error`, " - "`metadata_error`, or `validation`." - ), - examples=["validation"], - ) - diagnostics: ScrapeDiagnostics = Field( - description="Per-request tracing, strategy, timing, and challenge signals.", - examples=[ - { - "request_id": "b01ef2f8-f641-4e75-8ef2-0b73f7b4f372", - "attempts": 0, - "strategy_used": None, - "render_ms": 0, - "execution_tier": None, - "challenge": None, - } - ], - ) - - -class HealthResponse(BaseModel): - status: Literal["ok"] = Field( - description="Liveness marker. Always `ok` when the process can serve requests.", - examples=["ok"], - ) - service: str = Field( - description="Service identity.", - examples=["botasaurus-scrape-api"], - ) - botasaurus_version: str = Field( - description="Installed Botasaurus package version, or `unknown` if metadata is missing.", - examples=["4.0.91"], - ) - - model_config = ConfigDict( - json_schema_extra={ - "examples": [ - { - "status": "ok", - "service": "botasaurus-scrape-api", - "botasaurus_version": "4.0.91", - } - ] - } - ) - - -def validation_error( - url: str, message: str, *, request_id: str | None = None -) -> ScrapeError: - return ScrapeError( - url=url, - error=message, - error_category=ErrorCategory.VALIDATION, - diagnostics=ScrapeDiagnostics(request_id=request_id or str(uuid.uuid4())), - ) - - -SCRAPE_SUCCESS_EXAMPLE = { - "url": "https://example.com", - "final_url": "https://example.com/", - "status_code": 200, - "headers": {"content-type": "text/html; charset=utf-8"}, - "html": "Example Domain", - "metadata_error": None, - "xhr_responses": [], - "diagnostics": { - "request_id": "b01ef2f8-f641-4e75-8ef2-0b73f7b4f372", - "attempts": 1, - "strategy_used": None, - "render_ms": 154, - "execution_tier": "http_request", - "challenge": {"blocked": False, "detected": False, "marker": None}, - }, -} - -SCRAPE_ERROR_EXAMPLE = { - "url": "https://example.com", - "error": "Target URL is blocked", - "error_category": "validation", - "diagnostics": { - "request_id": "b01ef2f8-f641-4e75-8ef2-0b73f7b4f372", - "attempts": 0, - "strategy_used": None, - "render_ms": 0, - "execution_tier": None, - "challenge": None, - }, -} - -HEALTH_EXAMPLE = { - "status": "ok", - "service": "botasaurus-scrape-api", - "botasaurus_version": "4.0.91", -} diff --git a/app/schemas/__init__.py b/app/schemas/__init__.py new file mode 100644 index 0000000..d2f351a --- /dev/null +++ b/app/schemas/__init__.py @@ -0,0 +1 @@ +"""Wire Pydantic models for scrape API request and response envelopes.""" diff --git a/app/schemas/enums.py b/app/schemas/enums.py new file mode 100644 index 0000000..ba5387e --- /dev/null +++ b/app/schemas/enums.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from enum import StrEnum + + +class ExecutionMode(StrEnum): + AUTO = "auto" + REQUEST = "request" + BROWSER = "browser" + + +class NavigationMode(StrEnum): + AUTO = "auto" + GET = "get" + GOOGLE_GET = "google_get" + GOOGLE_GET_BYPASS = "google_get_bypass" + ORGANIC_GET = "organic_get" + + +class ExecutionTier(StrEnum): + HTTP_REQUEST = "http_request" + BROWSER_DRIVER = "browser_driver" + + +class ErrorCategory(StrEnum): + TIMEOUT = "timeout" + CHALLENGE_BLOCK = "challenge_block" + NAVIGATION_ERROR = "navigation_error" + METADATA_ERROR = "metadata_error" + VALIDATION = "validation" + + +class TimeoutPhase(StrEnum): + QUEUE = "queue" + BOOT = "boot" + WORK = "work" diff --git a/app/schemas/request.py b/app/schemas/request.py new file mode 100644 index 0000000..4903b19 --- /dev/null +++ b/app/schemas/request.py @@ -0,0 +1,204 @@ +from __future__ import annotations + +from typing import Any +from urllib.parse import urlparse + +from pydantic import ( + BaseModel, + ConfigDict, + Field, + HttpUrl, + ValidationInfo, + field_validator, +) + +from app.config import get_settings +from app.logging_config import get_logger +from app.schemas.enums import ExecutionMode, NavigationMode + +logger = get_logger() + +_WAIT_TIMEOUT_DESCRIPTION = ( + "Selector wait timeout in seconds. Values outside " + "[1, SCRAPE_WORK_TIMEOUT_SECONDS] are clamped into that " + "range so scrape still runs; they are not rejected with 422." +) + + +def _default_wait_timeout_seconds() -> int: + return min(15, get_settings().scrape_work_timeout_seconds) + + +class WindowSize(BaseModel): + width: int = Field( + ge=1, + description="Browser viewport width in pixels.", + examples=[1920], + ) + height: int = Field( + ge=1, + description="Browser viewport height in pixels.", + examples=[1080], + ) + + +class ScrapeRequest(BaseModel): + url: HttpUrl = Field( + description=( + "Absolute http(s) URL to scrape. Localhost and private destinations " + "are rejected by SSRF guardrails." + ), + examples=["https://example.com"], + ) + execution_mode: ExecutionMode = Field( + default=ExecutionMode.AUTO, + description=( + "`auto` tries anti-detect HTTP first and escalates to the browser; " + "`request` stays on HTTP; `browser` always uses Chromium." + ), + examples=["auto"], + ) + navigation_mode: NavigationMode = Field( + default=NavigationMode.AUTO, + description=( + "Browser navigation strategy. `auto` tries `google_get`, then " + "`google_get_bypass`, then `get`." + ), + examples=["auto"], + ) + max_retries: int = Field( + default=2, + ge=0, + le=3, + description="Retries after the first attempt (`attempts = 1 + max_retries`). `auto` is capped at three strategy steps.", + examples=[2], + ) + wait_for_selector: str | None = Field( + default=None, + description="CSS selector to wait for before capture. When set, execution uses the browser tier.", + examples=["h1"], + ) + wait_timeout_seconds: int = Field( + default_factory=_default_wait_timeout_seconds, + description=_WAIT_TIMEOUT_DESCRIPTION, + examples=[15], + json_schema_extra={"default": 15}, + ) + scroll: bool = Field( + default=False, + description="Scroll to the bottom to trigger lazy-loaded content. Routes to the browser tier when true.", + examples=[False], + ) + block_images: bool = Field( + default=True, + description="Ask the driver to skip image downloads.", + examples=[True], + ) + block_images_and_css: bool = Field( + default=False, + description="Ask the driver to skip image and CSS downloads.", + examples=[False], + ) + block_trackers: bool = Field( + default=True, + description="Block tracking/ad networks and web fonts to speed up rendering.", + examples=[True], + ) + wait_for_complete_page_load: bool = Field( + default=True, + description="Wait for the driver complete-page-load signal before capture.", + examples=[True], + ) + user_agent: str | None = Field( + default=None, + description="Explicit User-Agent. Overrides a User-Agent header when both are set.", + examples=["Mozilla/5.0 (compatible; html2rss)"], + ) + headers: dict[str, str] | None = Field( + default=None, + description="Extra HTTP headers forwarded to the request client or browser session.", + examples=[{"Accept-Language": "en-US,en;q=0.9"}], + ) + cookies: dict[str, str] | None = Field( + default=None, + description="Cookie name/value map forwarded to the request client or browser session.", + examples=[{"session": "abc123"}], + ) + window_size: WindowSize | None = Field( + default=None, + description="Browser viewport size passed to the driver.", + examples=[{"width": 1920, "height": 1080}], + ) + lang: str | None = Field( + default=None, + description="Browser language passed to the driver.", + examples=["en-US"], + ) + headless: bool = Field( + default=False, + description="Run Chromium headless. Default false uses a virtual display in Docker.", + examples=[False], + ) + proxy: str | None = Field( + default=None, + description="Proxy URL passed to the driver. Invalid or blocked proxy URLs are rejected by SSRF guardrails.", + examples=["http://user:pass@proxy.example:8080"], + ) + + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + {"url": "https://example.com"}, + { + "url": "https://example.com", + "execution_mode": "auto", + "navigation_mode": "auto", + "max_retries": 2, + "wait_for_selector": "h1", + "wait_timeout_seconds": 15, + "scroll": True, + "block_images": True, + "block_images_and_css": False, + "block_trackers": True, + "wait_for_complete_page_load": True, + "user_agent": "Mozilla/5.0 (compatible; html2rss)", + "headers": {"Accept-Language": "en-US,en;q=0.9"}, + "cookies": {"session": "abc123"}, + "window_size": {"width": 1920, "height": 1080}, + "lang": "en-US", + "headless": False, + "proxy": "http://user:pass@proxy.example:8080", + }, + ] + } + ) + + @field_validator("wait_timeout_seconds", mode="before") + @classmethod + def clamp_wait_timeout_seconds(cls, value: Any, info: ValidationInfo) -> Any: + if value is None: + return value + try: + numeric = int(value) + except (TypeError, ValueError): # fmt: skip + return value + + clamped = max(1, min(numeric, get_settings().scrape_work_timeout_seconds)) + if clamped != numeric: + url = info.data.get("url") + logger.info( + "request_field_clamped host=%s field=%s from=%s to=%s", + urlparse(str(url)).hostname if url else None, + "wait_timeout_seconds", + numeric, + clamped, + ) + return clamped + + @property + def effective_user_agent(self) -> str | None: + if self.user_agent: + return self.user_agent + if self.headers: + return self.headers.get("User-Agent") or self.headers.get("user-agent") + return None diff --git a/app/schemas/response.py b/app/schemas/response.py new file mode 100644 index 0000000..f879fa5 --- /dev/null +++ b/app/schemas/response.py @@ -0,0 +1,224 @@ +from __future__ import annotations + +import uuid +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + +from app.constants import SERVICE_NAME +from app.schemas.enums import ( + ErrorCategory, + ExecutionTier, + NavigationMode, + TimeoutPhase, +) + + +class ChallengeSignal(BaseModel): + blocked: bool = Field( + description="True when HTTP status or HTML markers indicate an anti-bot block.", + examples=[False], + ) + detected: bool = Field( + description="True when a known challenge interstitial or driver bot-detection signal matched.", + examples=[False], + ) + marker: str | None = Field( + default=None, + description="Matched challenge marker, or null when none matched.", + examples=["Just a moment..."], + ) + + +class ScrapeDiagnostics(BaseModel): + request_id: str = Field( + description="Unique id for this scrape attempt, used for tracing and runtime isolation.", + examples=["b01ef2f8-f641-4e75-8ef2-0b73f7b4f372"], + ) + attempts: int = Field( + default=0, + description="Number of navigation or request attempts actually performed.", + examples=[1], + ) + strategy_used: NavigationMode | None = Field( + default=None, + description=( + "Browser navigation strategy used on the final attempt. Null for the " + "HTTP-request tier and for failures before navigation starts." + ), + examples=["google_get"], + ) + render_ms: int = Field( + default=0, + description="Elapsed scrape runtime in milliseconds.", + examples=[154], + ) + execution_tier: ExecutionTier | None = Field( + default=None, + description="Tier that produced this result, or null when execution never started.", + examples=["http_request"], + ) + challenge: ChallengeSignal | None = Field( + default=None, + description="Anti-bot assessment for this attempt, or null when detection did not run.", + examples=[{"blocked": False, "detected": False, "marker": None}], + ) + timeout_phase: TimeoutPhase | None = Field( + default=None, + description=( + "When `error_category` is `timeout`, which stage burned the budget: " + "`queue` (threadpool wait), `boot` (browser/driver start), or `work` " + "(navigate/wait/scroll). Null for non-timeout outcomes." + ), + examples=["work"], + ) + + +class XhrResponse(BaseModel): + url: str = Field( + description="Sub-resource URL whose JSON body was captured.", + examples=["https://api.example.com/items"], + ) + status_code: int = Field( + description="HTTP status code of the captured XHR/fetch response.", + examples=[200], + ) + headers: dict[str, str] = Field( + default_factory=dict, + description="Allowlisted response headers. Only `content-type` is kept.", + examples=[{"content-type": "application/json"}], + ) + body: str = Field( + description="Raw JSON response body as UTF-8 text.", + examples=['{"items":[]}'], + ) + + +def _empty_xhr_responses() -> list[XhrResponse]: + return [] + + +class ScrapeSuccess(BaseModel): + url: str = Field( + description="Requested scrape URL as submitted.", + examples=["https://example.com"], + ) + final_url: str | None = Field( + default=None, + description="Best-effort landing URL after redirects.", + examples=["https://example.com/"], + ) + status_code: int | None = Field( + default=None, + description="Best-effort HTTP status of the main document.", + examples=[200], + ) + headers: dict[str, str] | None = Field( + default=None, + description=( + "Best-effort response headers. When `html` is present, `content-type` " + "is `text/html; charset=utf-8`." + ), + examples=[{"content-type": "text/html; charset=utf-8"}], + ) + html: str = Field( + description=( + "Rendered page HTML, UTF-8-normalized. When present, headers " + "`content-type` is `text/html; charset=utf-8`." + ), + examples=["Example Domain"], + ) + metadata_error: str | None = Field( + default=None, + description="Metadata extraction failure message. HTML success is still returned.", + examples=[None], + ) + xhr_responses: list[XhrResponse] = Field( + default_factory=_empty_xhr_responses, + description=( + "JSON XHR/fetch sub-resource bodies captured on the browser tier " + "(empty on the HTTP-request tier)." + ), + ) + diagnostics: ScrapeDiagnostics = Field( + description="Per-request tracing, strategy, timing, and challenge signals.", + examples=[ + { + "request_id": "b01ef2f8-f641-4e75-8ef2-0b73f7b4f372", + "attempts": 1, + "strategy_used": None, + "render_ms": 154, + "execution_tier": "http_request", + "challenge": {"blocked": False, "detected": False, "marker": None}, + } + ], + ) + + +class ScrapeError(BaseModel): + url: str = Field( + description="Requested scrape URL as submitted, or empty when the body had no URL.", + examples=["https://example.com"], + ) + error: str = Field( + description="Human-readable failure message.", + examples=["Target URL is blocked"], + ) + error_category: ErrorCategory = Field( + description=( + "Closed failure class: `timeout`, `challenge_block`, `navigation_error`, " + "`metadata_error`, or `validation`." + ), + examples=["validation"], + ) + diagnostics: ScrapeDiagnostics = Field( + description="Per-request tracing, strategy, timing, and challenge signals.", + examples=[ + { + "request_id": "b01ef2f8-f641-4e75-8ef2-0b73f7b4f372", + "attempts": 0, + "strategy_used": None, + "render_ms": 0, + "execution_tier": None, + "challenge": None, + } + ], + ) + + +class HealthResponse(BaseModel): + status: Literal["ok"] = Field( + description="Liveness marker. Always `ok` when the process can serve requests.", + examples=["ok"], + ) + service: str = Field( + description="Service identity.", + examples=[SERVICE_NAME], + ) + botasaurus_version: str = Field( + description="Installed Botasaurus package version, or `unknown` if metadata is missing.", + examples=["4.0.91"], + ) + + model_config = ConfigDict( + json_schema_extra={ + "examples": [ + { + "status": "ok", + "service": SERVICE_NAME, + "botasaurus_version": "4.0.91", + } + ] + } + ) + + +def validation_error( + url: str, message: str, *, request_id: str | None = None +) -> ScrapeError: + return ScrapeError( + url=url, + error=message, + error_category=ErrorCategory.VALIDATION, + diagnostics=ScrapeDiagnostics(request_id=request_id or str(uuid.uuid4())), + ) diff --git a/app/security/__init__.py b/app/security/__init__.py new file mode 100644 index 0000000..9c3b115 --- /dev/null +++ b/app/security/__init__.py @@ -0,0 +1,5 @@ +"""Network security guardrails.""" + +from app.security.url_guard import UrlGuard, ValidationResult + +__all__ = ["UrlGuard", "ValidationResult"] diff --git a/app/security.py b/app/security/url_guard.py similarity index 96% rename from app/security.py rename to app/security/url_guard.py index f0efcf7..967c683 100644 --- a/app/security.py +++ b/app/security/url_guard.py @@ -1,4 +1,5 @@ -# app/security.py +"""SSRF guardrails for scrape target and proxy URLs.""" + from __future__ import annotations import ipaddress @@ -76,7 +77,7 @@ def validate(cls, raw_url: str) -> ValidationResult: for info in addr_infos: sockaddr = info[4] if sockaddr: - resolved_ips.add(sockaddr[0]) + resolved_ips.add(str(sockaddr[0])) for ip_text in resolved_ips: try: diff --git a/app/sentry.py b/app/sentry.py deleted file mode 100644 index fd17f0b..0000000 --- a/app/sentry.py +++ /dev/null @@ -1,124 +0,0 @@ -# app/sentry.py -from __future__ import annotations - -import logging -import os -from typing import Any - -logger = logging.getLogger("botasaurus_scrape_api") - -_INITIALIZED = False - - -def _parse_float(value: str | None, default: float) -> float: - if value is None: - return default - try: - parsed = float(value.strip()) - return max(0.0, min(1.0, parsed)) - except (ValueError, AttributeError): # fmt: skip - return default - - -def _parse_bool(value: str | None, default: bool = False) -> bool: - if value is None: - return default - normalized = value.strip().lower() - if normalized in ("true", "1", "yes", "on"): - return True - if normalized in ("false", "0", "no", "off"): - return False - return default - - -def is_sentry_enabled() -> bool: - return bool(os.getenv("SENTRY_DSN", "").strip()) - - -def sentry_is_ready() -> bool: - """Return True when Sentry DSN is set and init succeeded.""" - return is_sentry_enabled() and _INITIALIZED - - -def _before_send(event: dict[str, Any], hint: dict[str, Any]) -> dict[str, Any] | None: - tags = event.get("tags") or {} - if tags.get("error_category") == "challenge_block": - return None - - log_record = hint.get("log_record") - logger_name = event.get("logger") - if logger_name == "websocket" or getattr(log_record, "name", None) == "websocket": - return None - - logentry = event.get("logentry") or {} - message = logentry.get("formatted") or event.get("message") or "" - if "Connection to remote host was lost" in str(message): - return None - - return event - - -def setup_sentry() -> bool: - """Initialize Sentry when SENTRY_DSN is set. Returns True on success.""" - global _INITIALIZED - - dsn = os.getenv("SENTRY_DSN", "").strip() - if not dsn: - return False - - try: - import sentry_sdk - from sentry_sdk.integrations.fastapi import FastApiIntegration - from sentry_sdk.integrations.starlette import StarletteIntegration - except ImportError: - logger.warning( - "sentry_sdk_import_failed SENTRY_DSN is set but sentry-sdk package is not available" - ) - return False - - environment = ( - os.getenv("SENTRY_ENVIRONMENT") or os.getenv("ENVIRONMENT") or "production" - ).strip() - release = os.getenv("SENTRY_RELEASE") - traces_sample_rate = _parse_float(os.getenv("SENTRY_TRACES_SAMPLE_RATE"), 0.0) - profiles_sample_rate = _parse_float(os.getenv("SENTRY_PROFILES_SAMPLE_RATE"), 0.0) - send_default_pii = _parse_bool(os.getenv("SENTRY_SEND_DEFAULT_PII"), default=False) - - init_kwargs: dict[str, Any] = { - "dsn": dsn, - "environment": environment, - "traces_sample_rate": traces_sample_rate, - "send_default_pii": send_default_pii, - "integrations": [ - FastApiIntegration(), - StarletteIntegration(), - ], - "before_send": _before_send, - } - - if release: - init_kwargs["release"] = release.strip() - if profiles_sample_rate > 0.0: - init_kwargs["profiles_sample_rate"] = profiles_sample_rate - - sentry_sdk.init(**init_kwargs) - _INITIALIZED = True - - logger.info( - "sentry_initialized environment=%s release=%s traces_sample_rate=%.2f", - environment, - release, - traces_sample_rate, - ) - return True - - -def flush_sentry(timeout: float = 2.0) -> None: - if not _INITIALIZED: - return - try: - import sentry_sdk - - sentry_sdk.flush(timeout=timeout) - except Exception as exc: - logger.debug("sentry_flush_failed error=%s", str(exc)) diff --git a/docs/typing-residuals.md b/docs/typing-residuals.md new file mode 100644 index 0000000..8377702 --- /dev/null +++ b/docs/typing-residuals.md @@ -0,0 +1,55 @@ +# Typing residuals (strict mode) + +Pyright runs in **`strict`** mode for `app/` and `tests/`. Baseline before this pass: **553** errors with no relaxations; gate target: **≤28** documented residuals with **`make check` exit 0**. + +## Current gate + +| Scope | Errors | +| --- | ---: | +| `app/` | **0** | +| `tests/` | **0** | +| **Total** | **0** | + +## Test-module file directives + +Most test modules pass strict pyright with no relaxation. The remaining directives are scoped per module: + +| Module | Directive | Why | +| --- | --- | --- | +| `tests/api/test_request_schema.py` | full blanket line | dozens of raw-dict payload permutations | +| `tests/api/test_http_contract.py` | full blanket line | walks untyped `app.openapi()` dict | +| `tests/infra/test_xhr_collector.py` | full blanket line | drives protected handlers on dynamic CDP doubles | +| `tests/engine/test_timeout_progress.py` | full blanket line | `__getattr__`-based phase-probe driver | +| `tests/infra/test_sentry.py` | `reportPrivateUsage=false` | asserts module-private init state | +| `tests/engine/test_scraper_engine.py` | `reportPrivateUsage=false` | asserts `_active_request_ids` bookkeeping | + +The full blanket line is the canonical eleven-rule directive: + +```python +# pyright: reportMissingParameterType=false, reportUnknownParameterType=false, ... +``` + +**Rationale:** keeps strict checking on real argument/type errors (`reportArgumentType`, `reportReturnType`) while avoiding false positives on nested test doubles. Counted as **one policy per test module**, not per-error suppressions. Do not add the blanket line to new test modules that pass without it. + +## Intentional boundary casts (app) + +| Location | Rule avoided | Rationale | +| --- | --- | --- | +| `app/api/errors.py` | FastAPI handler registration | `RequestValidationError` handler registration uses `cast(Any, …)` because Starlette's `ExceptionHandler` union does not narrow on exception type. | +| `app/infra/metadata.py` | `getattr` on driver requests | Passive metadata reads duck-typed Botasaurus request objects; `getattr` + `list[object]` iteration at the vendor seam. | +| `tests/support/http.py` | `_EngineExecuteProxy` return | Dependency override injects a execute-only proxy; cast to `ScraperEngine` preserves FastAPI DI signature. | + +## Vendor ownership + +| Type surface | Owner | +| --- | --- | +| Botasaurus / CDP | `typings/` local `.pyi` stubs | +| Wire / domain models | `app/schemas/` | +| CDP log / pending XHR shapes | `app/infra/cdp_types.py` | +| Test `HttpUrl` construction | `tests/support/factories.py` (`scrape_request`, `example_url`) | + +## Policy + +- Do not add blanket `# type: ignore` in `app/`. +- New test code: prefer `scrape_request()` over raw `ScrapeRequest(url="…")`. +- If strict errors rise above **28**, fix or document before merging; do not reintroduce global test `executionEnvironments` relaxations. diff --git a/openapi.yaml b/openapi.yaml index bd44067..5537a87 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -423,9 +423,9 @@ components: wait_timeout_seconds: type: integer title: Wait Timeout Seconds - description: Selector wait timeout in seconds. Values outside [1, 30] are - clamped into that range so scrape still runs; they are not rejected with - 422. + description: Selector wait timeout in seconds. Values outside [1, SCRAPE_WORK_TIMEOUT_SECONDS] + are clamped into that range so scrape still runs; they are not rejected + with 422. default: 15 examples: - 15 @@ -615,8 +615,6 @@ components: title: Xhr Responses description: JSON XHR/fetch sub-resource bodies captured on the browser tier (empty on the HTTP-request tier). - examples: - - [] diagnostics: $ref: '#/components/schemas/ScrapeDiagnostics' description: Per-request tracing, strategy, timing, and challenge signals. diff --git a/pyproject.toml b/pyproject.toml index 7c2d189..a165aeb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,9 +1,41 @@ +[project] +name = "botasaurus-scrape-api" +version = "2.0.0" +description = "Docker-first FastAPI wrapper around Botasaurus for rendered HTML scraping" +readme = "README.md" +requires-python = ">=3.14" +dependencies = [ + "fastapi==0.141.1", + "uvicorn==0.52.4", + "pydantic-settings==2.13.1", + "sentry-sdk==2.68.0", + "botasaurus @ git+https://github.com/omkarcloud/botasaurus.git@7936afad5f8ca78fe0581a9dabd5f7d6964c6b1a", +] + +[project.optional-dependencies] +dev = [ + "ruff==0.16.4", + "PyYAML==6.0.3", + "pyright==1.1.406", +] + [tool.ruff] target-version = "py314" [tool.ruff.lint] select = ["E", "F", "W", "I", "UP", "B", "SIM", "RUF"] ignore = [ - "E501", # formatter owns line width - "SIM105", # keep try/except-pass in isolation/cleanup and best-effort metadata + "E501", + "SIM105", ] + +[tool.pyright] +pythonVersion = "3.14" +venvPath = "." +venv = ".venv" +stubPath = "typings" +typeCheckingMode = "strict" +include = ["app", "tests"] +exclude = [".venv"] +reportMissingTypeStubs = true +reportMissingImports = false diff --git a/requirements-dev.txt b/requirements-dev.txt index 41fbdbd..a6510a6 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,2 +1,3 @@ ruff==0.16.4 PyYAML==6.0.3 +pyright==1.1.406 diff --git a/requirements.txt b/requirements.txt index d7cc303..a830da6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ fastapi==0.141.1 uvicorn==0.52.4 +pydantic-settings==2.13.1 sentry-sdk==2.68.0 botasaurus @ git+https://github.com/omkarcloud/botasaurus.git@7936afad5f8ca78fe0581a9dabd5f7d6964c6b1a diff --git a/scripts/bench_scrape.py b/scripts/bench_scrape.py new file mode 100755 index 0000000..0443188 --- /dev/null +++ b/scripts/bench_scrape.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Time POST /scrape happy path and print wall-time percentiles.""" + +from __future__ import annotations + +import argparse +import statistics +import time + +from fastapi.testclient import TestClient + +from app.main import create_app + + +def percentile(values: list[float], pct: float) -> float: + if not values: + return 0.0 + ordered = sorted(values) + index = max(0, min(len(ordered) - 1, round((pct / 100) * (len(ordered) - 1)))) + return ordered[index] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description="Bench POST /scrape via TestClient") + parser.add_argument("--runs", type=int, default=5, help="Number of timed runs") + parser.add_argument( + "--url", + default="https://example.com", + help="Target URL (execution_mode=request for HTTP tier)", + ) + args = parser.parse_args(argv) + + app = create_app() + durations_ms: list[float] = [] + render_ms_values: list[int] = [] + + with TestClient(app) as client: + for _ in range(args.runs): + started = time.perf_counter() + response = client.post( + "/scrape", + json={"url": args.url, "execution_mode": "request"}, + ) + elapsed_ms = (time.perf_counter() - started) * 1000 + durations_ms.append(elapsed_ms) + if response.status_code == 200: + body = response.json() + render_ms_values.append( + int(body.get("diagnostics", {}).get("render_ms", 0)) + ) + + p50 = percentile(durations_ms, 50) + p95 = percentile(durations_ms, 95) + print(f"bench_runs={args.runs} wall_ms_p50={p50:.1f} wall_ms_p95={p95:.1f}") + if render_ms_values: + print( + "render_ms_p50=" + f"{statistics.median(render_ms_values):.0f} " + f"render_ms_max={max(render_ms_values)}" + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/xhr_spike.py b/scripts/xhr_spike.py deleted file mode 100644 index 801af36..0000000 --- a/scripts/xhr_spike.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -"""Phase 0 spike variant: defer get_response_body off the CDP handler thread.""" - -from __future__ import annotations - -import base64 -import sys -import threading -import traceback - -from botasaurus.browser import Driver -from botasaurus_driver import cdp -from botasaurus_driver.core.custom_storage_cdp import enable_network - -TARGET = "https://example.com/" -JSON_URL = "https://jsonplaceholder.typicode.com/todos/1" - -pending: dict[str, dict] = {} -finished_ids: list[str] = [] -collected: list[dict] = [] -lock = threading.Lock() - - -def on_response(request_id, response, _event) -> None: - url = str(response.url) - mime = (response.mime_type or "").lower() - if url.startswith("chrome:") or url.startswith("chrome-"): - return - print( - f"[ResponseReceived] id={request_id} status={response.status} mime={mime!r} url={url}" - ) - if url.rstrip("/") == TARGET.rstrip("/"): - return - if "json" not in mime: - return - with lock: - pending[str(request_id)] = { - "url": url, - "status": int(response.status), - "mime": mime, - "request_id": request_id, - } - print(f"[PENDING] id={request_id}") - - -def on_finished(event: cdp.network.LoadingFinished) -> None: - rid = str(event.request_id) - with lock: - if rid not in pending: - return - finished_ids.append(rid) - print(f"[LoadingFinished] id={rid} (deferred body fetch)") - - -def fetch_bodies(tab) -> None: - with lock: - ids = list(finished_ids) - metas = {rid: pending[rid] for rid in ids if rid in pending} - for rid, meta in metas.items(): - print(f"[FETCH] id={rid} url={meta['url']}") - try: - body, b64 = tab.send(cdp.network.get_response_body(meta["request_id"])) - if b64: - body = base64.b64decode(body).decode("utf-8", errors="replace") - print(f"[BODY] bytes={len(body)} preview={body[:240]!r}") - with lock: - collected.append({**meta, "body": body}) - pending.pop(rid, None) - except Exception as exc: - print(f"[BODY ERROR] id={rid} error={exc}") - traceback.print_exc() - - -def main() -> int: - driver = Driver( - headless=True, - enable_xvfb_virtual_display=False, - block_images=True, - wait_for_complete_page_load=True, - remove_default_browser_check_argument=True, - ) - try: - tab = driver._tab - tab.send(enable_network()) - tab.after_response_received(on_response) - tab.add_handler(cdp.network.LoadingFinished, on_finished) - - print(f"navigating to {TARGET}") - driver.get(TARGET, timeout=30) - driver.sleep(0.5) - - print(f"fetching {JSON_URL}") - js_result = driver.run_js( - f""" - return fetch({JSON_URL!r}) - .then(r => r.text()) - .then(t => t.slice(0, 120)) - .catch(e => String(e)); - """ - ) - print(f"run_js result={js_result!r}") - driver.sleep(1) - - fetch_bodies(tab) - - print(f"collected={len(collected)}") - for item in collected: - print(f"SUCCESS body={item['body'][:300]}") - - ok = any(item["body"].lstrip().startswith(("{", "[")) for item in collected) - if not ok: - print("FAIL: no decoded JSON sub-resource body captured") - return 1 - print("SPIKE_OK deferred_main_thread_fetch") - return 0 - finally: - try: - driver.close() - except Exception: - pass - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..849cb75 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Unit and contract tests for botasaurus-scrape-api.""" diff --git a/tests/api/__init__.py b/tests/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/api/test_http_contract.py b/tests/api/test_http_contract.py new file mode 100644 index 0000000..a9175a7 --- /dev/null +++ b/tests/api/test_http_contract.py @@ -0,0 +1,345 @@ +# pyright: reportMissingParameterType=false, reportUnknownParameterType=false, reportUnknownLambdaType=false, reportPrivateUsage=false, reportAttributeAccessIssue=false, reportFunctionMemberAccess=false, reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportOptionalSubscript=false, reportOptionalMemberAccess=false +import unittest + +from app.config import get_settings +from app.engine import ( + ScraperEngine, +) +from app.infra.scrape_progress import ScrapeProgress +from app.schemas.enums import ( + ExecutionTier, +) +from app.schemas.request import ScrapeRequest +from app.schemas.response import ScrapeDiagnostics, ScrapeSuccess +from tests.support.http import ExecuteSideEffect, test_client + + +class RequestIdContractTests(unittest.TestCase): + INBOUND_ID = "550e8400-e29b-41d4-a716-446655440000" + + def test_honored_request_id_on_200(self): + def fake_execute( + payload: ScrapeRequest, + deadline_monotonic: float | None = None, + *, + request_id: str | None = None, + progress: ScrapeProgress | None = None, + ) -> ScrapeSuccess: + del deadline_monotonic, progress + resolved_request_id = request_id or "req-unknown" + return ScrapeSuccess( + url=str(payload.url), + html="", + diagnostics=ScrapeDiagnostics( + request_id=resolved_request_id, + attempts=1, + render_ms=1, + execution_tier=ExecutionTier.HTTP_REQUEST, + ), + ) + + side_effect: ExecuteSideEffect = fake_execute + with test_client(execute_side_effect=side_effect) as client: + response = client.post( + "/scrape", + json={"url": "https://example.com"}, + headers={"X-Request-Id": self.INBOUND_ID}, + ) + + self.assertEqual(response.status_code, 200) + self.assertEqual(response.json()["diagnostics"]["request_id"], self.INBOUND_ID) + + def test_honored_request_id_on_400(self): + with test_client() as client: + response = client.post( + "/scrape", + json={"url": "https://this-host-does-not-exist-12345.invalid/"}, + headers={"X-Request-Id": self.INBOUND_ID}, + ) + + self.assertEqual(response.status_code, 400) + body = response.json() + self.assertEqual(body["diagnostics"]["request_id"], self.INBOUND_ID) + self.assertEqual(body["error_category"], "validation") + + def test_honored_request_id_on_422(self): + with test_client() as client: + response = client.post( + "/scrape", + json={ + "url": "https://example.com", + "window_size": [1920], + }, + headers={"X-Request-Id": self.INBOUND_ID}, + ) + + self.assertEqual(response.status_code, 422) + body = response.json() + self.assertEqual(body["diagnostics"]["request_id"], self.INBOUND_ID) + self.assertEqual(body["error_category"], "validation") + + def test_request_id_collision_returns_502(self): + from tests.support.http import test_client + + engine = ScraperEngine(settings=get_settings()) + engine.register_request_id(self.INBOUND_ID) + try: + with test_client(engine=engine) as client: + response = client.post( + "/scrape", + json={"url": "https://example.com"}, + headers={"X-Request-Id": self.INBOUND_ID}, + ) + finally: + engine.unregister_request_id(self.INBOUND_ID) + + self.assertEqual(response.status_code, 502) + body = response.json() + self.assertEqual(body["diagnostics"]["request_id"], self.INBOUND_ID) + self.assertEqual(body["error_category"], "navigation_error") + + +class SsrfGuardHttpTests(unittest.TestCase): + """Pin the SSRF guardrail at the HTTP seam (ScrapeService.process).""" + + REQUEST_ID = "550e8400-e29b-41d4-a716-446655440042" + + def _post(self, client, payload): + return client.post( + "/scrape", json=payload, headers={"X-Request-Id": self.REQUEST_ID} + ) + + def test_localhost_target_returns_403_error_envelope(self): + with test_client() as client: + for target in ("http://localhost/", "http://127.0.0.1:8080/admin"): + with self.subTest(target=target): + response = self._post(client, {"url": target}) + self.assertEqual(response.status_code, 403) + body = response.json() + self.assertEqual(body["error_category"], "validation") + self.assertEqual(body["diagnostics"]["request_id"], self.REQUEST_ID) + self.assertNotIn("html", body) + + def test_private_ip_target_returns_403(self): + with test_client() as client: + response = self._post(client, {"url": "http://192.168.1.10/"}) + self.assertEqual(response.status_code, 403) + self.assertEqual(response.json()["error_category"], "validation") + + def test_blocked_proxy_returns_403_before_execution(self): + with test_client() as client: + response = self._post( + client, + {"url": "https://example.com", "proxy": "http://127.0.0.1:9/"}, + ) + self.assertEqual(response.status_code, 403) + body = response.json() + self.assertEqual(body["error_category"], "validation") + self.assertEqual(body["diagnostics"]["request_id"], self.REQUEST_ID) + + +class SchemaValidationHttpTests(unittest.TestCase): + def test_schema_422_returns_scrape_envelope(self): + with ( + self.assertLogs("botasaurus_scrape_api", level="INFO") as captured, + test_client() as client, + ): + response = client.post( + "/scrape", + json={ + "url": "https://example.com", + "window_size": [1920], + }, + ) + + self.assertEqual(response.status_code, 422) + body = response.json() + self.assertNotIn("detail", body) + self.assertEqual(body["url"], "https://example.com") + self.assertTrue(body["error"]) + self.assertIn("window_size", body["error"]) + self.assertEqual(body["error_category"], "validation") + self.assertNotIn("html", body) + self.assertTrue(body["diagnostics"]["request_id"]) + log_text = "\n".join(captured.output) + self.assertIn("request_schema_422", log_text) + self.assertIn("host=example.com", log_text) + self.assertIn("field=window_size", log_text) + + def test_scrape_clamps_wait_timeout_instead_of_422(self): + from tests.support.http import test_client + + captured: dict[str, int] = {} + + def fake_execute( + payload: ScrapeRequest, + deadline_monotonic: float | None = None, + *, + request_id: str | None = None, + progress: ScrapeProgress | None = None, + ) -> ScrapeSuccess: + del deadline_monotonic, progress + captured["wait"] = payload.wait_timeout_seconds + return ScrapeSuccess( + url=str(payload.url), + html="", + diagnostics=ScrapeDiagnostics( + request_id=request_id or "req-wait-clamp", + attempts=1, + render_ms=1, + execution_tier=ExecutionTier.HTTP_REQUEST, + ), + ) + + side_effect: ExecuteSideEffect = fake_execute + with test_client(execute_side_effect=side_effect) as client: + response = client.post( + "/scrape", + json={ + "url": "https://example.com", + "wait_timeout_seconds": 35, + }, + ) + + self.assertNotEqual(response.status_code, 422) + self.assertEqual(response.status_code, 200) + self.assertEqual(captured["wait"], get_settings().scrape_work_timeout_seconds) + self.assertEqual(captured["wait"], 30) + + +def _schema_ref_names(node: object) -> set[str]: + names: set[str] = set() + if isinstance(node, dict): + ref = node.get("$ref") + if isinstance(ref, str) and "/schemas/" in ref: + names.add(ref.rsplit("/", 1)[-1]) + for value in node.values(): + names |= _schema_ref_names(value) + elif isinstance(node, list): + for item in node: + names |= _schema_ref_names(item) + return names + + +class OpenApiContractTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + from app.main import app + + cls.schema = app.openapi() + + def test_scrape_documents_x_request_id_header(self): + scrape = self.schema["paths"]["/scrape"]["post"] + parameters = scrape.get("parameters") or [] + header_params = { + param["name"]: param for param in parameters if param.get("in") == "header" + } + self.assertIn("X-Request-Id", header_params) + self.assertFalse(header_params["X-Request-Id"].get("required", True)) + + def test_documents_health_and_scrape_paths(self): + paths = self.schema["paths"] + self.assertIn("/health", paths) + self.assertIn("get", paths["/health"]) + self.assertIn("/scrape", paths) + self.assertIn("post", paths["/scrape"]) + + def test_operation_ids_and_tags(self): + health = self.schema["paths"]["/health"]["get"] + scrape = self.schema["paths"]["/scrape"]["post"] + self.assertEqual(health["operationId"], "get-health") + self.assertEqual(scrape["operationId"], "scrape-url") + self.assertEqual(health["tags"], ["health"]) + self.assertEqual(scrape["tags"], ["scrape"]) + tag_names = {tag["name"] for tag in self.schema["tags"]} + self.assertEqual(tag_names, {"health", "scrape"}) + for tag in self.schema["tags"]: + self.assertTrue(tag.get("description")) + + def test_info_servers_and_version(self): + info = self.schema["info"] + self.assertEqual(info["title"], "Botasaurus Scrape API") + self.assertEqual(info["version"], "2.0.0") + self.assertTrue(info.get("description")) + self.assertEqual(info["contact"]["name"], "html2rss") + self.assertEqual( + info["contact"]["url"], + "https://github.com/html2rss/botasaurus-scrape-api/issues", + ) + self.assertNotIn("email", info["contact"]) + self.assertEqual(info["license"]["name"], "MIT") + self.assertTrue(info["license"].get("url")) + servers = self.schema["servers"] + self.assertEqual(servers[0]["url"], "http://localhost:4010") + self.assertEqual(servers[0]["description"], "Local Docker (make serve)") + + def test_scrape_documents_contract_status_codes(self): + responses = self.schema["paths"]["/scrape"]["post"]["responses"] + for status in ("200", "400", "403", "422", "502", "504"): + self.assertIn(status, responses) + self.assertTrue(responses[status].get("description")) + + def test_scrape_error_statuses_use_scrape_envelope_not_fastapi_detail(self): + responses = self.schema["paths"]["/scrape"]["post"]["responses"] + success_refs = _schema_ref_names(responses["200"]) + self.assertIn("ScrapeSuccess", success_refs) + self.assertNotIn("ScrapeResponse", success_refs) + for status in ("400", "403", "422", "502", "504"): + with self.subTest(status=status): + refs = _schema_ref_names(responses[status]) + self.assertIn("ScrapeError", refs) + self.assertNotIn("ScrapeResponse", refs) + self.assertNotIn("HTTPValidationError", refs) + self.assertNotIn("ValidationError", refs) + + def test_wait_timeout_seconds_openapi_does_not_advertise_range_as_422(self): + props = self.schema["components"]["schemas"]["ScrapeRequest"]["properties"] + wait_schema = props["wait_timeout_seconds"] + self.assertNotIn("minimum", wait_schema) + self.assertNotIn("maximum", wait_schema) + description = wait_schema.get("description") or "" + self.assertIn("clamped", description) + + def test_window_size_openapi_is_object(self): + props = self.schema["components"]["schemas"]["ScrapeRequest"]["properties"] + window_schema = props["window_size"] + refs = _schema_ref_names(window_schema) + self.assertIn("WindowSize", refs) + size_schema = self.schema["components"]["schemas"]["WindowSize"] + size_props = size_schema["properties"] + self.assertIn("width", size_props) + self.assertIn("height", size_props) + self.assertNotIn("minItems", window_schema) + self.assertNotIn("maxItems", window_schema) + self.assertNotIn("scroll_to_bottom", props) + + def test_health_schema_includes_status_fields(self): + health_200 = self.schema["paths"]["/health"]["get"]["responses"]["200"] + refs = _schema_ref_names(health_200) + self.assertIn("HealthResponse", refs) + health_schema = self.schema["components"]["schemas"]["HealthResponse"] + properties = health_schema["properties"] + self.assertIn("status", properties) + self.assertIn("service", properties) + self.assertIn("botasaurus_version", properties) + status_schema = properties["status"] + self.assertTrue( + status_schema.get("const") == "ok" + or status_schema.get("enum") == ["ok"] + or "ok" in (status_schema.get("examples") or []) + ) + + def test_xhr_responses_use_xhr_response_model(self): + scrape_schema = self.schema["components"]["schemas"]["ScrapeSuccess"] + refs = _schema_ref_names(scrape_schema["properties"]["xhr_responses"]) + self.assertIn("XhrResponse", refs) + xhr_schema = self.schema["components"]["schemas"]["XhrResponse"] + properties = xhr_schema["properties"] + for field in ("url", "status_code", "headers", "body"): + self.assertIn(field, properties) + self.assertIn("diagnostics", scrape_schema["properties"]) + self.assertNotIn("ScrapeResponse", self.schema["components"]["schemas"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/api/test_request_schema.py b/tests/api/test_request_schema.py new file mode 100644 index 0000000..f13a6fa --- /dev/null +++ b/tests/api/test_request_schema.py @@ -0,0 +1,351 @@ +# pyright: reportMissingParameterType=false, reportUnknownParameterType=false, reportUnknownLambdaType=false, reportPrivateUsage=false, reportAttributeAccessIssue=false, reportFunctionMemberAccess=false, reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportOptionalSubscript=false, reportOptionalMemberAccess=false +import tempfile +import unittest +from pathlib import Path +from unittest.mock import MagicMock, patch + +from pydantic import ValidationError + +from app.config import get_settings +from app.engine import ( + ScraperEngine, + html_document_headers, + utf8_normalize_html, +) +from app.engine.strategies import ( + apply_scrolling, + resolve_strategies, +) +from app.schemas.enums import ( + ErrorCategory, + ExecutionMode, + ExecutionTier, + NavigationMode, +) +from app.schemas.request import WindowSize +from app.schemas.response import ScrapeError, ScrapeSuccess +from tests.support.factories import scrape_request +from tests.support.fakes import ( + ArticleDriver, + CaptureDriver, + FakeDriver, + FakeHttpResponse, + FakeRequest, +) + + +class RequestSchemaTests(unittest.TestCase): + def test_request_defaults(self): + payload = scrape_request() + self.assertEqual(payload.execution_mode, ExecutionMode.AUTO) + self.assertEqual(payload.navigation_mode, NavigationMode.AUTO) + self.assertEqual(payload.max_retries, 2) + self.assertEqual(payload.wait_timeout_seconds, 15) + self.assertFalse(payload.scroll) + self.assertTrue(payload.block_images) + self.assertFalse(payload.block_images_and_css) + self.assertTrue(payload.block_trackers) + self.assertTrue(payload.wait_for_complete_page_load) + self.assertIsNone(payload.user_agent) + self.assertIsNone(payload.headers) + self.assertIsNone(payload.cookies) + self.assertIsNone(payload.window_size) + self.assertIsNone(payload.lang) + self.assertFalse(payload.headless) + self.assertIsNone(payload.proxy) + + def test_scroll_parameters(self): + req_scroll = scrape_request(scroll=True) + self.assertTrue(req_scroll.scroll) + self.assertFalse(scrape_request().scroll) + + def test_window_size_validation_requires_object(self): + with self.assertRaises(ValidationError): + scrape_request(window_size=[1920, 1080]) + with self.assertRaises(ValidationError): + scrape_request(window_size={"width": 1920}) + + def test_wait_timeout_seconds_clamps_above_work_cap(self): + with self.assertLogs("botasaurus_scrape_api", level="INFO") as captured: + payload = scrape_request(wait_timeout_seconds=35) + + settings = get_settings() + self.assertEqual( + payload.wait_timeout_seconds, settings.scrape_work_timeout_seconds + ) + self.assertEqual(settings.scrape_work_timeout_seconds, 30) + self.assertEqual(settings.scrape_timeout_seconds, 45) + log_text = "\n".join(captured.output) + self.assertIn("host=example.com", log_text) + self.assertIn("field=wait_timeout_seconds", log_text) + self.assertIn("from=35", log_text) + self.assertIn("to=30", log_text) + + def test_wait_timeout_seconds_clamps_below_one(self): + with self.assertLogs("botasaurus_scrape_api", level="INFO") as captured: + payload = scrape_request(wait_timeout_seconds=0) + + self.assertEqual(payload.wait_timeout_seconds, 1) + log_text = "\n".join(captured.output) + self.assertIn("field=wait_timeout_seconds", log_text) + self.assertIn("from=0", log_text) + self.assertIn("to=1", log_text) + + def test_clamped_wait_timeout_allows_execute(self): + payload = scrape_request( + execution_mode="browser", + navigation_mode="get", + max_retries=0, + wait_timeout_seconds=35, + ) + self.assertEqual( + payload.wait_timeout_seconds, get_settings().scrape_work_timeout_seconds + ) + + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + with patch("botasaurus.browser.Driver", FakeDriver): + result = engine.execute(payload) + + self.assertIsNone(result.error if isinstance(result, ScrapeError) else None) + self.assertIsInstance(result, ScrapeSuccess) + self.assertEqual( + result.html, "

Example Domain

" + ) + + def test_html_response_sets_utf8_content_type_and_normalizes_body(self): + FakeRequest.response = FakeHttpResponse( + text="

Caffè

", + status_code=200, + headers={"content-type": "application/octet-stream"}, + url="https://example.com/", + ) + payload = scrape_request( + execution_mode="request", + ) + + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + with patch("botasaurus.request.Request", FakeRequest): + result = engine.execute(payload) + + self.assertIsInstance(result, ScrapeSuccess) + self.assertEqual(result.diagnostics.execution_tier, ExecutionTier.HTTP_REQUEST) + self.assertIsNotNone(result.headers) + self.assertEqual(result.headers["content-type"], "text/html; charset=utf-8") + self.assertNotIn("application/octet-stream", result.headers.values()) + self.assertIn("Caffè", result.html) + self.assertNotIn("Caffè", result.html) + result.html.encode("utf-8") + + def test_utf8_normalize_leaves_correct_unicode_unchanged(self): + html = "

Caffè 日本語

" + self.assertEqual(utf8_normalize_html(html), html) + + normalized, headers = html_document_headers(html, {"content-type": "text/html"}) + self.assertEqual(normalized, html) + self.assertEqual(headers["content-type"], "text/html; charset=utf-8") + + FakeRequest.response = FakeHttpResponse( + text=html, + status_code=200, + headers={"content-type": "text/html"}, + url="https://example.com/", + ) + payload = scrape_request(execution_mode="request") + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + with patch("botasaurus.request.Request", FakeRequest): + result = engine.execute(payload) + + self.assertEqual(result.html, html) + self.assertEqual(result.headers["content-type"], "text/html; charset=utf-8") + + def test_request_tier_blocked_status_escalates_to_browser(self): + payload = scrape_request() + for status in (401, 403, 429): + with self.subTest(status=status): + FakeRequest.response = FakeHttpResponse( + text="Forbidden", + status_code=status, + headers={"content-type": "text/html"}, + url="https://example.com/", + ) + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine( + settings=get_settings(), runtime_root=Path(tmp) + ) + with ( + patch("botasaurus.request.Request", FakeRequest), + patch("botasaurus.browser.Driver", ArticleDriver), + ): + result = engine.execute(payload) + + self.assertIsInstance(result, ScrapeSuccess) + self.assertEqual( + result.diagnostics.execution_tier, ExecutionTier.BROWSER_DRIVER + ) + self.assertIn("
", result.html) + self.assertIn("Headline", result.html) + self.assertEqual( + result.headers["content-type"], "text/html; charset=utf-8" + ) + + def test_strategy_selection(self): + self.assertEqual( + resolve_strategies(NavigationMode.AUTO, 0), + [NavigationMode.GOOGLE_GET], + ) + self.assertEqual( + resolve_strategies(NavigationMode.AUTO, 2), + [ + NavigationMode.GOOGLE_GET, + NavigationMode.GOOGLE_GET_BYPASS, + NavigationMode.GET, + ], + ) + self.assertEqual( + resolve_strategies(NavigationMode.GET, 2), + [NavigationMode.GET, NavigationMode.GET, NavigationMode.GET], + ) + self.assertEqual( + resolve_strategies(NavigationMode.ORGANIC_GET, 2), + [ + NavigationMode.ORGANIC_GET, + NavigationMode.ORGANIC_GET, + NavigationMode.ORGANIC_GET, + ], + ) + + def test_cleanup_runs_on_navigation_error(self): + payload = scrape_request( + execution_mode="browser", + navigation_mode="get", + max_retries=0, + wait_for_selector="#missing", + wait_timeout_seconds=1, + ) + + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) + engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root) + with patch("botasaurus.browser.Driver", FakeDriver): + result = engine.execute(payload) + + self.assertEqual(result.error_category, ErrorCategory.NAVIGATION_ERROR) + self.assertEqual(list(runtime_root.iterdir()), []) + + def test_prepare_profile_dirs_enospc_returns_navigation_error(self): + payload = scrape_request( + execution_mode="browser", + navigation_mode="get", + max_retries=0, + ) + + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) + engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root) + + def boom_mkdir(*_args, exist_ok=False, **_kwargs): + if exist_ok: + return None + raise OSError(28, "No space left on device") + + with ( + patch("botasaurus.browser.Driver", FakeDriver), + patch.object(Path, "mkdir", side_effect=boom_mkdir), + ): + result = engine.execute(payload) + + self.assertEqual(result.error_category, ErrorCategory.NAVIGATION_ERROR) + self.assertIn("runtime storage full", result.error) + self.assertEqual(result.diagnostics.timeout_phase.value, "boot") + self.assertEqual(list(runtime_root.iterdir()), []) + + def test_prune_orphan_runtime_dirs_before_new_request(self): + payload = scrape_request( + execution_mode="browser", + navigation_mode="get", + max_retries=0, + ) + + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) + orphan = runtime_root / "stale-request" + orphan.mkdir() + (orphan / "profile").mkdir() + + engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root) + with patch("botasaurus.browser.Driver", FakeDriver): + result = engine.execute(payload) + + self.assertIsInstance(result, ScrapeSuccess) + self.assertEqual( + [entry.name for entry in runtime_root.iterdir()], + [], + ) + + def test_prune_orphan_runtime_dirs_before_http_request(self): + payload = scrape_request( + execution_mode="request", + ) + html = "

Example Domain

" + FakeRequest.response = FakeHttpResponse( + text=html, + status_code=200, + headers={"content-type": "text/html"}, + url="https://example.com/", + ) + + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) + orphan = runtime_root / "stale-request" + orphan.mkdir() + (orphan / "profile").mkdir() + + engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root) + with patch("botasaurus.request.Request", FakeRequest): + result = engine.execute(payload) + + self.assertEqual(result.html, html) + self.assertFalse(orphan.exists()) + self.assertEqual(list(runtime_root.iterdir()), []) + + def test_run_scrape_forwards_driver_kwargs(self): + CaptureDriver.last_init_kwargs = None + payload = scrape_request( + execution_mode="browser", + block_images=True, + block_images_and_css=True, + wait_for_complete_page_load=False, + user_agent="MyAgent/1.0", + window_size=WindowSize(width=1920, height=1080), + lang="en-US", + headless=True, + proxy="http://proxy.example:8080", + ) + + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) + engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root) + with patch("botasaurus.browser.Driver", CaptureDriver): + result = engine.execute(payload) + + self.assertIsInstance(result, ScrapeSuccess) + self.assertIsNotNone(CaptureDriver.last_init_kwargs) + self.assertTrue(CaptureDriver.last_init_kwargs["block_images"]) + self.assertTrue(CaptureDriver.last_init_kwargs["block_images_and_css"]) + self.assertFalse(CaptureDriver.last_init_kwargs["wait_for_complete_page_load"]) + self.assertEqual(CaptureDriver.last_init_kwargs["user_agent"], "MyAgent/1.0") + self.assertEqual(CaptureDriver.last_init_kwargs["window_size"], [1920, 1080]) + self.assertEqual(CaptureDriver.last_init_kwargs["lang"], "en-US") + self.assertTrue(CaptureDriver.last_init_kwargs["headless"]) + self.assertEqual( + CaptureDriver.last_init_kwargs["proxy"], "http://proxy.example:8080" + ) + + def test_apply_scrolling(self): + mock_driver = MagicMock() + mock_driver.scroll_to_bottom = MagicMock() + apply_scrolling(mock_driver) + mock_driver.scroll_to_bottom.assert_called_once() diff --git a/tests/api/test_timeout_http.py b/tests/api/test_timeout_http.py new file mode 100644 index 0000000..b50f6e4 --- /dev/null +++ b/tests/api/test_timeout_http.py @@ -0,0 +1,70 @@ +"""HTTP 504 envelope carries progress-derived timeout diagnostics.""" + +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.infra.scrape_progress import ScrapeProgress +from app.schemas.enums import ErrorCategory, ExecutionTier, TimeoutPhase +from app.schemas.request import ScrapeRequest +from app.schemas.response import ScrapeDiagnostics, ScrapeError +from tests.support.http import ExecuteSideEffect, test_client + +_URL = "https://example.com" + + +class HandlerTimeoutHttpTests(unittest.TestCase): + def test_scrape_handler_timeout_uses_progress(self): + def fake_execute( + payload: ScrapeRequest, + deadline_monotonic: float | None = None, + *, + request_id: str | None = None, + progress: ScrapeProgress | None = None, + ) -> ScrapeError: + del payload, deadline_monotonic, request_id + assert progress is not None + progress.mark( + TimeoutPhase.BOOT, execution_tier=ExecutionTier.BROWSER_DRIVER + ) + # Sentinel only: the patched wait_for raises TimeoutError, so the + # 504 envelope must come from the handler's own timeout path. + return ScrapeError( + url=_URL, + error="sentinel-discarded", + error_category=ErrorCategory.NAVIGATION_ERROR, + diagnostics=ScrapeDiagnostics(request_id="sentinel"), + ) + + side_effect: ExecuteSideEffect = fake_execute + + async def boom(awaitable: object, timeout: float | None = None) -> None: + del timeout + await awaitable # type: ignore[misc] + raise TimeoutError + + with ( + test_client(execute_side_effect=side_effect) as client, + patch("asyncio.wait_for", side_effect=boom), + ): + response = client.post( + "/scrape", + json={ + "url": _URL, + "execution_mode": "browser", + "navigation_mode": "get", + }, + ) + + self.assertEqual(response.status_code, 504) + body = response.json() + self.assertEqual(body["error_category"], "timeout") + self.assertEqual(body["diagnostics"]["timeout_phase"], "boot") + self.assertEqual(body["diagnostics"]["attempts"], 0) + self.assertEqual(body["diagnostics"]["execution_tier"], "browser_driver") + self.assertIn("phase=boot", body["error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/domain/__init__.py b/tests/domain/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/domain/test_timeout_error.py b/tests/domain/test_timeout_error.py new file mode 100644 index 0000000..3f67b40 --- /dev/null +++ b/tests/domain/test_timeout_error.py @@ -0,0 +1,55 @@ +"""ScrapeService.build_timeout_error phase and diagnostics mapping.""" + +from __future__ import annotations + +import time +import unittest + +from app.domain.scrape_service import ScrapeService +from app.infra.scrape_progress import ScrapeProgress +from app.schemas.enums import ExecutionTier, NavigationMode, TimeoutPhase + +_URL = "https://example.com" + + +class BuildTimeoutErrorTests(unittest.TestCase): + def test_queue_phase_keeps_zero_attempts(self): + result = ScrapeService.build_timeout_error( + _URL, + request_id="req-queue", + started_monotonic=time.monotonic(), + progress=ScrapeProgress(), + timeout_seconds=45, + ) + self.assertEqual(result.error_category.value, "timeout") + self.assertIn("phase=queue", result.error) + self.assertEqual(result.diagnostics.timeout_phase, TimeoutPhase.QUEUE) + self.assertEqual(result.diagnostics.attempts, 0) + self.assertIsNone(result.diagnostics.strategy_used) + + def test_work_phase_preserves_attempts_and_strategy(self): + progress = ScrapeProgress() + progress.mark( + TimeoutPhase.WORK, + attempts=2, + strategy_used=NavigationMode.GOOGLE_GET, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ) + result = ScrapeService.build_timeout_error( + _URL, + request_id="req-work", + started_monotonic=time.monotonic() - 1, + progress=progress, + timeout_seconds=45, + ) + diagnostics = result.diagnostics + self.assertEqual(diagnostics.timeout_phase, TimeoutPhase.WORK) + self.assertEqual(diagnostics.attempts, 2) + self.assertEqual(diagnostics.strategy_used, NavigationMode.GOOGLE_GET) + self.assertEqual(diagnostics.execution_tier, ExecutionTier.BROWSER_DRIVER) + self.assertIn("phase=work", result.error) + self.assertGreaterEqual(diagnostics.render_ms, 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/engine/__init__.py b/tests/engine/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/engine/test_budget.py b/tests/engine/test_budget.py new file mode 100644 index 0000000..60b8b63 --- /dev/null +++ b/tests/engine/test_budget.py @@ -0,0 +1,68 @@ +"""Pure unit tests for wall-clock budget math clamps and composition.""" + +from __future__ import annotations + +import time +import unittest + +from app.config import get_settings +from app.engine.budget import ( + browser_step_budget_seconds, + elapsed_ms, + is_timeout_exception, + remaining_total_seconds, + remaining_work_seconds, +) + + +class BudgetMathTests(unittest.TestCase): + def setUp(self): + self.settings = get_settings() + self.now = time.monotonic() + + def test_elapsed_ms_is_non_negative_and_scales(self): + self.assertGreaterEqual(elapsed_ms(self.now), 0) + self.assertGreaterEqual(elapsed_ms(self.now - 1.5), 1500) + + def test_remaining_total_counts_down_from_scrape_timeout(self): + fresh = remaining_total_seconds(self.settings, self.now) + self.assertLessEqual(fresh, self.settings.scrape_timeout_seconds) + self.assertGreaterEqual(fresh, self.settings.scrape_timeout_seconds - 1) + + def test_remaining_total_floors_at_one_when_exhausted(self): + exhausted = self.now - (self.settings.scrape_timeout_seconds + 60) + self.assertEqual(remaining_total_seconds(self.settings, exhausted), 1) + + def test_remaining_work_floors_at_one_when_exhausted(self): + exhausted = self.now - (self.settings.scrape_work_timeout_seconds + 60) + self.assertEqual(remaining_work_seconds(self.settings, exhausted), 1) + + def test_browser_step_budget_takes_the_tighter_constraint(self): + # Total budget nearly burnt, work budget fresh: total wins. + almost_burnt = self.now - (self.settings.scrape_timeout_seconds - 5) + tight_total = browser_step_budget_seconds(self.settings, almost_burnt, self.now) + self.assertLessEqual(tight_total, 5) + + # Both fresh: bounded by the smaller work budget. + fresh = browser_step_budget_seconds(self.settings, self.now, self.now) + self.assertLessEqual(fresh, self.settings.scrape_work_timeout_seconds) + self.assertGreaterEqual(fresh, 1) + + def test_browser_step_budget_never_returns_zero_or_negative(self): + long_ago = self.now - 10_000 + self.assertEqual( + browser_step_budget_seconds(self.settings, long_ago, long_ago), 1 + ) + + +class TimeoutExceptionClassificationTests(unittest.TestCase): + def test_matches_timeout_messages_case_insensitively(self): + self.assertTrue(is_timeout_exception(TimeoutError("navigation Timeout"))) + self.assertTrue(is_timeout_exception(RuntimeError("HTTP read TIMEOUT"))) + + def test_ignores_non_timeout_messages(self): + self.assertFalse(is_timeout_exception(RuntimeError("connection refused"))) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/engine/test_isolation.py b/tests/engine/test_isolation.py new file mode 100644 index 0000000..6f8dab6 --- /dev/null +++ b/tests/engine/test_isolation.py @@ -0,0 +1,196 @@ +"""Engine singleton and per-request isolation regression tests.""" + +from __future__ import annotations + +import tempfile +import threading +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.config import get_settings +from app.engine import ScraperEngine +from app.engine.session import ScrapeSession +from tests.support.fakes import fake_request_cls +from tests.support.http import test_client + + +class EngineSingletonTests(unittest.TestCase): + def test_app_state_shares_one_engine_and_executor(self): + with test_client() as client: + engine_a = client.app.state.engine + engine_b = client.app.state.engine + executor_a = client.app.state.executor + executor_b = client.app.state.executor + + self.assertIs(engine_a, engine_b) + self.assertIs(executor_a, executor_b) + + +class IsolationRegressionTests(unittest.TestCase): + INBOUND_ID_A = "550e8400-e29b-41d4-a716-446655440001" + INBOUND_ID_B = "550e8400-e29b-41d4-a716-446655440002" + COLLISION_ID = "550e8400-e29b-41d4-a716-446655440000" + + def test_concurrent_scrapes_use_distinct_runtime_dirs(self): + runtime_dirs: list[Path] = [] + gate = threading.Event() + release = threading.Event() + original_enter = ScrapeSession.__enter__ + + def tracking_enter(self: ScrapeSession) -> ScrapeSession: + session = original_enter(self) + runtime_dirs.append(session.runtime_dir) + if len(runtime_dirs) == 1: + gate.set() + release.wait(timeout=5) + return session + + with ( + patch.object(ScrapeSession, "__enter__", tracking_enter), + patch("botasaurus.request.Request", fake_request_cls()), + test_client() as client, + ): + first = threading.Thread( + target=lambda: client.post( + "/scrape", + json={"url": "https://example.com", "execution_mode": "request"}, + headers={"X-Request-Id": self.INBOUND_ID_A}, + ) + ) + second = threading.Thread( + target=lambda: client.post( + "/scrape", + json={"url": "https://example.com", "execution_mode": "request"}, + headers={"X-Request-Id": self.INBOUND_ID_B}, + ) + ) + first.start() + self.assertTrue(gate.wait(timeout=5)) + second.start() + release.set() + first.join(timeout=10) + second.join(timeout=10) + + self.assertEqual(len(runtime_dirs), 2) + self.assertNotEqual(runtime_dirs[0], runtime_dirs[1]) + + def test_duplicate_request_id_while_active_returns_502(self): + active = threading.Event() + release = threading.Event() + original_enter = ScrapeSession.__enter__ + collision_id = self.COLLISION_ID + + def slow_enter(self: ScrapeSession) -> ScrapeSession: + session = original_enter(self) + if self.request_id == collision_id: + active.set() + release.wait(timeout=5) + return session + + with ( + patch.object(ScrapeSession, "__enter__", slow_enter), + patch("botasaurus.request.Request", fake_request_cls()), + test_client() as client, + ): + first = threading.Thread( + target=lambda: client.post( + "/scrape", + json={"url": "https://example.com", "execution_mode": "request"}, + headers={"X-Request-Id": self.COLLISION_ID}, + ) + ) + first.start() + self.assertTrue(active.wait(timeout=5)) + + response = client.post( + "/scrape", + json={"url": "https://example.com", "execution_mode": "request"}, + headers={"X-Request-Id": self.COLLISION_ID}, + ) + release.set() + first.join(timeout=10) + + self.assertEqual(response.status_code, 502) + body = response.json() + self.assertEqual(body["diagnostics"]["request_id"], self.COLLISION_ID) + self.assertEqual(body["error_category"], "navigation_error") + + def test_runtime_dir_removed_after_scrape_completes(self): + captured_dir: Path | None = None + original_exit = ScrapeSession.__exit__ + + def capture_exit( + self: ScrapeSession, + exc_type: object, + exc_val: object, + exc_tb: object, + ) -> bool | None: + nonlocal captured_dir + captured_dir = self.runtime_dir + return original_exit(self, exc_type, exc_val, exc_tb) # type: ignore[arg-type] + + with ( + patch.object(ScrapeSession, "__exit__", capture_exit), + patch("botasaurus.request.Request", fake_request_cls()), + test_client() as client, + ): + response = client.post( + "/scrape", + json={"url": "https://example.com", "execution_mode": "request"}, + headers={"X-Request-Id": "req-cleanup"}, + ) + + self.assertEqual(response.status_code, 200) + self.assertIsNotNone(captured_dir) + assert captured_dir is not None + self.assertFalse(captured_dir.exists()) + + def test_shared_engine_tracks_active_request_ids(self): + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + request_id = "req-active-track" + engine.register_request_id(request_id) + try: + self.assertIn(request_id, engine._active_request_ids) # pyright: ignore[reportPrivateUsage] + finally: + engine.unregister_request_id(request_id) + self.assertNotIn(request_id, engine._active_request_ids) # pyright: ignore[reportPrivateUsage] + + def test_session_unregisters_when_prepare_runtime_fails(self): + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + request_id = "req-prepare-fail" + with ( + patch.object( + engine, + "prepare_runtime_for_request", + side_effect=OSError("boom"), + ), + self.assertRaises(OSError), + ScrapeSession(engine, request_id), + ): + pass + self.assertNotIn(request_id, engine._active_request_ids) # pyright: ignore[reportPrivateUsage] + + def test_prune_keeps_registered_request_dirs(self): + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) + engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root) + orphan = runtime_root / "orphan" + orphan.mkdir() + live = runtime_root / "live-req" + live.mkdir() + engine.register_request_id("live-req") + try: + removed = engine.prune_runtime_dirs() + finally: + engine.unregister_request_id("live-req") + + self.assertEqual(removed, 1) + self.assertFalse(orphan.exists()) + self.assertTrue(live.is_dir()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/engine/test_scraper_engine.py b/tests/engine/test_scraper_engine.py new file mode 100644 index 0000000..1c60ed8 --- /dev/null +++ b/tests/engine/test_scraper_engine.py @@ -0,0 +1,250 @@ +# pyright: reportPrivateUsage=false +import tempfile +import unittest +from pathlib import Path +from typing import Any +from unittest.mock import MagicMock, patch + +from app.config import get_settings +from app.engine import ( + ScraperEngine, +) +from app.engine.strategies import ( + wait_for_readiness, +) +from app.exceptions import RequestIdCollisionError +from app.schemas.enums import ( + ErrorCategory, + ExecutionTier, + NavigationMode, + TimeoutPhase, +) +from app.schemas.response import ( + ScrapeDiagnostics, + ScrapeError, + ScrapeSuccess, + XhrResponse, +) +from tests.support.factories import scrape_request +from tests.support.fakes import ( + FakeDriver, +) + + +class ScraperEngineUnitTests(unittest.TestCase): + def test_browser_tier_step_budget_is_boot_aware(self): + captured: dict[str, int | None] = {"navigate_timeout": None} + + class _NavigateCaptureDriver(FakeDriver): + def get(self, *_args: object, **kwargs: Any) -> None: + captured["navigate_timeout"] = kwargs.get("timeout") + return None + + payload = scrape_request( + execution_mode="browser", + navigation_mode="get", + max_retries=0, + ) + + monotonic_values = [ + 1000.0, # execute started + 1020.0, # browser ready after boot + 1020.0, # remaining total + 1020.0, # remaining work + 1020.0, # render_ms + ] + + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + with ( + patch("botasaurus.browser.Driver", _NavigateCaptureDriver), + patch( + "app.engine.browser_tier.time.monotonic", + side_effect=monotonic_values, + ), + ): + result = engine.execute(payload) + + self.assertIsInstance(result, ScrapeSuccess) + self.assertEqual(captured["navigate_timeout"], 25) + + def test_request_id_collision_raises(self): + engine = ScraperEngine(settings=get_settings()) + engine.register_request_id("req-123") + with self.assertRaises(RequestIdCollisionError): + engine.register_request_id("req-123") + engine.unregister_request_id("req-123") + # Should be re-registerable after unregistering + engine.register_request_id("req-123") + engine.unregister_request_id("req-123") + + def test_scrape_session_context_manager(self): + from app.engine import ScrapeSession + + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + with ScrapeSession(engine, "req-session-1") as session: + self.assertIn("req-session-1", engine._active_request_ids) + session.prepare_profile_dirs() + self.assertTrue(session.profile_dir.is_dir()) + + self.assertNotIn("req-session-1", engine._active_request_ids) + self.assertFalse(session.runtime_dir.exists()) + + def test_effective_user_agent_resolution(self): + req1 = scrape_request( + user_agent="CustomAgent/1.0", + headers={"User-Agent": "HeaderAgent/1.0"}, + ) + self.assertEqual(req1.effective_user_agent, "CustomAgent/1.0") + + req2 = scrape_request( + headers={"User-Agent": "HeaderAgent/1.0"}, + ) + self.assertEqual(req2.effective_user_agent, "HeaderAgent/1.0") + + req3 = scrape_request() + self.assertIsNone(req3.effective_user_agent) + + def test_scrape_envelope_constructors(self): + success = ScrapeSuccess( + url="https://example.com", + final_url="https://example.com", + status_code=200, + headers={"content-type": "text/html; charset=utf-8"}, + html="", + metadata_error=None, + xhr_responses=[], + diagnostics=ScrapeDiagnostics( + request_id="req-abc", + attempts=1, + strategy_used=NavigationMode.GET, + render_ms=120, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ), + ) + dumped = success.model_dump(mode="json") + self.assertEqual(dumped["status_code"], 200) + self.assertNotIn("error", dumped) + self.assertEqual(dumped["diagnostics"]["execution_tier"], "browser_driver") + self.assertEqual(dumped["final_url"], "https://example.com") + self.assertEqual(dumped["xhr_responses"], []) + self.assertEqual(dumped["headers"]["content-type"], "text/html; charset=utf-8") + + with_xhr = ScrapeSuccess( + url="https://example.com", + html="", + xhr_responses=[ + XhrResponse( + url="https://api.example.com/items", + status_code=200, + headers={"content-type": "application/json"}, + body='{"items":[]}', + ) + ], + diagnostics=ScrapeDiagnostics( + request_id="req-xhr", + attempts=1, + strategy_used=NavigationMode.GET, + render_ms=10, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ), + ) + self.assertEqual(len(with_xhr.xhr_responses), 1) + self.assertEqual(with_xhr.xhr_responses[0].url, "https://api.example.com/items") + + err = ScrapeError( + url="https://example.com", + error="Something broke", + error_category=ErrorCategory.NAVIGATION_ERROR, + diagnostics=ScrapeDiagnostics(request_id="req-err"), + ) + err_dump = err.model_dump(mode="json") + self.assertEqual(err_dump["error"], "Something broke") + self.assertEqual(err_dump["error_category"], "navigation_error") + self.assertNotIn("html", err_dump) + self.assertNotIn("xhr_responses", err_dump) + + def test_wait_for_readiness_uses_sleep_random_when_available(self): + mock_driver = MagicMock() + mock_driver.sleep_random = MagicMock() + wait_for_readiness(mock_driver, selector=None, timeout_seconds=10) + mock_driver.sleep_random.assert_called_once_with(0.5, 1.2) + + def test_execute_honors_submission_deadline_after_queue_wait(self): + settings = get_settings() + payload = scrape_request( + execution_mode="browser", + navigation_mode="get", + max_retries=0, + ) + submit_at = 1000.0 + deadline = submit_at + settings.scrape_timeout_seconds + worker_start = submit_at + 30.0 + captured: dict[str, float] = {} + + def fake_browser_tier( + _payload: object, + _session: object, + started_monotonic: float, + _progress: object, + *, + settings: object, + ) -> ScrapeSuccess: + del settings + captured["started_monotonic"] = started_monotonic + return ScrapeSuccess( + url="https://example.com", + html="", + diagnostics=ScrapeDiagnostics( + request_id="req-deadline", + attempts=1, + render_ms=1, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ), + ) + + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=settings, runtime_root=Path(tmp)) + with ( + patch( + "app.engine.orchestrator.time.monotonic", + return_value=worker_start, + ), + patch( + "app.engine.orchestrator.run_browser_tier", + side_effect=fake_browser_tier, + ), + ): + result = engine.execute(payload, deadline_monotonic=deadline) + + self.assertIsInstance(result, ScrapeSuccess) + self.assertEqual(captured["started_monotonic"], submit_at) + + def test_browser_driver_constructor_failure_returns_navigation_error(self): + payload = scrape_request( + execution_mode="browser", + navigation_mode="get", + max_retries=0, + ) + + with tempfile.TemporaryDirectory() as tmp: + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + with patch( + "botasaurus.browser.Driver", + side_effect=RuntimeError("chrome binary missing"), + ): + result = engine.execute(payload, request_id="req-boot-fail") + + self.assertIsInstance(result, ScrapeError) + assert isinstance(result, ScrapeError) + self.assertEqual(result.error_category, ErrorCategory.NAVIGATION_ERROR) + self.assertEqual(result.diagnostics.timeout_phase, TimeoutPhase.BOOT) + self.assertEqual( + result.diagnostics.execution_tier, ExecutionTier.BROWSER_DRIVER + ) + self.assertIn("chrome binary missing", result.error) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_timeout_phase.py b/tests/engine/test_timeout_progress.py similarity index 52% rename from tests/test_timeout_phase.py rename to tests/engine/test_timeout_progress.py index b5c2b89..13cfb7d 100644 --- a/tests/test_timeout_phase.py +++ b/tests/engine/test_timeout_progress.py @@ -1,35 +1,32 @@ -# tests/test_timeout_phase.py +# pyright: reportMissingParameterType=false, reportUnknownParameterType=false, reportUnknownLambdaType=false, reportPrivateUsage=false, reportAttributeAccessIssue=false, reportFunctionMemberAccess=false, reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportOptionalSubscript=false, reportOptionalMemberAccess=false +"""Engine progress marking across queue, boot, and work phases.""" + from __future__ import annotations import tempfile import time import unittest +from contextlib import ExitStack from pathlib import Path from types import SimpleNamespace from unittest.mock import patch +from app.config import get_settings from app.engine import ScraperEngine -from app.main import handler_timeout_error -from app.schemas import ( +from app.infra.scrape_progress import ScrapeProgress +from app.schemas.enums import ( ExecutionMode, ExecutionTier, NavigationMode, - ScrapeRequest, TimeoutPhase, ) -from app.scrape_progress import ScrapeProgress +from tests.support.factories import scrape_request +from tests.support.fakes import fake_request_cls _URL = "https://example.com" _HTML = "

Example Domain

" -def _snap_eq(test, snap, *, phase, attempts=0, strategy=None, tier=None): - test.assertEqual(snap.phase, phase) - test.assertEqual(snap.attempts, attempts) - test.assertEqual(snap.strategy_used, strategy) - test.assertEqual(snap.execution_tier, tier) - - class _PhaseProbeDriver: """Records progress phase at Driver construction time.""" @@ -56,94 +53,33 @@ def __getattr__(self, _name): return lambda *_a, **_k: None -def _fake_request_cls(html=_HTML, url=f"{_URL}/"): - response = SimpleNamespace( - text=html, - status_code=200, - headers={"content-type": "text/html"}, - url=url, - ) - return type( - "FakeRequest", - (), - {"get": lambda self, *_a, **_k: response, "close": lambda self: None}, - ) - - def _execute(payload, *, progress, request_id, **patches): with tempfile.TemporaryDirectory() as tmp: - engine = ScraperEngine(runtime_root=Path(tmp)) - with patch.multiple("app.engine", create=True, **patches): + engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp)) + with ExitStack() as stack: + if "Driver" in patches: + stack.enter_context( + patch("botasaurus.browser.Driver", patches["Driver"]) + ) + if "Request" in patches: + stack.enter_context( + patch("botasaurus.request.Request", patches["Request"]) + ) return engine.execute(payload, request_id=request_id, progress=progress) -class ScrapeProgressTests(unittest.TestCase): - def test_snapshot_defaults_to_queue(self): - _snap_eq(self, ScrapeProgress().snapshot(), phase=TimeoutPhase.QUEUE) - - def test_mark_updates_snapshot(self): - progress = ScrapeProgress() - progress.mark( - TimeoutPhase.WORK, - attempts=2, - strategy_used=NavigationMode.GET, - execution_tier=ExecutionTier.BROWSER_DRIVER, - ) - _snap_eq( - self, - progress.snapshot(), - phase=TimeoutPhase.WORK, - attempts=2, - strategy=NavigationMode.GET, - tier=ExecutionTier.BROWSER_DRIVER, - ) - - -class HandlerTimeoutErrorTests(unittest.TestCase): - def test_queue_phase_keeps_zero_attempts(self): - result = handler_timeout_error( - _URL, - request_id="req-queue", - started_monotonic=time.monotonic(), - progress=ScrapeProgress(), - timeout_seconds=45, - ) - self.assertEqual(result.error_category.value, "timeout") - self.assertIn("phase=queue", result.error) - self.assertEqual(result.diagnostics.timeout_phase, TimeoutPhase.QUEUE) - self.assertEqual(result.diagnostics.attempts, 0) - self.assertIsNone(result.diagnostics.strategy_used) - - def test_work_phase_preserves_attempts_and_strategy(self): - progress = ScrapeProgress() - progress.mark( - TimeoutPhase.WORK, - attempts=2, - strategy_used=NavigationMode.GOOGLE_GET, - execution_tier=ExecutionTier.BROWSER_DRIVER, - ) - result = handler_timeout_error( - _URL, - request_id="req-work", - started_monotonic=time.monotonic() - 1, - progress=progress, - timeout_seconds=45, - ) - d = result.diagnostics - self.assertEqual(d.timeout_phase, TimeoutPhase.WORK) - self.assertEqual(d.attempts, 2) - self.assertEqual(d.strategy_used, NavigationMode.GOOGLE_GET) - self.assertEqual(d.execution_tier, ExecutionTier.BROWSER_DRIVER) - self.assertIn("phase=work", result.error) - self.assertGreaterEqual(d.render_ms, 0) +def _snap_eq(test, snap, *, phase, attempts=0, strategy=None, tier=None): + test.assertEqual(snap.phase, phase) + test.assertEqual(snap.attempts, attempts) + test.assertEqual(snap.strategy_used, strategy) + test.assertEqual(snap.execution_tier, tier) class EngineProgressMarkTests(unittest.TestCase): def test_execute_queue_timeout_sets_phase(self): progress = ScrapeProgress() - result = ScraperEngine().execute( - ScrapeRequest( - url=_URL, + result = ScraperEngine(settings=get_settings()).execute( + scrape_request( execution_mode=ExecutionMode.BROWSER, navigation_mode=NavigationMode.GET, ), @@ -160,8 +96,7 @@ def test_browser_tier_marks_boot_before_driver_then_work(self): _PhaseProbeDriver.progress = progress _PhaseProbeDriver.construction_phase = None result = _execute( - ScrapeRequest( - url=_URL, + scrape_request( execution_mode=ExecutionMode.BROWSER, navigation_mode=NavigationMode.GET, max_retries=0, @@ -184,10 +119,10 @@ def test_browser_tier_marks_boot_before_driver_then_work(self): def test_request_tier_marks_work_with_attempt(self): progress = ScrapeProgress() result = _execute( - ScrapeRequest(url=_URL, execution_mode=ExecutionMode.REQUEST), + scrape_request(url=_URL, execution_mode=ExecutionMode.REQUEST), progress=progress, request_id="req-http-mark", - Request=_fake_request_cls(), + Request=fake_request_cls(html=_HTML, url=f"{_URL}/"), ) self.assertIsNone(getattr(result, "error", None)) _snap_eq( @@ -208,7 +143,7 @@ def close(self): progress = ScrapeProgress() result = _execute( - ScrapeRequest(url=_URL, execution_mode=ExecutionMode.REQUEST), + scrape_request(url=_URL, execution_mode=ExecutionMode.REQUEST), progress=progress, request_id="req-http-timeout", Request=BoomRequest, @@ -226,8 +161,7 @@ def get(self, *_a, **_k): progress = ScrapeProgress() BoomDriver.progress = progress result = _execute( - ScrapeRequest( - url=_URL, + scrape_request( execution_mode=ExecutionMode.BROWSER, navigation_mode=NavigationMode.GET, max_retries=0, @@ -241,45 +175,5 @@ def get(self, *_a, **_k): self.assertEqual(result.diagnostics.strategy_used, NavigationMode.GET) -class HandlerTimeoutHttpTests(unittest.TestCase): - def test_scrape_handler_timeout_uses_progress(self): - from fastapi.testclient import TestClient - - import app.main as main_mod - from app.main import app - - def fake_execute(_payload, _deadline=None, *, request_id=None, progress=None): - assert progress is not None - progress.mark( - TimeoutPhase.BOOT, execution_tier=ExecutionTier.BROWSER_DRIVER - ) - - async def boom(awaitable, timeout=None): - del timeout - await awaitable - raise TimeoutError - - with ( - patch.object(main_mod._engine, "execute", side_effect=fake_execute), - patch("asyncio.wait_for", side_effect=boom), - ): - response = TestClient(app).post( - "/scrape", - json={ - "url": _URL, - "execution_mode": "browser", - "navigation_mode": "get", - }, - ) - - self.assertEqual(response.status_code, 504) - body = response.json() - self.assertEqual(body["error_category"], "timeout") - self.assertEqual(body["diagnostics"]["timeout_phase"], "boot") - self.assertEqual(body["diagnostics"]["attempts"], 0) - self.assertEqual(body["diagnostics"]["execution_tier"], "browser_driver") - self.assertIn("phase=boot", body["error"]) - - if __name__ == "__main__": unittest.main() diff --git a/tests/infra/__init__.py b/tests/infra/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/infra/test_challenge_detector.py b/tests/infra/test_challenge_detector.py new file mode 100644 index 0000000..feaf49e --- /dev/null +++ b/tests/infra/test_challenge_detector.py @@ -0,0 +1,37 @@ +import unittest +from unittest.mock import MagicMock + +from app.infra.detector import ChallengeDetector + + +class ChallengeDetectorUnitTests(unittest.TestCase): + def test_detects_challenge_marker(self): + res = ChallengeDetector.detect("Just a moment...", 200) + self.assertTrue(res.challenge_detected) + self.assertTrue(res.blocked_detected) + self.assertEqual(res.detected_marker, "Just a moment...") + self.assertFalse(res.is_clean) + + def test_detects_http_status_block_without_marker(self): + res = ChallengeDetector.detect("Forbidden", 403) + self.assertFalse(res.challenge_detected) + self.assertTrue(res.blocked_detected) + self.assertIsNone(res.detected_marker) + self.assertFalse(res.is_clean) + + def test_clean_response(self): + res = ChallengeDetector.detect("

Hello

", 200) + self.assertTrue(res.is_clean) + self.assertFalse(res.blocked_detected) + self.assertFalse(res.challenge_detected) + + def test_driver_bot_detection_integration(self): + mock_driver = MagicMock() + mock_driver.is_bot_detected.return_value = True + + res = ChallengeDetector.detect( + "Clean page", 200, driver=mock_driver + ) + self.assertTrue(res.challenge_detected) + self.assertTrue(res.blocked_detected) + self.assertEqual(res.detected_marker, "botasaurus_driver_bot_detected") diff --git a/tests/infra/test_metadata_extractor.py b/tests/infra/test_metadata_extractor.py new file mode 100644 index 0000000..388c5f9 --- /dev/null +++ b/tests/infra/test_metadata_extractor.py @@ -0,0 +1,77 @@ +import unittest +from typing import cast + +from app.engine.driver_capabilities import DriverProtocol +from app.infra.metadata import MetadataExtractor + + +class MetadataExtractorUnitTests(unittest.TestCase): + def test_extract_passive_metadata_from_requests_list(self): + class _Req: + def __init__(self, status: int, headers: dict[str, str], url: str) -> None: + self.response = type( + "Resp", (), {"status_code": status, "headers": headers} + )() + self.url = url + + driver = type( + "D", + (), + { + "requests": [ + _Req( + 200, {"content-type": "text/html"}, "https://example.com/final" + ) + ] + }, + )() + status, headers, final_url = MetadataExtractor.extract_from_requests( + cast(DriverProtocol, driver) + ) + self.assertEqual(status, 200) + self.assertEqual(headers, {"content-type": "text/html"}) + self.assertEqual(final_url, "https://example.com/final") + + def test_extract_passive_metadata_from_performance_logs(self): + import json + + perf_log = [ + { + "message": json.dumps( + { + "message": { + "method": "Network.responseReceived", + "params": { + "type": "Document", + "response": { + "status": 200, + "headers": {"content-type": "text/html"}, + "url": "https://example.com/cdp-final", + }, + }, + } + } + ) + } + ] + + class _LogDriver: + def get_log(self, log_type: str) -> list[dict[str, str]]: + return perf_log if log_type == "performance" else [] + + status, headers, final_url = MetadataExtractor.extract_from_cdp_logs( + cast(DriverProtocol, _LogDriver()) + ) + self.assertEqual(status, 200) + self.assertEqual(headers, {"content-type": "text/html"}) + self.assertEqual(final_url, "https://example.com/cdp-final") + + def test_extract_falls_back_to_200_when_no_driver_metadata(self): + driver = type("EmptyDriver", (), {"current_url": "https://example.com/dest"})() + meta = MetadataExtractor.fetch( + cast(DriverProtocol, driver), "https://example.com" + ) + self.assertEqual(meta.status_code, 200) + self.assertEqual(meta.final_url, "https://example.com/dest") + self.assertIsNone(meta.headers) + self.assertIsNone(meta.metadata_error) diff --git a/tests/test_ops_telemetry.py b/tests/infra/test_ops_telemetry.py similarity index 85% rename from tests/test_ops_telemetry.py rename to tests/infra/test_ops_telemetry.py index 3a5e851..4643ed4 100644 --- a/tests/test_ops_telemetry.py +++ b/tests/infra/test_ops_telemetry.py @@ -4,19 +4,13 @@ import unittest from unittest.mock import MagicMock, patch -from app.ops_telemetry import ( +from app.infra.ops_telemetry import ( emit_terminal_telemetry, record_challenge_block, report_terminal_outcome, ) -from app.schemas import ( - ErrorCategory, - ExecutionTier, - NavigationMode, - ScrapeDiagnostics, - ScrapeError, - TimeoutPhase, -) +from app.schemas.enums import ErrorCategory, ExecutionTier, NavigationMode, TimeoutPhase +from app.schemas.response import ScrapeDiagnostics, ScrapeError def _scrape_error( @@ -46,7 +40,7 @@ class OpsTelemetryTests(unittest.TestCase): def test_report_terminal_outcome_noop_when_sentry_not_ready(self): result = _scrape_error(category=ErrorCategory.NAVIGATION_ERROR) with ( - patch("app.ops_telemetry.sentry_is_ready", return_value=False), + patch("app.infra.sentry.sentry_is_ready", return_value=False), patch("sentry_sdk.capture_message") as mock_capture, ): report_terminal_outcome(result, http_status=502) @@ -56,7 +50,7 @@ def test_report_terminal_outcome_emits_p0_navigation_error(self): result = _scrape_error(category=ErrorCategory.NAVIGATION_ERROR) mock_scope = MagicMock() with ( - patch("app.ops_telemetry.sentry_is_ready", return_value=True), + patch("app.infra.sentry.sentry_is_ready", return_value=True), patch("sentry_sdk.new_scope") as mock_new_scope, patch("sentry_sdk.capture_message") as mock_capture, ): @@ -82,7 +76,7 @@ def test_report_terminal_outcome_emits_p0_timeout(self): result = _scrape_error(category=ErrorCategory.TIMEOUT) mock_scope = MagicMock() with ( - patch("app.ops_telemetry.sentry_is_ready", return_value=True), + patch("app.infra.sentry.sentry_is_ready", return_value=True), patch("sentry_sdk.new_scope") as mock_new_scope, patch("sentry_sdk.capture_message") as mock_capture, ): @@ -101,7 +95,7 @@ def test_report_terminal_outcome_tags_timeout_phase(self): result.diagnostics.timeout_phase = TimeoutPhase.BOOT mock_scope = MagicMock() with ( - patch("app.ops_telemetry.sentry_is_ready", return_value=True), + patch("app.infra.sentry.sentry_is_ready", return_value=True), patch("sentry_sdk.new_scope") as mock_new_scope, patch("sentry_sdk.capture_message") as mock_capture, ): @@ -127,7 +121,7 @@ def test_report_terminal_outcome_tags_timeout_phase(self): def test_report_terminal_outcome_skips_challenge_block(self): result = _scrape_error(category=ErrorCategory.CHALLENGE_BLOCK) with ( - patch("app.ops_telemetry.sentry_is_ready", return_value=True), + patch("app.infra.sentry.sentry_is_ready", return_value=True), patch("sentry_sdk.capture_message") as mock_capture, ): report_terminal_outcome(result, http_status=502) @@ -140,7 +134,7 @@ def test_record_challenge_block_increments_metric_only(self): execution_tier=ExecutionTier.BROWSER_DRIVER, ) with ( - patch("app.ops_telemetry.sentry_is_ready", return_value=True), + patch("app.infra.sentry.sentry_is_ready", return_value=True), patch("sentry_sdk.metrics.count") as mock_count, patch("sentry_sdk.capture_message") as mock_capture, ): @@ -160,7 +154,7 @@ def test_record_challenge_block_increments_metric_only(self): def test_record_challenge_block_noop_when_sentry_not_ready(self): result = _scrape_error(category=ErrorCategory.CHALLENGE_BLOCK) with ( - patch("app.ops_telemetry.sentry_is_ready", return_value=False), + patch("app.infra.sentry.sentry_is_ready", return_value=False), patch("sentry_sdk.metrics.count") as mock_count, ): record_challenge_block(result) @@ -169,7 +163,7 @@ def test_record_challenge_block_noop_when_sentry_not_ready(self): def test_record_challenge_block_skips_non_challenge_categories(self): result = _scrape_error(category=ErrorCategory.NAVIGATION_ERROR) with ( - patch("app.ops_telemetry.sentry_is_ready", return_value=True), + patch("app.infra.sentry.sentry_is_ready", return_value=True), patch("sentry_sdk.metrics.count") as mock_count, ): record_challenge_block(result) @@ -178,8 +172,8 @@ def test_record_challenge_block_skips_non_challenge_categories(self): def test_emit_terminal_telemetry_routes_challenge_block_to_metric(self): result = _scrape_error(category=ErrorCategory.CHALLENGE_BLOCK) with ( - patch("app.ops_telemetry.record_challenge_block") as mock_metric, - patch("app.ops_telemetry.report_terminal_outcome") as mock_issue, + patch("app.infra.ops_telemetry.record_challenge_block") as mock_metric, + patch("app.infra.ops_telemetry.report_terminal_outcome") as mock_issue, ): emit_terminal_telemetry(result, http_status=502) mock_metric.assert_called_once_with(result) @@ -188,8 +182,8 @@ def test_emit_terminal_telemetry_routes_challenge_block_to_metric(self): def test_emit_terminal_telemetry_routes_p0_errors_to_issues(self): result = _scrape_error(category=ErrorCategory.NAVIGATION_ERROR) with ( - patch("app.ops_telemetry.record_challenge_block") as mock_metric, - patch("app.ops_telemetry.report_terminal_outcome") as mock_issue, + patch("app.infra.ops_telemetry.record_challenge_block") as mock_metric, + patch("app.infra.ops_telemetry.report_terminal_outcome") as mock_issue, ): emit_terminal_telemetry(result, http_status=502) mock_issue.assert_called_once_with(result, http_status=502) diff --git a/tests/test_request_id.py b/tests/infra/test_request_id.py similarity index 89% rename from tests/test_request_id.py rename to tests/infra/test_request_id.py index 9ba0fbe..48be769 100644 --- a/tests/test_request_id.py +++ b/tests/infra/test_request_id.py @@ -1,8 +1,9 @@ import unittest import uuid +from typing import cast from unittest.mock import patch -from app.request_id import resolve_request_id +from app.infra.request_id import resolve_request_id VALID_UUID = "550e8400-e29b-41d4-a716-446655440000" FALLBACK_UUID = "11111111-2222-4333-8444-555555555555" @@ -96,10 +97,11 @@ def test_resolve_request_id_table(self): for case in self.cases: with self.subTest(case=case["name"]): with patch( - "app.request_id.uuid.uuid4", return_value=uuid.UUID(FALLBACK_UUID) + "app.infra.request_id.uuid.uuid4", + return_value=uuid.UUID(FALLBACK_UUID), ): request_id, used_fallback = resolve_request_id( - case["inbound"], host="example.com" + cast("str | None", case["inbound"]), host="example.com" ) self.assertEqual(request_id, case["expected_id"]) @@ -107,7 +109,9 @@ def test_resolve_request_id_table(self): def test_fallback_logs_reason_without_rejected_value(self): with ( - patch("app.request_id.uuid.uuid4", return_value=uuid.UUID(FALLBACK_UUID)), + patch( + "app.infra.request_id.uuid.uuid4", return_value=uuid.UUID(FALLBACK_UUID) + ), self.assertLogs("botasaurus_scrape_api", level="INFO") as captured, ): resolve_request_id("bad/id", host="example.com") @@ -118,7 +122,9 @@ def test_fallback_logs_reason_without_rejected_value(self): def test_absent_fallback_logs_absent_reason(self): with ( - patch("app.request_id.uuid.uuid4", return_value=uuid.UUID(FALLBACK_UUID)), + patch( + "app.infra.request_id.uuid.uuid4", return_value=uuid.UUID(FALLBACK_UUID) + ), self.assertLogs("botasaurus_scrape_api", level="INFO") as captured, ): resolve_request_id(None, host="example.com") diff --git a/tests/infra/test_runtime_cleanup.py b/tests/infra/test_runtime_cleanup.py new file mode 100644 index 0000000..3ddafee --- /dev/null +++ b/tests/infra/test_runtime_cleanup.py @@ -0,0 +1,74 @@ +from __future__ import annotations + +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.config import get_settings, reset_settings_cache +from app.infra.runtime_cleanup import ( + prune_orphan_runtime_dirs, + runtime_root_low_on_space, +) + +MIN_FREE_BYTES = get_settings().scrape_runtime_min_free_bytes + + +class RuntimeCleanupTests(unittest.TestCase): + def setUp(self): + reset_settings_cache() + + def test_prune_orphan_runtime_dirs_removes_inactive_dirs(self): + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) + orphan = runtime_root / "orphan-req" + orphan.mkdir() + (orphan / "profile").mkdir() + active = runtime_root / "active-req" + active.mkdir() + + removed = prune_orphan_runtime_dirs(runtime_root, {"active-req"}) + + self.assertEqual(removed, 1) + self.assertFalse(orphan.exists()) + self.assertTrue(active.is_dir()) + + def test_prune_orphan_runtime_dirs_noop_when_root_missing(self): + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) / "missing" + self.assertEqual(prune_orphan_runtime_dirs(runtime_root, set()), 0) + + def test_runtime_root_low_on_space_uses_threshold(self): + with tempfile.TemporaryDirectory() as tmp: + runtime_root = Path(tmp) + low_usage = type( + "Usage", + (), + {"total": 10_000_000, "used": 1_000_000, "free": MIN_FREE_BYTES - 1}, + )() + with patch( + "app.infra.runtime_cleanup.shutil.disk_usage", return_value=low_usage + ): + self.assertTrue( + runtime_root_low_on_space( + runtime_root, min_free_bytes=MIN_FREE_BYTES + ) + ) + + ok_usage = type( + "Usage", + (), + {"total": 10_000_000, "used": 1_000_000, "free": MIN_FREE_BYTES + 1}, + )() + with patch( + "app.infra.runtime_cleanup.shutil.disk_usage", return_value=ok_usage + ): + self.assertFalse( + runtime_root_low_on_space( + runtime_root, min_free_bytes=MIN_FREE_BYTES + ) + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/infra/test_scrape_progress.py b/tests/infra/test_scrape_progress.py new file mode 100644 index 0000000..d600140 --- /dev/null +++ b/tests/infra/test_scrape_progress.py @@ -0,0 +1,35 @@ +"""ScrapeProgress snapshot unit tests.""" + +from __future__ import annotations + +import unittest + +from app.infra.scrape_progress import ScrapeProgress, ScrapeProgressSnapshot +from app.schemas.enums import ExecutionTier, NavigationMode, TimeoutPhase + + +class ScrapeProgressTests(unittest.TestCase): + def test_snapshot_defaults_to_queue(self): + snap: ScrapeProgressSnapshot = ScrapeProgress().snapshot() + self.assertEqual(snap.phase, TimeoutPhase.QUEUE) + self.assertEqual(snap.attempts, 0) + self.assertIsNone(snap.strategy_used) + self.assertIsNone(snap.execution_tier) + + def test_mark_updates_snapshot(self): + progress = ScrapeProgress() + progress.mark( + TimeoutPhase.WORK, + attempts=2, + strategy_used=NavigationMode.GET, + execution_tier=ExecutionTier.BROWSER_DRIVER, + ) + snap = progress.snapshot() + self.assertEqual(snap.phase, TimeoutPhase.WORK) + self.assertEqual(snap.attempts, 2) + self.assertEqual(snap.strategy_used, NavigationMode.GET) + self.assertEqual(snap.execution_tier, ExecutionTier.BROWSER_DRIVER) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_sentry.py b/tests/infra/test_sentry.py similarity index 56% rename from tests/test_sentry.py rename to tests/infra/test_sentry.py index 131e5ea..780ff61 100644 --- a/tests/test_sentry.py +++ b/tests/infra/test_sentry.py @@ -1,88 +1,80 @@ -# tests/test_sentry.py +# pyright: reportPrivateUsage=false +"""Sentry setup, readiness, and event filtering tests (exercise module privates).""" + from __future__ import annotations import os import unittest +from contextlib import contextmanager from unittest.mock import patch -import app.sentry as sentry_mod -from app.sentry import ( - _before_send, - flush_sentry, - is_sentry_enabled, - sentry_is_ready, - setup_sentry, -) - - -class SentryIntegrationTests(unittest.TestCase): - def setUp(self): - sentry_mod._INITIALIZED = False - - def test_is_sentry_enabled(self): - with patch.dict(os.environ, {}, clear=True): - self.assertFalse(is_sentry_enabled()) +import app.infra.sentry as sentry_mod +from app.config import reset_settings_cache - with patch.dict(os.environ, {"SENTRY_DSN": ""}): - self.assertFalse(is_sentry_enabled()) - with patch.dict(os.environ, {"SENTRY_DSN": " "}): - self.assertFalse(is_sentry_enabled()) +@contextmanager +def env(**values: str): + with patch.dict(os.environ, values, clear=True): + reset_settings_cache() + yield - with patch.dict(os.environ, {"SENTRY_DSN": "https://key@sentry.io/123"}): - self.assertTrue(is_sentry_enabled()) - def test_sentry_is_ready_requires_init(self): - with patch.dict(os.environ, {}, clear=True): - self.assertFalse(sentry_is_ready()) +class SentryIntegrationTests(unittest.TestCase): + def setUp(self): + sentry_mod._initialized = False + reset_settings_cache() - with patch.dict(os.environ, {"SENTRY_DSN": "https://key@sentry.io/123"}): - self.assertFalse(sentry_is_ready()) + def test_sentry_is_ready_requires_successful_init(self): + self.assertFalse(sentry_mod.sentry_is_ready()) - sentry_mod._INITIALIZED = True - with patch.dict(os.environ, {"SENTRY_DSN": "https://key@sentry.io/123"}): - self.assertTrue(sentry_is_ready()) + with ( + env(SENTRY_DSN="https://key@sentry.io/123"), + patch("sentry_sdk.init"), + ): + self.assertFalse(sentry_mod.sentry_is_ready()) + self.assertTrue(sentry_mod.setup_sentry()) + self.assertTrue(sentry_mod.sentry_is_ready()) def test_setup_sentry_noop_when_dsn_absent(self): with ( - patch.dict(os.environ, {}, clear=True), + env(), patch("sentry_sdk.init") as mock_init, ): - result = setup_sentry() + result = sentry_mod.setup_sentry() self.assertFalse(result) mock_init.assert_not_called() - self.assertFalse(sentry_mod._INITIALIZED) + self.assertFalse(sentry_mod._initialized) def test_setup_sentry_noop_when_dsn_empty_or_whitespace(self): for val in ("", " ", "\t\n"): with ( self.subTest(val=repr(val)), - patch.dict(os.environ, {"SENTRY_DSN": val}), + env(SENTRY_DSN=val), patch("sentry_sdk.init") as mock_init, ): - result = setup_sentry() + result = sentry_mod.setup_sentry() self.assertFalse(result) mock_init.assert_not_called() - self.assertFalse(sentry_mod._INITIALIZED) + self.assertFalse(sentry_mod._initialized) def test_setup_sentry_initializes_with_defaults(self): dsn = "https://key@o123.ingest.sentry.io/456" with ( - patch.dict(os.environ, {"SENTRY_DSN": dsn}, clear=True), + env(SENTRY_DSN=dsn), patch("sentry_sdk.init") as mock_init, self.assertLogs("botasaurus_scrape_api", level="INFO") as captured, ): - result = setup_sentry() + result = sentry_mod.setup_sentry() self.assertTrue(result) - self.assertTrue(sentry_mod._INITIALIZED) + self.assertTrue(sentry_mod._initialized) mock_init.assert_called_once() init_kwargs = mock_init.call_args.kwargs self.assertEqual(init_kwargs["dsn"], dsn) self.assertEqual(init_kwargs["environment"], "production") self.assertEqual(init_kwargs["traces_sample_rate"], 0.0) self.assertFalse(init_kwargs["send_default_pii"]) - self.assertIs(init_kwargs["before_send"], _before_send) + self.assertIs(init_kwargs["before_send"], sentry_mod._before_send) integration_names = [ type(integration).__name__ for integration in init_kwargs["integrations"] @@ -97,19 +89,18 @@ def test_setup_sentry_initializes_with_defaults(self): def test_setup_sentry_reads_custom_env_options(self): dsn = "https://key@o123.ingest.sentry.io/456" - env_vars = { - "SENTRY_DSN": dsn, - "SENTRY_ENVIRONMENT": "staging", - "SENTRY_RELEASE": "v2.0.0+abc1234", - "SENTRY_TRACES_SAMPLE_RATE": "0.25", - "SENTRY_PROFILES_SAMPLE_RATE": "0.10", - "SENTRY_SEND_DEFAULT_PII": "true", - } with ( - patch.dict(os.environ, env_vars, clear=True), + env( + SENTRY_DSN=dsn, + SENTRY_ENVIRONMENT="staging", + SENTRY_RELEASE="v2.0.0+abc1234", + SENTRY_TRACES_SAMPLE_RATE="0.25", + SENTRY_PROFILES_SAMPLE_RATE="0.10", + SENTRY_SEND_DEFAULT_PII="true", + ), patch("sentry_sdk.init") as mock_init, ): - result = setup_sentry() + result = sentry_mod.setup_sentry() self.assertTrue(result) mock_init.assert_called_once() @@ -120,17 +111,22 @@ def test_setup_sentry_reads_custom_env_options(self): self.assertEqual(init_kwargs["traces_sample_rate"], 0.25) self.assertEqual(init_kwargs["profiles_sample_rate"], 0.10) self.assertTrue(init_kwargs["send_default_pii"]) - self.assertIs(init_kwargs["before_send"], _before_send) + self.assertIs(init_kwargs["before_send"], sentry_mod._before_send) def test_before_send_drops_challenge_block_issues(self): - dropped = _before_send({"tags": {"error_category": "challenge_block"}}, {}) + dropped = sentry_mod._before_send( + {"tags": {"error_category": "challenge_block"}}, {} + ) self.assertIsNone(dropped) - kept = _before_send({"tags": {"error_category": "navigation_error"}}, {}) - self.assertEqual(kept["tags"]["error_category"], "navigation_error") + kept = sentry_mod._before_send( + {"tags": {"error_category": "navigation_error"}}, {} + ) + assert kept is not None + self.assertEqual(kept.get("tags", {}).get("error_category"), "navigation_error") def test_before_send_drops_websocket_teardown(self): - dropped = _before_send( + dropped = sentry_mod._before_send( { "logger": "websocket", "logentry": {"formatted": "Connection to remote host was lost"}, @@ -139,37 +135,27 @@ def test_before_send_drops_websocket_teardown(self): ) self.assertIsNone(dropped) - kept = _before_send({"logger": "botasaurus_scrape_api"}, {}) + kept = sentry_mod._before_send({"logger": "botasaurus_scrape_api"}, {}) self.assertIsNotNone(kept) def test_setup_sentry_clamps_sample_rates_and_handles_invalid_floats(self): dsn = "https://key@o123.ingest.sentry.io/456" - # Invalid string falls back to 0.0 with ( - patch.dict( - os.environ, - {"SENTRY_DSN": dsn, "SENTRY_TRACES_SAMPLE_RATE": "invalid_float"}, - clear=True, - ), + env(SENTRY_DSN=dsn, SENTRY_TRACES_SAMPLE_RATE="invalid_float"), patch("sentry_sdk.init") as mock_init, ): - result = setup_sentry() + result = sentry_mod.setup_sentry() self.assertTrue(result) mock_init.assert_called_once() init_kwargs = mock_init.call_args.kwargs self.assertEqual(init_kwargs["traces_sample_rate"], 0.0) - # > 1.0 is clamped to 1.0 - sentry_mod._INITIALIZED = False + sentry_mod._initialized = False with ( - patch.dict( - os.environ, - {"SENTRY_DSN": dsn, "SENTRY_TRACES_SAMPLE_RATE": "2.5"}, - clear=True, - ), + env(SENTRY_DSN=dsn, SENTRY_TRACES_SAMPLE_RATE="2.5"), patch("sentry_sdk.init") as mock_init, ): - result = setup_sentry() + result = sentry_mod.setup_sentry() self.assertTrue(result) mock_init.assert_called_once() init_kwargs = mock_init.call_args.kwargs @@ -177,20 +163,19 @@ def test_setup_sentry_clamps_sample_rates_and_handles_invalid_floats(self): def test_flush_sentry_noop_when_not_initialized(self): with patch("sentry_sdk.flush") as mock_flush: - flush_sentry() + sentry_mod.flush_sentry() mock_flush.assert_not_called() def test_flush_sentry_invokes_sdk_flush_when_initialized(self): - sentry_mod._INITIALIZED = True + sentry_mod._initialized = True with patch("sentry_sdk.flush") as mock_flush: - flush_sentry(timeout=3.0) + sentry_mod.flush_sentry(timeout=3.0) mock_flush.assert_called_once_with(timeout=3.0) def test_flush_sentry_swallows_exceptions_cleanly(self): - sentry_mod._INITIALIZED = True + sentry_mod._initialized = True with patch("sentry_sdk.flush", side_effect=RuntimeError("flush timeout")): - # Should not raise - flush_sentry() + sentry_mod.flush_sentry() if __name__ == "__main__": diff --git a/tests/infra/test_xhr_collector.py b/tests/infra/test_xhr_collector.py new file mode 100644 index 0000000..b60682f --- /dev/null +++ b/tests/infra/test_xhr_collector.py @@ -0,0 +1,163 @@ +# pyright: reportMissingParameterType=false, reportUnknownParameterType=false, reportUnknownLambdaType=false, reportPrivateUsage=false, reportAttributeAccessIssue=false, reportFunctionMemberAccess=false, reportUnknownMemberType=false, reportUnknownArgumentType=false, reportUnknownVariableType=false, reportOptionalSubscript=false, reportOptionalMemberAccess=false +import unittest + +from tests.support.fakes import ( + FakeNetworkResponse, + FakeRequestId, + FakeTab, +) + + +class XhrCollectorTests(unittest.TestCase): + def setUp(self): + from app.infra.xhr_collector import XhrCollector + + self.XhrCollector = XhrCollector + self.target = "https://example.com/" + self.collector = XhrCollector(self.target) + + def _drive_json(self, tab, request_id, url, body, mime="application/json"): + rid = FakeRequestId(request_id) + tab.bodies[str(rid)] = (body, False) + self.collector._on_response( + rid, + FakeNetworkResponse(url, 200, mime, {"content-type": mime}), + None, + ) + self.collector._on_finished(type("E", (), {"request_id": rid})()) + + def test_install_enables_network_and_registers_handlers(self): + tab = FakeTab() + self.collector.install(tab) + self.assertTrue(tab.network_enabled) + self.assertEqual( + tab.response_handler.__func__, self.collector._on_response.__func__ + ) + self.assertIs(tab.response_handler.__self__, self.collector) + self.assertEqual( + tab.finished_handler.__func__, self.collector._on_finished.__func__ + ) + self.assertIs(tab.finished_handler.__self__, self.collector) + + def test_captures_json_subresource(self): + tab = FakeTab() + self.collector.install(tab) + self._drive_json( + tab, "1", "https://api.example.com/feed", '{"items":[{"title":"A"}]}' + ) + results = self.collector.harvest(tab) + self.assertEqual(len(results), 1) + self.assertEqual(results[0].url, "https://api.example.com/feed") + self.assertEqual(results[0].status_code, 200) + self.assertEqual(results[0].headers, {"content-type": "application/json"}) + self.assertIn("items", results[0].body) + + def test_skips_non_json_mime(self): + tab = FakeTab() + self.collector.install(tab) + self._drive_json( + tab, + "1", + "https://cdn.example.com/app.js", + "console.log(1)", + mime="application/javascript", + ) + self.assertEqual(self.collector.harvest(tab), []) + + def test_skips_main_document(self): + tab = FakeTab() + self.collector.install(tab) + rid = FakeRequestId("doc") + tab.bodies[str(rid)] = ('{"nope":true}', False) + self.collector._on_response( + rid, + FakeNetworkResponse(self.target, 200, "application/json"), + None, + ) + self.collector._on_finished(type("E", (), {"request_id": rid})()) + self.assertEqual(self.collector.harvest(tab), []) + + def test_skips_empty_body(self): + tab = FakeTab() + self.collector.install(tab) + self._drive_json(tab, "1", "https://api.example.com/empty", "") + self.assertEqual(self.collector.harvest(tab), []) + + def test_enforces_max_responses_cap(self): + tab = FakeTab() + self.collector.install(tab) + for i in range(self.XhrCollector.MAX_RESPONSES + 5): + self._drive_json( + tab, str(i), f"https://api.example.com/i/{i}", f'{{"i":{i}}}' + ) + results = self.collector.harvest(tab) + self.assertEqual(len(results), self.XhrCollector.MAX_RESPONSES) + + def test_enforces_max_body_bytes_cap(self): + tab = FakeTab() + self.collector.install(tab) + oversized = "x" * (self.XhrCollector.MAX_BODY_BYTES + 1) + self._drive_json(tab, "1", "https://api.example.com/big", oversized) + self.assertEqual(self.collector.harvest(tab), []) + + def test_enforces_aggregate_bytes_cap(self): + tab = FakeTab() + self.collector.install(tab) + # Five near-max bodies would exceed 2 MB aggregate; stop once budget trips. + chunk = "y" * self.XhrCollector.MAX_BODY_BYTES + for i in range(5): + self._drive_json(tab, str(i), f"https://api.example.com/chunk/{i}", chunk) + results = self.collector.harvest(tab) + total = sum(len(entry.body.encode("utf-8")) for entry in results) + self.assertLessEqual(total, self.XhrCollector.MAX_AGGREGATE_BYTES) + self.assertEqual(len(results), 4) + self.assertLess(len(results), 5) + + def test_headers_allowlist_keeps_only_content_type(self): + tab = FakeTab() + self.collector.install(tab) + rid = FakeRequestId("hdr") + tab.bodies[str(rid)] = ('{"ok":true}', False) + self.collector._on_response( + rid, + FakeNetworkResponse( + "https://api.example.com/secure", + 200, + "application/json", + { + "Content-Type": "application/json; charset=utf-8", + "Set-Cookie": "session=secret", + "X-Request-Id": "abc", + }, + ), + None, + ) + self.collector._on_finished(type("E", (), {"request_id": rid})()) + results = self.collector.harvest(tab) + self.assertEqual(len(results), 1) + self.assertEqual( + results[0].headers, + {"content-type": "application/json; charset=utf-8"}, + ) + self.assertNotIn("Set-Cookie", results[0].headers) + self.assertNotIn("set-cookie", results[0].headers) + + def test_reset_clears_pending_ready_and_collected(self): + tab = FakeTab() + self.collector.install(tab) + self._drive_json( + tab, "1", "https://api.example.com/old", '{"from":"failed-attempt"}' + ) + first = self.collector.harvest(tab) + self.assertEqual(len(first), 1) + + self.collector.reset() + self.assertEqual(self.collector.results(), []) + + self._drive_json( + tab, "2", "https://api.example.com/new", '{"from":"success-attempt"}' + ) + second = self.collector.harvest(tab) + self.assertEqual(len(second), 1) + self.assertEqual(second[0].body, '{"from":"success-attempt"}') + self.assertNotIn("failed-attempt", second[0].body) diff --git a/tests/security/__init__.py b/tests/security/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/security/test_url_guard.py b/tests/security/test_url_guard.py new file mode 100644 index 0000000..db4ef98 --- /dev/null +++ b/tests/security/test_url_guard.py @@ -0,0 +1,40 @@ +import ipaddress +import unittest + +from app.security import UrlGuard + + +class UrlGuardUnitTests(unittest.TestCase): + def test_validate_rejects_non_http_schemes(self): + res = UrlGuard.validate("ftp://example.com/file") + self.assertFalse(res.is_allowed) + self.assertEqual(res.status_code, 400) + error_message = res.error_message + assert error_message is not None + self.assertIn("Only http/https", error_message) + + def test_validate_rejects_missing_hostname(self): + res = UrlGuard.validate("http://") + self.assertFalse(res.is_allowed) + self.assertEqual(res.status_code, 400) + + def test_validate_rejects_localhost(self): + res = UrlGuard.validate("http://localhost:8080/test") + self.assertFalse(res.is_allowed) + self.assertEqual(res.status_code, 403) + + def test_validate_proxy_rejects_blocked_host(self): + res = UrlGuard.validate_proxy("http://127.0.0.1:8080") + self.assertFalse(res.is_allowed) + self.assertEqual(res.status_code, 403) + error_message = res.error_message + assert error_message is not None + self.assertIn("Proxy URL is invalid or blocked", error_message) + + def test_is_blocked_ip_allows_well_known_nat64_prefix(self): + nat64_ip = ipaddress.ip_address("64:ff9b::3691:8e03") + self.assertFalse(UrlGuard.is_blocked_ip(nat64_ip)) + + def test_is_blocked_ip_still_blocks_loopback(self): + loopback = ipaddress.ip_address("127.0.0.1") + self.assertTrue(UrlGuard.is_blocked_ip(loopback)) diff --git a/tests/support/__init__.py b/tests/support/__init__.py new file mode 100644 index 0000000..293feb7 --- /dev/null +++ b/tests/support/__init__.py @@ -0,0 +1 @@ +"""Shared test utilities.""" diff --git a/tests/support/factories.py b/tests/support/factories.py new file mode 100644 index 0000000..de67e51 --- /dev/null +++ b/tests/support/factories.py @@ -0,0 +1,20 @@ +"""Validated test constructors for scrape request wire types.""" + +from __future__ import annotations + +from typing import Any, cast + +from pydantic import HttpUrl + +from app.schemas.request import ScrapeRequest + +EXAMPLE_URL = "https://example.com" + + +def example_url() -> HttpUrl: + return cast(HttpUrl, EXAMPLE_URL) + + +def scrape_request(**kwargs: Any) -> ScrapeRequest: + payload = {"url": EXAMPLE_URL, **kwargs} + return ScrapeRequest.model_validate(payload) diff --git a/tests/support/fakes.py b/tests/support/fakes.py new file mode 100644 index 0000000..47b95d8 --- /dev/null +++ b/tests/support/fakes.py @@ -0,0 +1,204 @@ +"""Shared test fakes for scrape API layers.""" + +from __future__ import annotations + +from collections.abc import Callable +from pathlib import Path +from typing import Any, ClassVar + + +class FakeMetadataResponse: + status_code = 200 + headers: ClassVar[dict[str, str]] = {"content-type": "text/html"} + url = "https://example.com/" + + +class FakeRequests: + def get(self, _url: str) -> FakeMetadataResponse: + return FakeMetadataResponse() + + +class FakeDriverTab: + def block_urls(self, patterns: list[str]) -> None: + del patterns + + def set_extra_http_headers(self, headers: dict[str, str]) -> None: + del headers + + +class FakeDriver: + page_html: str | None + current_url: str | None + requests: list[object] + + def __init__(self, *args: object, **kwargs: Any) -> None: + del args + self.page_html = "

Example Domain

" + self.current_url = "https://example.com/" + self.requests = [FakeRequests()] + self._raise_wait = kwargs.pop("raise_wait", False) + self.scrolled = False + self._tab = FakeDriverTab() + + def get(self, *_args: object, **_kwargs: Any) -> None: + return None + + def google_get(self, *_args: object, **_kwargs: Any) -> None: + return None + + def organic_get(self, *_args: object, **_kwargs: Any) -> None: + return None + + def wait_for_element(self, *_args: object, **_kwargs: Any) -> None: + raise RuntimeError("missing selector") + + def scroll_to_bottom(self) -> None: + self.scrolled = True + + def scroll(self) -> None: + return None + + def sleep(self, *_args: object, **_kwargs: Any) -> None: + return None + + def sleep_random(self, *_args: object, **_kwargs: Any) -> None: + return None + + def run_js(self, _script: str) -> None: + return None + + def execute_script(self, _script: str) -> None: + return None + + def add_cookies(self, _cookies: list[dict[str, str]]) -> None: + return None + + def bypass_cloudflare(self) -> None: + return None + + def save_screenshot(self, filename: str) -> None: + Path(filename).write_bytes(b"fake") + + def close(self) -> None: + return None + + def get_log(self, _log_type: str) -> list[dict[str, str]]: + return [] + + +class CaptureDriver(FakeDriver): + last_init_kwargs: ClassVar[dict[str, Any] | None] = None + + def __init__(self, *args: object, **kwargs: Any) -> None: + type(self).last_init_kwargs = dict(kwargs) + super().__init__(*args, **kwargs) + + +class FakeHttpResponse: + def __init__( + self, + *, + text: str, + status_code: int, + headers: dict[str, str], + url: str, + ) -> None: + self.text = text + self.status_code = status_code + self.headers = headers + self.url = url + + +class FakeRequest: + response: FakeHttpResponse | None = None + + def get(self, *_args: object, **_kwargs: Any) -> FakeHttpResponse | None: + return type(self).response + + def close(self) -> None: + return None + + +def fake_request_cls( + *, + html: str = "ok", + url: str = "https://example.com/", + status_code: int = 200, + headers: dict[str, str] | None = None, +) -> type[FakeRequest]: + """Build a patchable botasaurus Request substitute returning one canned response.""" + response = FakeHttpResponse( + text=html, + status_code=status_code, + headers=headers if headers is not None else {"content-type": "text/html"}, + url=url, + ) + return type("CannedFakeRequest", (FakeRequest,), {"response": response}) + + +class ArticleDriver(FakeDriver): + ARTICLE_HTML = ( + "

Headline

" + "

Lead paragraph

" + ) + + def __init__(self, *args: object, **kwargs: Any) -> None: + super().__init__(*args, **kwargs) + self.page_html = self.ARTICLE_HTML + + +class FakeRequestId(str): + def to_json(self) -> str: + return str(self) + + +class FakeNetworkResponse: + def __init__( + self, + url: str, + status: int, + mime_type: str, + headers: dict[str, str] | None = None, + ) -> None: + self.url = url + self.status = status + self.mime_type = mime_type + self.headers = headers or {} + + +class FakeTab: + """Minimal CDP tab stub for XhrCollector unit tests.""" + + def __init__(self, bodies: dict[str, tuple[str, bool]] | None = None) -> None: + self.bodies = bodies or {} + self.network_enabled = False + self.response_handler: Callable[..., None] | None = None + self.finished_handler: Callable[..., None] | None = None + + def send(self, cdp_obj: Any) -> Any: + cmd = next(cdp_obj) + method = cmd.get("method") + if method == "Network.enable": + self.network_enabled = True + try: + cdp_obj.send({}) + except StopIteration as exc: + return exc.value + return None + if method == "Network.getResponseBody": + rid = str(cmd["params"]["requestId"]) + body, b64 = self.bodies.get(rid, ("", False)) + try: + cdp_obj.send({"body": body, "base64Encoded": b64}) + except StopIteration as exc: + return exc.value + return None + raise AssertionError(f"unexpected CDP method: {method}") + + def after_response_received(self, handler: Callable[..., None]) -> None: + self.response_handler = handler + + def add_handler( + self, _event_type: type[object], handler: Callable[..., None] + ) -> None: + self.finished_handler = handler diff --git a/tests/support/http.py b/tests/support/http.py new file mode 100644 index 0000000..2ecd79d --- /dev/null +++ b/tests/support/http.py @@ -0,0 +1,72 @@ +"""Shared HTTP test helpers.""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Protocol, cast + +from fastapi import FastAPI, Request +from fastapi.testclient import TestClient + +from app.api.deps import get_engine +from app.engine import ScraperEngine +from app.infra.scrape_progress import ScrapeProgress +from app.main import create_app +from app.schemas.request import ScrapeRequest +from app.schemas.response import ScrapeError, ScrapeSuccess + + +class ExecuteSideEffect(Protocol): + def __call__( + self, + payload: ScrapeRequest, + deadline_monotonic: float | None = ..., + *, + request_id: str | None = ..., + progress: ScrapeProgress | None = ..., + ) -> ScrapeSuccess | ScrapeError: ... + + +class _EngineExecuteProxy: + def __init__( + self, + engine: ScraperEngine, + execute: ExecuteSideEffect, + ) -> None: + self._engine = engine + self.execute = execute + + def __getattr__(self, name: str) -> object: + return getattr(self._engine, name) + + +@contextmanager +def test_client( + *, + engine: ScraperEngine | None = None, + execute_side_effect: ExecuteSideEffect | None = None, +) -> Iterator[TestClient]: + app: FastAPI = create_app() + bound_engine: ScraperEngine | _EngineExecuteProxy | None = engine + + if bound_engine is not None: + + def _override_engine(_request: Request) -> ScraperEngine: + return cast(ScraperEngine, bound_engine) + + app.dependency_overrides[get_engine] = _override_engine + + with TestClient(app) as client: + if bound_engine is None: + bound_engine = client.app.state.engine + if execute_side_effect is not None: + proxy_base = cast(ScraperEngine, bound_engine) + bound_engine = _EngineExecuteProxy(proxy_base, execute_side_effect) + + def _override_engine_with_proxy(_request: Request) -> ScraperEngine: + return cast(ScraperEngine, bound_engine) + + app.dependency_overrides[get_engine] = _override_engine_with_proxy + yield client + app.dependency_overrides.clear() diff --git a/tests/test_api_contract.py b/tests/test_api_contract.py deleted file mode 100644 index 1995af8..0000000 --- a/tests/test_api_contract.py +++ /dev/null @@ -1,1126 +0,0 @@ -import ipaddress -import tempfile -import unittest -from pathlib import Path -from typing import ClassVar -from unittest.mock import MagicMock, patch - -from pydantic import ValidationError - -from app.detector import ChallengeDetector -from app.engine import ( - ScraperEngine, - html_document_headers, - utf8_normalize_html, -) -from app.metadata import MetadataExtractor -from app.schemas import ( - DEFAULT_SCRAPE_TIMEOUT_SECONDS, - DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS, - ErrorCategory, - ExecutionMode, - ExecutionTier, - NavigationMode, - ScrapeDiagnostics, - ScrapeError, - ScrapeRequest, - ScrapeSuccess, - WindowSize, -) -from app.security import UrlGuard - - -class _FakeMetadataResponse: - status_code = 200 - headers: ClassVar[dict[str, str]] = {"content-type": "text/html"} - url = "https://example.com/" - - -class _FakeRequests: - def get(self, _url): - return _FakeMetadataResponse() - - -class _FakeDriver: - def __init__(self, *args, **kwargs): - self.page_html = "

Example Domain

" - self.current_url = "https://example.com/" - self.requests = _FakeRequests() - self._raise_wait = kwargs.pop("raise_wait", False) - self.scrolled = False - - def get(self, *_args, **_kwargs): - return None - - def google_get(self, *_args, **_kwargs): - return None - - def organic_get(self, *_args, **_kwargs): - return None - - def wait_for_element(self, *_args, **_kwargs): - raise RuntimeError("missing selector") - - def scroll_to_bottom(self): - self.scrolled = True - - def sleep(self, *_args, **_kwargs): - return None - - def save_screenshot(self, filename): - Path(filename).write_bytes(b"fake") - - def close(self): - return None - - -class _CaptureDriver(_FakeDriver): - last_init_kwargs = None - - def __init__(self, *args, **kwargs): - type(self).last_init_kwargs = dict(kwargs) - super().__init__(*args, **kwargs) - - -class _FakeHttpResponse: - def __init__(self, *, text, status_code, headers, url): - self.text = text - self.status_code = status_code - self.headers = headers - self.url = url - - -class _FakeRequest: - response = None - - def get(self, *_args, **_kwargs): - return type(self).response - - def close(self): - return None - - -class _ArticleDriver(_FakeDriver): - ARTICLE_HTML = ( - "

Headline

" - "

Lead paragraph

" - ) - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.page_html = self.ARTICLE_HTML - - -class MainUnitTests(unittest.TestCase): - def test_request_defaults(self): - payload = ScrapeRequest(url="https://example.com") - self.assertEqual(payload.execution_mode, ExecutionMode.AUTO) - self.assertEqual(payload.navigation_mode, NavigationMode.AUTO) - self.assertEqual(payload.max_retries, 2) - self.assertEqual(payload.wait_timeout_seconds, 15) - self.assertFalse(payload.scroll) - self.assertTrue(payload.block_images) - self.assertFalse(payload.block_images_and_css) - self.assertTrue(payload.block_trackers) - self.assertTrue(payload.wait_for_complete_page_load) - self.assertIsNone(payload.user_agent) - self.assertIsNone(payload.headers) - self.assertIsNone(payload.cookies) - self.assertIsNone(payload.window_size) - self.assertIsNone(payload.lang) - self.assertFalse(payload.headless) - self.assertIsNone(payload.proxy) - - def test_scroll_parameters(self): - req_scroll = ScrapeRequest(url="https://example.com", scroll=True) - self.assertTrue(req_scroll.scroll) - self.assertFalse(ScrapeRequest(url="https://example.com").scroll) - - def test_window_size_validation_requires_object(self): - with self.assertRaises(ValidationError): - ScrapeRequest(url="https://example.com", window_size=[1920, 1080]) - with self.assertRaises(ValidationError): - ScrapeRequest(url="https://example.com", window_size={"width": 1920}) - - def test_wait_timeout_seconds_clamps_above_work_cap(self): - with self.assertLogs("botasaurus_scrape_api", level="INFO") as captured: - payload = ScrapeRequest(url="https://example.com", wait_timeout_seconds=35) - - self.assertEqual( - payload.wait_timeout_seconds, DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS - ) - self.assertEqual(DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS, 30) - self.assertEqual(DEFAULT_SCRAPE_TIMEOUT_SECONDS, 45) - log_text = "\n".join(captured.output) - self.assertIn("host=example.com", log_text) - self.assertIn("field=wait_timeout_seconds", log_text) - self.assertIn("from=35", log_text) - self.assertIn("to=30", log_text) - - def test_wait_timeout_seconds_clamps_below_one(self): - with self.assertLogs("botasaurus_scrape_api", level="INFO") as captured: - payload = ScrapeRequest(url="https://example.com", wait_timeout_seconds=0) - - self.assertEqual(payload.wait_timeout_seconds, 1) - log_text = "\n".join(captured.output) - self.assertIn("field=wait_timeout_seconds", log_text) - self.assertIn("from=0", log_text) - self.assertIn("to=1", log_text) - - def test_clamped_wait_timeout_allows_execute(self): - payload = ScrapeRequest( - url="https://example.com", - execution_mode="browser", - navigation_mode="get", - max_retries=0, - wait_timeout_seconds=35, - ) - self.assertEqual( - payload.wait_timeout_seconds, DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS - ) - - with tempfile.TemporaryDirectory() as tmp: - engine = ScraperEngine(runtime_root=Path(tmp)) - with patch("app.engine.Driver", _FakeDriver): - result = engine.execute(payload) - - self.assertIsNone(result.error if isinstance(result, ScrapeError) else None) - self.assertIsInstance(result, ScrapeSuccess) - self.assertEqual( - result.html, "

Example Domain

" - ) - - def test_html_response_sets_utf8_content_type_and_normalizes_body(self): - _FakeRequest.response = _FakeHttpResponse( - text="

Caffè

", - status_code=200, - headers={"content-type": "application/octet-stream"}, - url="https://example.com/", - ) - payload = ScrapeRequest( - url="https://example.com", - execution_mode="request", - ) - - with tempfile.TemporaryDirectory() as tmp: - engine = ScraperEngine(runtime_root=Path(tmp)) - with patch("app.engine.Request", _FakeRequest): - result = engine.execute(payload) - - self.assertIsInstance(result, ScrapeSuccess) - self.assertEqual(result.diagnostics.execution_tier, ExecutionTier.HTTP_REQUEST) - self.assertIsNotNone(result.headers) - self.assertEqual(result.headers["content-type"], "text/html; charset=utf-8") - self.assertNotIn("application/octet-stream", result.headers.values()) - self.assertIn("Caffè", result.html) - self.assertNotIn("Caffè", result.html) - result.html.encode("utf-8") - - def test_utf8_normalize_leaves_correct_unicode_unchanged(self): - html = "

Caffè 日本語

" - self.assertEqual(utf8_normalize_html(html), html) - - normalized, headers = html_document_headers(html, {"content-type": "text/html"}) - self.assertEqual(normalized, html) - self.assertEqual(headers["content-type"], "text/html; charset=utf-8") - - _FakeRequest.response = _FakeHttpResponse( - text=html, - status_code=200, - headers={"content-type": "text/html"}, - url="https://example.com/", - ) - payload = ScrapeRequest(url="https://example.com", execution_mode="request") - with tempfile.TemporaryDirectory() as tmp: - engine = ScraperEngine(runtime_root=Path(tmp)) - with patch("app.engine.Request", _FakeRequest): - result = engine.execute(payload) - - self.assertEqual(result.html, html) - self.assertEqual(result.headers["content-type"], "text/html; charset=utf-8") - - def test_request_tier_blocked_status_escalates_to_browser(self): - payload = ScrapeRequest(url="https://example.com") - for status in (401, 403, 429): - with self.subTest(status=status): - _FakeRequest.response = _FakeHttpResponse( - text="Forbidden", - status_code=status, - headers={"content-type": "text/html"}, - url="https://example.com/", - ) - with tempfile.TemporaryDirectory() as tmp: - engine = ScraperEngine(runtime_root=Path(tmp)) - with ( - patch("app.engine.Request", _FakeRequest), - patch("app.engine.Driver", _ArticleDriver), - ): - result = engine.execute(payload) - - self.assertIsInstance(result, ScrapeSuccess) - self.assertEqual( - result.diagnostics.execution_tier, ExecutionTier.BROWSER_DRIVER - ) - self.assertIn("
", result.html) - self.assertIn("Headline", result.html) - self.assertEqual( - result.headers["content-type"], "text/html; charset=utf-8" - ) - - def test_strategy_selection(self): - self.assertEqual(ScraperEngine.resolve_strategies("auto", 0), ["google_get"]) - self.assertEqual( - ScraperEngine.resolve_strategies("auto", 2), - ["google_get", "google_get_bypass", "get"], - ) - self.assertEqual( - ScraperEngine.resolve_strategies("get", 2), ["get", "get", "get"] - ) - self.assertEqual( - ScraperEngine.resolve_strategies("organic_get", 2), - ["organic_get", "organic_get", "organic_get"], - ) - - def test_cleanup_runs_on_navigation_error(self): - payload = ScrapeRequest( - url="https://example.com", - execution_mode="browser", - navigation_mode="get", - max_retries=0, - wait_for_selector="#missing", - wait_timeout_seconds=1, - ) - - with tempfile.TemporaryDirectory() as tmp: - runtime_root = Path(tmp) - engine = ScraperEngine(runtime_root=runtime_root) - with patch("app.engine.Driver", _FakeDriver): - result = engine.execute(payload) - - self.assertEqual(result.error_category, ErrorCategory.NAVIGATION_ERROR) - self.assertEqual(list(runtime_root.iterdir()), []) - - def test_run_scrape_forwards_driver_kwargs(self): - _CaptureDriver.last_init_kwargs = None - payload = ScrapeRequest( - url="https://example.com", - execution_mode="browser", - block_images=True, - block_images_and_css=True, - wait_for_complete_page_load=False, - user_agent="MyAgent/1.0", - window_size=WindowSize(width=1920, height=1080), - lang="en-US", - headless=True, - proxy="http://proxy.example:8080", - ) - - with tempfile.TemporaryDirectory() as tmp: - runtime_root = Path(tmp) - engine = ScraperEngine(runtime_root=runtime_root) - with patch("app.engine.Driver", _CaptureDriver): - result = engine.execute(payload) - - self.assertIsInstance(result, ScrapeSuccess) - self.assertIsNotNone(_CaptureDriver.last_init_kwargs) - self.assertTrue(_CaptureDriver.last_init_kwargs["block_images"]) - self.assertTrue(_CaptureDriver.last_init_kwargs["block_images_and_css"]) - self.assertFalse(_CaptureDriver.last_init_kwargs["wait_for_complete_page_load"]) - self.assertEqual(_CaptureDriver.last_init_kwargs["user_agent"], "MyAgent/1.0") - self.assertEqual(_CaptureDriver.last_init_kwargs["window_size"], [1920, 1080]) - self.assertEqual(_CaptureDriver.last_init_kwargs["lang"], "en-US") - self.assertTrue(_CaptureDriver.last_init_kwargs["headless"]) - self.assertEqual( - _CaptureDriver.last_init_kwargs["proxy"], "http://proxy.example:8080" - ) - - def test_apply_scrolling(self): - mock_driver = MagicMock() - mock_driver.scroll_to_bottom = MagicMock() - ScraperEngine.apply_scrolling(mock_driver) - mock_driver.scroll_to_bottom.assert_called_once() - - -class UrlGuardUnitTests(unittest.TestCase): - def test_validate_rejects_non_http_schemes(self): - res = UrlGuard.validate("ftp://example.com/file") - self.assertFalse(res.is_allowed) - self.assertEqual(res.status_code, 400) - self.assertIn("Only http/https", res.error_message) - - def test_validate_rejects_missing_hostname(self): - res = UrlGuard.validate("http://") - self.assertFalse(res.is_allowed) - self.assertEqual(res.status_code, 400) - - def test_validate_rejects_localhost(self): - res = UrlGuard.validate("http://localhost:8080/test") - self.assertFalse(res.is_allowed) - self.assertEqual(res.status_code, 403) - - def test_validate_proxy_rejects_blocked_host(self): - res = UrlGuard.validate_proxy("http://127.0.0.1:8080") - self.assertFalse(res.is_allowed) - self.assertEqual(res.status_code, 403) - self.assertIn("Proxy URL is invalid or blocked", res.error_message) - - def test_is_blocked_ip_allows_well_known_nat64_prefix(self): - nat64_ip = ipaddress.ip_address("64:ff9b::3691:8e03") - self.assertFalse(UrlGuard.is_blocked_ip(nat64_ip)) - - def test_is_blocked_ip_still_blocks_loopback(self): - loopback = ipaddress.ip_address("127.0.0.1") - self.assertTrue(UrlGuard.is_blocked_ip(loopback)) - - -class ChallengeDetectorUnitTests(unittest.TestCase): - def test_detects_challenge_marker_and_category(self): - res = ChallengeDetector.detect("Just a moment...", 200) - self.assertTrue(res.challenge_detected) - self.assertTrue(res.blocked_detected) - self.assertEqual(res.detected_marker, "Just a moment...") - self.assertEqual(res.error_category, "challenge_block") - self.assertFalse(res.is_clean) - - def test_detects_http_status_block_without_marker(self): - res = ChallengeDetector.detect("Forbidden", 403) - self.assertFalse(res.challenge_detected) - self.assertTrue(res.blocked_detected) - self.assertIsNone(res.detected_marker) - self.assertEqual(res.error_category, "challenge_block") - self.assertFalse(res.is_clean) - - def test_clean_response(self): - res = ChallengeDetector.detect("

Hello

", 200) - self.assertTrue(res.is_clean) - self.assertFalse(res.blocked_detected) - self.assertFalse(res.challenge_detected) - self.assertIsNone(res.error_category) - - def test_driver_bot_detection_integration(self): - mock_driver = MagicMock() - mock_driver.is_bot_detected.return_value = True - - res = ChallengeDetector.detect( - "Clean page", 200, driver=mock_driver - ) - self.assertTrue(res.challenge_detected) - self.assertTrue(res.blocked_detected) - self.assertEqual(res.detected_marker, "botasaurus_driver_bot_detected") - self.assertEqual(res.error_category, "challenge_block") - - -class MetadataExtractorUnitTests(unittest.TestCase): - def test_extract_passive_metadata_from_requests_list(self): - class _Req: - def __init__(self, status, headers, url): - self.response = type( - "Resp", (), {"status_code": status, "headers": headers} - )() - self.url = url - - driver = type( - "D", - (), - { - "requests": [ - _Req( - 200, {"content-type": "text/html"}, "https://example.com/final" - ) - ] - }, - )() - status, headers, final_url = MetadataExtractor.extract_from_requests( - driver, "https://example.com" - ) - self.assertEqual(status, 200) - self.assertEqual(headers, {"content-type": "text/html"}) - self.assertEqual(final_url, "https://example.com/final") - - def test_extract_passive_metadata_from_performance_logs(self): - import json - - perf_log = [ - { - "message": json.dumps( - { - "message": { - "method": "Network.responseReceived", - "params": { - "type": "Document", - "response": { - "status": 200, - "headers": {"content-type": "text/html"}, - "url": "https://example.com/cdp-final", - }, - }, - } - } - ) - } - ] - driver = type( - "D", - (), - { - "get_log": lambda self, log_type: ( - perf_log if log_type == "performance" else [] - ) - }, - )() - status, headers, final_url = MetadataExtractor.extract_from_cdp_logs( - driver, "https://example.com" - ) - self.assertEqual(status, 200) - self.assertEqual(headers, {"content-type": "text/html"}) - self.assertEqual(final_url, "https://example.com/cdp-final") - - def test_extract_falls_back_to_200_when_no_driver_metadata(self): - driver = type("EmptyDriver", (), {"current_url": "https://example.com/dest"})() - meta = MetadataExtractor.fetch(driver, "https://example.com") - self.assertEqual(meta.status_code, 200) - self.assertEqual(meta.final_url, "https://example.com/dest") - self.assertIsNone(meta.headers) - self.assertIsNone(meta.metadata_error) - - -class ScraperEngineUnitTests(unittest.TestCase): - def test_browser_tier_step_budget_is_boot_aware(self): - captured: dict[str, int | None] = {"navigate_timeout": None} - - class _NavigateCaptureDriver(_FakeDriver): - def get(self, *_args, **kwargs): - captured["navigate_timeout"] = kwargs.get("timeout") - return None - - payload = ScrapeRequest( - url="https://example.com", - execution_mode="browser", - navigation_mode="get", - max_retries=0, - ) - - monotonic_values = [ - 1000.0, # execute started - 1020.0, # browser ready after boot - 1020.0, # remaining total - 1020.0, # remaining work - 1020.0, # render_ms - ] - - with tempfile.TemporaryDirectory() as tmp: - engine = ScraperEngine(runtime_root=Path(tmp)) - with ( - patch("app.engine.Driver", _NavigateCaptureDriver), - patch("app.engine.time.monotonic", side_effect=monotonic_values), - ): - result = engine.execute(payload) - - self.assertIsInstance(result, ScrapeSuccess) - self.assertEqual(captured["navigate_timeout"], 25) - - def test_request_id_collision_raises(self): - engine = ScraperEngine() - engine.register_request_id("req-123") - with self.assertRaises(RuntimeError): - engine.register_request_id("req-123") - engine.unregister_request_id("req-123") - # Should be re-registerable after unregistering - engine.register_request_id("req-123") - engine.unregister_request_id("req-123") - - def test_scrape_session_context_manager(self): - from app.engine import ScrapeSession - - with tempfile.TemporaryDirectory() as tmp: - engine = ScraperEngine(runtime_root=Path(tmp)) - with ScrapeSession(engine, "req-session-1") as session: - self.assertIn("req-session-1", engine._active_request_ids) - session.prepare_profile_dirs() - self.assertTrue(session.profile_dir.is_dir()) - - self.assertNotIn("req-session-1", engine._active_request_ids) - self.assertFalse(session.runtime_dir.exists()) - - def test_effective_user_agent_resolution(self): - req1 = ScrapeRequest( - url="https://example.com", - user_agent="CustomAgent/1.0", - headers={"User-Agent": "HeaderAgent/1.0"}, - ) - self.assertEqual(req1.effective_user_agent, "CustomAgent/1.0") - - req2 = ScrapeRequest( - url="https://example.com", - headers={"User-Agent": "HeaderAgent/1.0"}, - ) - self.assertEqual(req2.effective_user_agent, "HeaderAgent/1.0") - - req3 = ScrapeRequest(url="https://example.com") - self.assertIsNone(req3.effective_user_agent) - - def test_scrape_envelope_constructors(self): - success = ScrapeSuccess( - url="https://example.com", - final_url="https://example.com", - status_code=200, - headers={"content-type": "text/html; charset=utf-8"}, - html="", - metadata_error=None, - xhr_responses=[], - diagnostics=ScrapeDiagnostics( - request_id="req-abc", - attempts=1, - strategy_used=NavigationMode.GET, - render_ms=120, - execution_tier=ExecutionTier.BROWSER_DRIVER, - ), - ) - dumped = success.model_dump(mode="json") - self.assertEqual(dumped["status_code"], 200) - self.assertNotIn("error", dumped) - self.assertEqual(dumped["diagnostics"]["execution_tier"], "browser_driver") - self.assertEqual(dumped["final_url"], "https://example.com") - self.assertEqual(dumped["xhr_responses"], []) - self.assertEqual(dumped["headers"]["content-type"], "text/html; charset=utf-8") - - with_xhr = ScrapeSuccess( - url="https://example.com", - html="", - xhr_responses=[ - { - "url": "https://api.example.com/items", - "status_code": 200, - "headers": {"content-type": "application/json"}, - "body": '{"items":[]}', - } - ], - diagnostics=ScrapeDiagnostics( - request_id="req-xhr", - attempts=1, - strategy_used=NavigationMode.GET, - render_ms=10, - execution_tier=ExecutionTier.BROWSER_DRIVER, - ), - ) - self.assertEqual(len(with_xhr.xhr_responses), 1) - self.assertEqual(with_xhr.xhr_responses[0].url, "https://api.example.com/items") - - err = ScrapeError( - url="https://example.com", - error="Something broke", - error_category=ErrorCategory.NAVIGATION_ERROR, - diagnostics=ScrapeDiagnostics(request_id="req-err"), - ) - err_dump = err.model_dump(mode="json") - self.assertEqual(err_dump["error"], "Something broke") - self.assertEqual(err_dump["error_category"], "navigation_error") - self.assertNotIn("html", err_dump) - self.assertNotIn("xhr_responses", err_dump) - - def test_wait_for_readiness_uses_sleep_random_when_available(self): - mock_driver = MagicMock() - mock_driver.sleep_random = MagicMock() - ScraperEngine.wait_for_readiness(mock_driver, selector=None, timeout_seconds=10) - mock_driver.sleep_random.assert_called_once_with(0.5, 1.2) - - -class _FakeRequestId(str): - def to_json(self): - return str(self) - - -class _FakeNetworkResponse: - def __init__(self, url, status, mime_type, headers=None): - self.url = url - self.status = status - self.mime_type = mime_type - self.headers = headers or {} - - -class _FakeTab: - """Minimal CDP tab stub for XhrCollector unit tests.""" - - def __init__(self, bodies=None): - self.bodies = bodies or {} - self.network_enabled = False - self.response_handler = None - self.finished_handler = None - - def send(self, cdp_obj): - cmd = next(cdp_obj) - method = cmd.get("method") - if method == "Network.enable": - self.network_enabled = True - try: - cdp_obj.send({}) - except StopIteration as exc: - return exc.value - return None - if method == "Network.getResponseBody": - rid = str(cmd["params"]["requestId"]) - body, b64 = self.bodies.get(rid, ("", False)) - try: - cdp_obj.send({"body": body, "base64Encoded": b64}) - except StopIteration as exc: - return exc.value - return None - raise AssertionError(f"unexpected CDP method: {method}") - - def after_response_received(self, handler): - self.response_handler = handler - - def add_handler(self, _event_type, handler): - self.finished_handler = handler - - -class XhrCollectorTests(unittest.TestCase): - def setUp(self): - from app.xhr_collector import XhrCollector - - self.XhrCollector = XhrCollector - self.target = "https://example.com/" - self.collector = XhrCollector(self.target) - - def _drive_json(self, tab, request_id, url, body, mime="application/json"): - rid = _FakeRequestId(request_id) - tab.bodies[str(rid)] = (body, False) - self.collector._on_response( - rid, - _FakeNetworkResponse(url, 200, mime, {"content-type": mime}), - None, - ) - self.collector._on_finished(type("E", (), {"request_id": rid})()) - - def test_install_enables_network_and_registers_handlers(self): - tab = _FakeTab() - self.collector.install(tab) - self.assertTrue(tab.network_enabled) - self.assertEqual( - tab.response_handler.__func__, self.collector._on_response.__func__ - ) - self.assertIs(tab.response_handler.__self__, self.collector) - self.assertEqual( - tab.finished_handler.__func__, self.collector._on_finished.__func__ - ) - self.assertIs(tab.finished_handler.__self__, self.collector) - - def test_captures_json_subresource(self): - tab = _FakeTab() - self.collector.install(tab) - self._drive_json( - tab, "1", "https://api.example.com/feed", '{"items":[{"title":"A"}]}' - ) - results = self.collector.harvest(tab) - self.assertEqual(len(results), 1) - self.assertEqual(results[0]["url"], "https://api.example.com/feed") - self.assertEqual(results[0]["status_code"], 200) - self.assertEqual(results[0]["headers"], {"content-type": "application/json"}) - self.assertIn("items", results[0]["body"]) - - def test_skips_non_json_mime(self): - tab = _FakeTab() - self.collector.install(tab) - self._drive_json( - tab, - "1", - "https://cdn.example.com/app.js", - "console.log(1)", - mime="application/javascript", - ) - self.assertEqual(self.collector.harvest(tab), []) - - def test_skips_main_document(self): - tab = _FakeTab() - self.collector.install(tab) - rid = _FakeRequestId("doc") - tab.bodies[str(rid)] = ('{"nope":true}', False) - self.collector._on_response( - rid, - _FakeNetworkResponse(self.target, 200, "application/json"), - None, - ) - self.collector._on_finished(type("E", (), {"request_id": rid})()) - self.assertEqual(self.collector.harvest(tab), []) - - def test_skips_empty_body(self): - tab = _FakeTab() - self.collector.install(tab) - self._drive_json(tab, "1", "https://api.example.com/empty", "") - self.assertEqual(self.collector.harvest(tab), []) - - def test_enforces_max_responses_cap(self): - tab = _FakeTab() - self.collector.install(tab) - for i in range(self.XhrCollector.MAX_RESPONSES + 5): - self._drive_json( - tab, str(i), f"https://api.example.com/i/{i}", f'{{"i":{i}}}' - ) - results = self.collector.harvest(tab) - self.assertEqual(len(results), self.XhrCollector.MAX_RESPONSES) - - def test_enforces_max_body_bytes_cap(self): - tab = _FakeTab() - self.collector.install(tab) - oversized = "x" * (self.XhrCollector.MAX_BODY_BYTES + 1) - self._drive_json(tab, "1", "https://api.example.com/big", oversized) - self.assertEqual(self.collector.harvest(tab), []) - - def test_enforces_aggregate_bytes_cap(self): - tab = _FakeTab() - self.collector.install(tab) - # Five near-max bodies would exceed 2 MB aggregate; stop once budget trips. - chunk = "y" * self.XhrCollector.MAX_BODY_BYTES - for i in range(5): - self._drive_json(tab, str(i), f"https://api.example.com/chunk/{i}", chunk) - results = self.collector.harvest(tab) - total = sum(len(entry["body"].encode("utf-8")) for entry in results) - self.assertLessEqual(total, self.XhrCollector.MAX_AGGREGATE_BYTES) - self.assertEqual(len(results), 4) - self.assertLess(len(results), 5) - - def test_headers_allowlist_keeps_only_content_type(self): - tab = _FakeTab() - self.collector.install(tab) - rid = _FakeRequestId("hdr") - tab.bodies[str(rid)] = ('{"ok":true}', False) - self.collector._on_response( - rid, - _FakeNetworkResponse( - "https://api.example.com/secure", - 200, - "application/json", - { - "Content-Type": "application/json; charset=utf-8", - "Set-Cookie": "session=secret", - "X-Request-Id": "abc", - }, - ), - None, - ) - self.collector._on_finished(type("E", (), {"request_id": rid})()) - results = self.collector.harvest(tab) - self.assertEqual(len(results), 1) - self.assertEqual( - results[0]["headers"], - {"content-type": "application/json; charset=utf-8"}, - ) - self.assertNotIn("Set-Cookie", results[0]["headers"]) - self.assertNotIn("set-cookie", results[0]["headers"]) - - def test_reset_clears_pending_ready_and_collected(self): - tab = _FakeTab() - self.collector.install(tab) - self._drive_json( - tab, "1", "https://api.example.com/old", '{"from":"failed-attempt"}' - ) - first = self.collector.harvest(tab) - self.assertEqual(len(first), 1) - - self.collector.reset() - self.assertEqual(self.collector.results(), []) - - self._drive_json( - tab, "2", "https://api.example.com/new", '{"from":"success-attempt"}' - ) - second = self.collector.harvest(tab) - self.assertEqual(len(second), 1) - self.assertEqual(second[0]["body"], '{"from":"success-attempt"}') - self.assertNotIn("failed-attempt", second[0]["body"]) - - -class RequestIdContractTests(unittest.TestCase): - INBOUND_ID = "550e8400-e29b-41d4-a716-446655440000" - - def test_honored_request_id_on_200(self): - from fastapi.testclient import TestClient - - import app.main as main_mod - from app.main import app - - def fake_execute(payload, _deadline=None, *, request_id=None, progress=None): - return ScrapeSuccess( - url=str(payload.url), - html="", - diagnostics=ScrapeDiagnostics( - request_id=request_id, - attempts=1, - render_ms=1, - execution_tier=ExecutionTier.HTTP_REQUEST, - ), - ) - - with patch.object(main_mod._engine, "execute", side_effect=fake_execute): - client = TestClient(app) - response = client.post( - "/scrape", - json={"url": "https://example.com"}, - headers={"X-Request-Id": self.INBOUND_ID}, - ) - - self.assertEqual(response.status_code, 200) - self.assertEqual(response.json()["diagnostics"]["request_id"], self.INBOUND_ID) - - def test_honored_request_id_on_400(self): - from fastapi.testclient import TestClient - - from app.main import app - - client = TestClient(app) - response = client.post( - "/scrape", - json={"url": "https://this-host-does-not-exist-12345.invalid/"}, - headers={"X-Request-Id": self.INBOUND_ID}, - ) - - self.assertEqual(response.status_code, 400) - body = response.json() - self.assertEqual(body["diagnostics"]["request_id"], self.INBOUND_ID) - self.assertEqual(body["error_category"], "validation") - - def test_honored_request_id_on_422(self): - from fastapi.testclient import TestClient - - from app.main import app - - client = TestClient(app) - response = client.post( - "/scrape", - json={ - "url": "https://example.com", - "window_size": [1920], - }, - headers={"X-Request-Id": self.INBOUND_ID}, - ) - - self.assertEqual(response.status_code, 422) - body = response.json() - self.assertEqual(body["diagnostics"]["request_id"], self.INBOUND_ID) - self.assertEqual(body["error_category"], "validation") - - def test_request_id_collision_returns_502(self): - from fastapi.testclient import TestClient - - import app.main as main_mod - from app.main import app - - main_mod._engine.register_request_id(self.INBOUND_ID) - try: - client = TestClient(app) - response = client.post( - "/scrape", - json={"url": "https://example.com"}, - headers={"X-Request-Id": self.INBOUND_ID}, - ) - finally: - main_mod._engine.unregister_request_id(self.INBOUND_ID) - - self.assertEqual(response.status_code, 502) - body = response.json() - self.assertEqual(body["diagnostics"]["request_id"], self.INBOUND_ID) - self.assertEqual(body["error_category"], "navigation_error") - - -class SchemaValidationHttpTests(unittest.TestCase): - def test_schema_422_returns_scrape_envelope(self): - from fastapi.testclient import TestClient - - from app.main import app - - with self.assertLogs("botasaurus_scrape_api", level="INFO") as captured: - client = TestClient(app) - response = client.post( - "/scrape", - json={ - "url": "https://example.com", - "window_size": [1920], - }, - ) - - self.assertEqual(response.status_code, 422) - body = response.json() - self.assertNotIn("detail", body) - self.assertEqual(body["url"], "https://example.com") - self.assertTrue(body["error"]) - self.assertIn("window_size", body["error"]) - self.assertEqual(body["error_category"], "validation") - self.assertNotIn("html", body) - self.assertTrue(body["diagnostics"]["request_id"]) - log_text = "\n".join(captured.output) - self.assertIn("request_schema_422", log_text) - self.assertIn("host=example.com", log_text) - self.assertIn("field=window_size", log_text) - - def test_scrape_clamps_wait_timeout_instead_of_422(self): - from fastapi.testclient import TestClient - - import app.main as main_mod - from app.main import app - - captured: dict[str, int] = {} - - def fake_execute(payload, _deadline=None, *, request_id=None, progress=None): - captured["wait"] = payload.wait_timeout_seconds - return ScrapeSuccess( - url=str(payload.url), - html="", - diagnostics=ScrapeDiagnostics( - request_id=request_id or "req-wait-clamp", - attempts=1, - render_ms=1, - execution_tier=ExecutionTier.HTTP_REQUEST, - ), - ) - - with patch.object(main_mod._engine, "execute", side_effect=fake_execute): - client = TestClient(app) - response = client.post( - "/scrape", - json={ - "url": "https://example.com", - "wait_timeout_seconds": 35, - }, - ) - - self.assertNotEqual(response.status_code, 422) - self.assertEqual(response.status_code, 200) - self.assertEqual(captured["wait"], DEFAULT_SCRAPE_WORK_TIMEOUT_SECONDS) - self.assertEqual(captured["wait"], 30) - - -def _schema_ref_names(node: object) -> set[str]: - names: set[str] = set() - if isinstance(node, dict): - ref = node.get("$ref") - if isinstance(ref, str) and "/schemas/" in ref: - names.add(ref.rsplit("/", 1)[-1]) - for value in node.values(): - names |= _schema_ref_names(value) - elif isinstance(node, list): - for item in node: - names |= _schema_ref_names(item) - return names - - -class OpenApiContractTests(unittest.TestCase): - @classmethod - def setUpClass(cls): - from app.main import app - - cls.schema = app.openapi() - - def test_scrape_documents_x_request_id_header(self): - scrape = self.schema["paths"]["/scrape"]["post"] - parameters = scrape.get("parameters") or [] - header_params = { - param["name"]: param for param in parameters if param.get("in") == "header" - } - self.assertIn("X-Request-Id", header_params) - self.assertFalse(header_params["X-Request-Id"].get("required", True)) - - def test_documents_health_and_scrape_paths(self): - paths = self.schema["paths"] - self.assertIn("/health", paths) - self.assertIn("get", paths["/health"]) - self.assertIn("/scrape", paths) - self.assertIn("post", paths["/scrape"]) - - def test_operation_ids_and_tags(self): - health = self.schema["paths"]["/health"]["get"] - scrape = self.schema["paths"]["/scrape"]["post"] - self.assertEqual(health["operationId"], "get-health") - self.assertEqual(scrape["operationId"], "scrape-url") - self.assertEqual(health["tags"], ["health"]) - self.assertEqual(scrape["tags"], ["scrape"]) - tag_names = {tag["name"] for tag in self.schema["tags"]} - self.assertEqual(tag_names, {"health", "scrape"}) - for tag in self.schema["tags"]: - self.assertTrue(tag.get("description")) - - def test_info_servers_and_version(self): - info = self.schema["info"] - self.assertEqual(info["title"], "Botasaurus Scrape API") - self.assertEqual(info["version"], "2.0.0") - self.assertTrue(info.get("description")) - self.assertEqual(info["contact"]["name"], "html2rss") - self.assertEqual( - info["contact"]["url"], - "https://github.com/html2rss/botasaurus-scrape-api/issues", - ) - self.assertNotIn("email", info["contact"]) - self.assertEqual(info["license"]["name"], "MIT") - self.assertTrue(info["license"].get("url")) - servers = self.schema["servers"] - self.assertEqual(servers[0]["url"], "http://localhost:4010") - self.assertEqual(servers[0]["description"], "Local Docker (make serve)") - - def test_scrape_documents_contract_status_codes(self): - responses = self.schema["paths"]["/scrape"]["post"]["responses"] - for status in ("200", "400", "403", "422", "502", "504"): - self.assertIn(status, responses) - self.assertTrue(responses[status].get("description")) - - def test_scrape_error_statuses_use_scrape_envelope_not_fastapi_detail(self): - responses = self.schema["paths"]["/scrape"]["post"]["responses"] - success_refs = _schema_ref_names(responses["200"]) - self.assertIn("ScrapeSuccess", success_refs) - self.assertNotIn("ScrapeResponse", success_refs) - for status in ("400", "403", "422", "502", "504"): - with self.subTest(status=status): - refs = _schema_ref_names(responses[status]) - self.assertIn("ScrapeError", refs) - self.assertNotIn("ScrapeResponse", refs) - self.assertNotIn("HTTPValidationError", refs) - self.assertNotIn("ValidationError", refs) - - def test_wait_timeout_seconds_openapi_does_not_advertise_range_as_422(self): - props = self.schema["components"]["schemas"]["ScrapeRequest"]["properties"] - wait_schema = props["wait_timeout_seconds"] - self.assertNotIn("minimum", wait_schema) - self.assertNotIn("maximum", wait_schema) - description = wait_schema.get("description") or "" - self.assertIn("clamped", description) - - def test_window_size_openapi_is_object(self): - props = self.schema["components"]["schemas"]["ScrapeRequest"]["properties"] - window_schema = props["window_size"] - refs = _schema_ref_names(window_schema) - self.assertIn("WindowSize", refs) - size_schema = self.schema["components"]["schemas"]["WindowSize"] - size_props = size_schema["properties"] - self.assertIn("width", size_props) - self.assertIn("height", size_props) - self.assertNotIn("minItems", window_schema) - self.assertNotIn("maxItems", window_schema) - self.assertNotIn("scroll_to_bottom", props) - - def test_health_schema_includes_status_fields(self): - health_200 = self.schema["paths"]["/health"]["get"]["responses"]["200"] - refs = _schema_ref_names(health_200) - self.assertIn("HealthResponse", refs) - health_schema = self.schema["components"]["schemas"]["HealthResponse"] - properties = health_schema["properties"] - self.assertIn("status", properties) - self.assertIn("service", properties) - self.assertIn("botasaurus_version", properties) - status_schema = properties["status"] - self.assertTrue( - status_schema.get("const") == "ok" - or status_schema.get("enum") == ["ok"] - or "ok" in (status_schema.get("examples") or []) - ) - - def test_xhr_responses_use_xhr_response_model(self): - scrape_schema = self.schema["components"]["schemas"]["ScrapeSuccess"] - refs = _schema_ref_names(scrape_schema["properties"]["xhr_responses"]) - self.assertIn("XhrResponse", refs) - xhr_schema = self.schema["components"]["schemas"]["XhrResponse"] - properties = xhr_schema["properties"] - for field in ("url", "status_code", "headers", "body"): - self.assertIn(field, properties) - self.assertIn("diagnostics", scrape_schema["properties"]) - self.assertNotIn("ScrapeResponse", self.schema["components"]["schemas"]) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_bench_regression.py b/tests/test_bench_regression.py new file mode 100644 index 0000000..046c08f --- /dev/null +++ b/tests/test_bench_regression.py @@ -0,0 +1,27 @@ +"""Lightweight guard that the scrape bench harness completes.""" + +from __future__ import annotations + +import os +import subprocess +import sys +import unittest +from pathlib import Path + + +class BenchRegressionTests(unittest.TestCase): + def test_bench_script_completes_under_generous_ceiling(self): + repo_root = Path(__file__).resolve().parents[1] + script = repo_root / "scripts" / "bench_scrape.py" + env = {**os.environ, "PYTHONPATH": str(repo_root)} + completed = subprocess.run( + [sys.executable, str(script), "--runs", "2"], + cwd=repo_root, + env=env, + capture_output=True, + text=True, + check=False, + timeout=120, + ) + self.assertEqual(completed.returncode, 0, completed.stderr) + self.assertIn("wall_ms_p50=", completed.stdout) diff --git a/typings/botasaurus/browser.pyi b/typings/botasaurus/browser.pyi new file mode 100644 index 0000000..f6a77a8 --- /dev/null +++ b/typings/botasaurus/browser.pyi @@ -0,0 +1,45 @@ +from typing import Any + +class Driver: + page_html: str | None + current_url: str | None + requests: list[Any] + + def __init__( + self, + *, + headless: bool = ..., + enable_xvfb_virtual_display: bool = ..., + proxy: str | None = ..., + profile: str | None = ..., + tiny_profile: bool = ..., + block_images: bool = ..., + block_images_and_css: bool = ..., + wait_for_complete_page_load: bool = ..., + chrome_executable_path: str | None = ..., + extensions: list[Any] = ..., + arguments: list[str] = ..., + remove_default_browser_check_argument: bool = ..., + user_agent: str | None = ..., + window_size: list[int] | None = ..., + lang: str | None = ..., + beep: bool = ..., + host: str | None = ..., + port: int | None = ..., + ) -> None: ... + def get(self, url: str, /, **kwargs: Any) -> Any: ... + def google_get(self, url: str, /, **kwargs: Any) -> Any: ... + def organic_get(self, url: str, /, **kwargs: Any) -> Any: ... + def wait_for_element(self, selector: str, /, **kwargs: Any) -> Any: ... + def sleep(self, seconds: float, /) -> None: ... + def sleep_random(self, minimum: float, maximum: float, /) -> None: ... + def scroll_to_bottom(self) -> None: ... + def scroll(self) -> None: ... + def run_js(self, script: str, /) -> Any: ... + def execute_script(self, script: str, /) -> Any: ... + def add_cookies(self, cookies: list[dict[str, str]]) -> None: ... + def bypass_cloudflare(self) -> None: ... + def close(self) -> None: ... + def get_log(self, log_type: str) -> list[dict[str, str]]: ... + @property + def _tab(self) -> Any: ... diff --git a/typings/botasaurus/request.pyi b/typings/botasaurus/request.pyi new file mode 100644 index 0000000..2efb34e --- /dev/null +++ b/typings/botasaurus/request.pyi @@ -0,0 +1,34 @@ +from typing import Any, Literal + +class HttpResponse: + text: str | None + status_code: int | None + headers: dict[str, str] + url: str + +class Request: + def get( + self, + url: str, + /, + *, + referer: str = ..., + params: dict[str, Any] | None = ..., + data: Any = ..., + headers: dict[str, str] | None = ..., + browser: Literal["firefox", "chrome"] | None = ..., + os: Literal["windows", "mac", "linux"] | None = ..., + user_agent: str | None = ..., + cookies: dict[str, str] | None = ..., + files: Any = ..., + auth: Any = ..., + timeout: int | None = ..., + allow_redirects: bool = ..., + proxies: dict[str, str] | None = ..., + hooks: Any = ..., + stream: bool | None = ..., + verify: bool | None = ..., + cert: Any = ..., + json: Any = ..., + ) -> HttpResponse: ... + def close(self) -> None: ... diff --git a/typings/botasaurus_driver/__init__.pyi b/typings/botasaurus_driver/__init__.pyi new file mode 100644 index 0000000..36461db --- /dev/null +++ b/typings/botasaurus_driver/__init__.pyi @@ -0,0 +1,14 @@ +from collections.abc import Generator +from typing import Any + +from botasaurus_driver import cdp as cdp + +class CdpCommandGenerator(Generator[dict[str, Any], dict[str, Any], Any]): + pass + +class CdpTab: + def send(self, cdp_obj: CdpCommandGenerator) -> Any: ... + def after_response_received(self, handler: Any, /) -> None: ... + def add_handler(self, event_type: type[Any], handler: Any, /) -> None: ... + +def enable_network() -> CdpCommandGenerator: ... diff --git a/typings/botasaurus_driver/cdp/__init__.pyi b/typings/botasaurus_driver/cdp/__init__.pyi new file mode 100644 index 0000000..bc0ec4f --- /dev/null +++ b/typings/botasaurus_driver/cdp/__init__.pyi @@ -0,0 +1 @@ +from botasaurus_driver.cdp import network as network diff --git a/typings/botasaurus_driver/cdp/network.pyi b/typings/botasaurus_driver/cdp/network.pyi new file mode 100644 index 0000000..1dce72e --- /dev/null +++ b/typings/botasaurus_driver/cdp/network.pyi @@ -0,0 +1,19 @@ +from collections.abc import Generator +from typing import Any + +RequestId = str | int + +class LoadingFinished: + request_id: RequestId + + def __init__(self, *, request_id: RequestId) -> None: ... + +class NetworkResponse: + url: str + status: int + mime_type: str | None + headers: dict[str, str] + +def get_response_body( + request_id: RequestId, +) -> Generator[dict[str, Any], dict[str, Any], tuple[str, bool]]: ... diff --git a/typings/botasaurus_driver/core/custom_storage_cdp.pyi b/typings/botasaurus_driver/core/custom_storage_cdp.pyi new file mode 100644 index 0000000..0bed13e --- /dev/null +++ b/typings/botasaurus_driver/core/custom_storage_cdp.pyi @@ -0,0 +1,4 @@ +from collections.abc import Generator +from typing import Any + +def enable_network() -> Generator[dict[str, Any], dict[str, Any], Any]: ... diff --git a/typings/httpx/__init__.pyi b/typings/httpx/__init__.pyi new file mode 100644 index 0000000..c785264 --- /dev/null +++ b/typings/httpx/__init__.pyi @@ -0,0 +1,9 @@ +from typing import Any + +class Response: + status_code: int + text: str + headers: dict[str, str] + + def json(self) -> dict[str, Any]: ... + def raise_for_status(self) -> None: ... diff --git a/typings/starlette/testclient.pyi b/typings/starlette/testclient.pyi new file mode 100644 index 0000000..6a5d861 --- /dev/null +++ b/typings/starlette/testclient.pyi @@ -0,0 +1,13 @@ +from contextlib import AbstractContextManager +from typing import Any + +from httpx import Response + +class TestClient(AbstractContextManager["TestClient"]): + app: Any + + def __init__(self, app: Any, base_url: str = ...) -> None: ... + def __enter__(self) -> TestClient: ... + def __exit__(self, *args: object) -> None: ... + def get(self, url: str, **kwargs: Any) -> Response: ... + def post(self, url: str, **kwargs: Any) -> Response: ...