perf(engine): lazy imports and dedupe browser-tier hot path

Defer Botasaurus/CDP imports to tier entrypoints, consolidate post-bypass
page-state collection, and document SCRAPE_MAX_WORKERS tuning for low-RAM
hosts. Bench p50 wall_ms: 106.3 -> 102.9 (no regression).

Author: gildesmarais

README.md

@@ -333,7 +333,7 @@ Use a **separate Sentry project** from html2rss-web (`BOTASAURUS_SENTRY_DSN` →
 
 | Variable | Default | Description |
 | :--- | :--- | :--- |
-| `SCRAPE_MAX_WORKERS` | `4` | Threadpool worker limit for sync browser execution. |
+| `SCRAPE_MAX_WORKERS` | `4` | Threadpool worker limit for sync browser execution. On low-RAM hosts (for example Docker with `--memory=768m` or a 1–2 vCPU VM), use `1` or `2` to limit concurrent Chromium boots; higher values increase queue wait and swap pressure without improving wall time. |
 | `SCRAPE_TIMEOUT_SECONDS` | `45` | Handler wall-clock budget in seconds (queue, browser boot, and work). |
 | `SCRAPE_WORK_TIMEOUT_SECONDS` | `30` | Post-boot navigate, selector wait, and scroll budget in seconds. |
 | `SCRAPE_RUNTIME_MIN_FREE_BYTES` | `268435456` (256 MiB) | Prune orphan runtime dirs when free space drops below this threshold. |

app/engine/browser_tier.py

@@ -5,12 +5,11 @@
 import errno
 import logging
 import time
+from typing import Any
 from urllib.parse import urlparse
 
-from botasaurus.browser import Driver
-
 from app.config import Settings
-from app.engine.driver_capabilities import call_if_available
+from app.engine.driver_capabilities import DriverProtocol, call_if_available
 from app.engine.envelope import build_error, build_success
 from app.engine.request_tier import remaining_total_seconds
 from app.engine.session import ScrapeSession
@@ -22,8 +21,8 @@
     resolve_strategies,
     wait_for_readiness,
 )
-from app.infra.detector import ChallengeDetector
-from app.infra.metadata import MetadataExtractor
+from app.infra.detector import ChallengeAssessment, ChallengeDetector
+from app.infra.metadata import MetadataExtractor, MetadataResult
 from app.infra.scrape_progress import ScrapeProgress
 from app.infra.xhr_collector import XhrCollector
 from app.schemas.enums import ErrorCategory, ExecutionTier, TimeoutPhase
@@ -56,6 +55,20 @@ def is_timeout_exception(exc: Exception) -> bool:
     return "timeout" in str(exc).lower()
 
 
+def collect_page_state(
+    driver: DriverProtocol,
+    target_url: str,
+    collector: XhrCollector,
+    *,
+    include_xhr: bool = True,
+) -> tuple[str, MetadataResult, ChallengeAssessment, list[dict[str, Any]]]:
+    xhr_responses = harvest_xhr(collector, driver) if include_xhr else []
+    html = driver.page_html or ""
+    meta = MetadataExtractor.fetch(driver, target_url)
+    assessment = ChallengeDetector.detect(html, meta.status_code, driver=driver)
+    return html, meta, assessment, xhr_responses
+
+
 def run_browser_tier(
     payload: ScrapeRequest,
     session: ScrapeSession,
@@ -64,6 +77,8 @@ def run_browser_tier(
     *,
     settings: Settings,
 ) -> ScrapeSuccess | ScrapeError:
+    from botasaurus.browser import Driver
+
     target_url = str(payload.url)
     request_id = session.request_id
     progress.mark(
@@ -141,21 +156,22 @@ def run_browser_tier(
             if payload.scroll:
                 apply_scrolling(session.driver)
 
-            xhr_responses = harvest_xhr(collector, session.driver)
-
-            html = session.driver.page_html or ""
-            meta = MetadataExtractor.fetch(session.driver, target_url)
-            assessment = ChallengeDetector.detect(
-                html, meta.status_code, driver=session.driver
+            html, meta, assessment, xhr_responses = collect_page_state(
+                session.driver,
+                target_url,
+                collector,
+                include_xhr=False,
             )
 
             if assessment.challenge_detected or assessment.blocked_detected:
                 call_if_available(session.driver, "bypass_cloudflare")
-                html = session.driver.page_html or ""
-                meta = MetadataExtractor.fetch(session.driver, target_url)
-                assessment = ChallengeDetector.detect(
-                    html, meta.status_code, driver=session.driver
+                html, meta, assessment, xhr_responses = collect_page_state(
+                    session.driver,
+                    target_url,
+                    collector,
+                    include_xhr=True,
                 )
+            else:
                 xhr_responses = harvest_xhr(collector, session.driver)
 
             if assessment.challenge_detected or assessment.blocked_detected:

app/engine/request_tier.py

@@ -6,8 +6,6 @@
 import time
 from urllib.parse import urlparse
 
-from botasaurus.request import Request
-
 from app.config import Settings
 from app.engine.envelope import build_error, build_success
 from app.infra.detector import ChallengeDetector
@@ -34,6 +32,8 @@ def run_request_tier(
     *,
     settings: Settings,
 ) -> ScrapeSuccess | ScrapeError | None:
+    from botasaurus.request import Request
+
     target_url = str(payload.url)
     remaining_budget = remaining_total_seconds(settings, started_monotonic)
     progress.mark(

app/infra/xhr_collector.py

@@ -13,9 +13,6 @@
 import threading
 from typing import Any
 
-from botasaurus_driver import cdp
-from botasaurus_driver.core.custom_storage_cdp import enable_network
-
 logger = logging.getLogger("botasaurus_scrape_api")
 
 
@@ -37,6 +34,9 @@ def __init__(self, target_url: str) -> None:
 
     def install(self, tab: Any) -> None:
         """Enable the network domain and register handlers before navigation."""
+        from botasaurus_driver import cdp
+        from botasaurus_driver.core.custom_storage_cdp import enable_network
+
         tab.send(enable_network())
         tab.after_response_received(self._on_response)
         tab.add_handler(cdp.network.LoadingFinished, self._on_finished)
@@ -74,7 +74,7 @@ def _on_response(self, request_id: Any, response: Any, _event: Any) -> None:
                 "request_id": request_id,
             }
 
-    def _on_finished(self, event: cdp.network.LoadingFinished) -> None:
+    def _on_finished(self, event: Any) -> None:
         # Do not call get_response_body here — CDP deadlocks (Phase 0 spike).
         rid = str(event.request_id)
         with self._lock:
@@ -116,6 +116,8 @@ def harvest(self, tab: Any) -> list[dict[str, Any]]:
         return self.results()
 
     def _fetch_body(self, tab: Any, request_id: Any, rid: str) -> str | None:
+        from botasaurus_driver import cdp
+
         try:
             body, b64 = tab.send(cdp.network.get_response_body(request_id))
         except Exception as exc:

tests/api/test_request_schema.py

@@ -103,7 +103,7 @@ def test_clamped_wait_timeout_allows_execute(self):
 
         with tempfile.TemporaryDirectory() as tmp:
             engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp))
-            with patch("app.engine.browser_tier.Driver", FakeDriver):
+            with patch("botasaurus.browser.Driver", FakeDriver):
                 result = engine.execute(payload)
 
         self.assertIsNone(result.error if isinstance(result, ScrapeError) else None)
@@ -126,7 +126,7 @@ def test_html_response_sets_utf8_content_type_and_normalizes_body(self):
 
         with tempfile.TemporaryDirectory() as tmp:
             engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp))
-            with patch("app.engine.request_tier.Request", FakeRequest):
+            with patch("botasaurus.request.Request", FakeRequest):
                 result = engine.execute(payload)
 
         self.assertIsInstance(result, ScrapeSuccess)
@@ -155,7 +155,7 @@ def test_utf8_normalize_leaves_correct_unicode_unchanged(self):
         payload = ScrapeRequest(url="https://example.com", execution_mode="request")
         with tempfile.TemporaryDirectory() as tmp:
             engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp))
-            with patch("app.engine.request_tier.Request", FakeRequest):
+            with patch("botasaurus.request.Request", FakeRequest):
                 result = engine.execute(payload)
 
         self.assertEqual(result.html, html)
@@ -176,8 +176,8 @@ def test_request_tier_blocked_status_escalates_to_browser(self):
                         settings=get_settings(), runtime_root=Path(tmp)
                     )
                     with (
-                        patch("app.engine.request_tier.Request", FakeRequest),
-                        patch("app.engine.browser_tier.Driver", ArticleDriver),
+                        patch("botasaurus.request.Request", FakeRequest),
+                        patch("botasaurus.browser.Driver", ArticleDriver),
                     ):
                         result = engine.execute(payload)
 
@@ -230,7 +230,7 @@ def test_cleanup_runs_on_navigation_error(self):
         with tempfile.TemporaryDirectory() as tmp:
             runtime_root = Path(tmp)
             engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root)
-            with patch("app.engine.browser_tier.Driver", FakeDriver):
+            with patch("botasaurus.browser.Driver", FakeDriver):
                 result = engine.execute(payload)
 
             self.assertEqual(result.error_category, ErrorCategory.NAVIGATION_ERROR)
@@ -254,7 +254,7 @@ def boom_mkdir(*_args, exist_ok=False, **_kwargs):
                 raise OSError(28, "No space left on device")
 
             with (
-                patch("app.engine.browser_tier.Driver", FakeDriver),
+                patch("botasaurus.browser.Driver", FakeDriver),
                 patch.object(Path, "mkdir", side_effect=boom_mkdir),
             ):
                 result = engine.execute(payload)
@@ -279,7 +279,7 @@ def test_prune_orphan_runtime_dirs_before_new_request(self):
             (orphan / "profile").mkdir()
 
             engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root)
-            with patch("app.engine.browser_tier.Driver", FakeDriver):
+            with patch("botasaurus.browser.Driver", FakeDriver):
                 result = engine.execute(payload)
 
             self.assertIsInstance(result, ScrapeSuccess)
@@ -308,7 +308,7 @@ def test_prune_orphan_runtime_dirs_before_http_request(self):
             (orphan / "profile").mkdir()
 
             engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root)
-            with patch("app.engine.request_tier.Request", FakeRequest):
+            with patch("botasaurus.request.Request", FakeRequest):
                 result = engine.execute(payload)
 
             self.assertEqual(result.html, html)
@@ -333,7 +333,7 @@ def test_run_scrape_forwards_driver_kwargs(self):
         with tempfile.TemporaryDirectory() as tmp:
             runtime_root = Path(tmp)
             engine = ScraperEngine(settings=get_settings(), runtime_root=runtime_root)
-            with patch("app.engine.browser_tier.Driver", CaptureDriver):
+            with patch("botasaurus.browser.Driver", CaptureDriver):
                 result = engine.execute(payload)
 
         self.assertIsInstance(result, ScrapeSuccess)

tests/engine/test_scraper_engine.py

@@ -50,7 +50,7 @@ def get(self, *_args, **kwargs):
         with tempfile.TemporaryDirectory() as tmp:
             engine = ScraperEngine(settings=get_settings(), runtime_root=Path(tmp))
             with (
-                patch("app.engine.browser_tier.Driver", _NavigateCaptureDriver),
+                patch("botasaurus.browser.Driver", _NavigateCaptureDriver),
                 patch(
                     "app.engine.browser_tier.time.monotonic",
                     side_effect=monotonic_values,

tests/test_engine_isolation.py

@@ -69,7 +69,7 @@ def tracking_enter(self: ScrapeSession) -> ScrapeSession:
                     headers={"X-Request-Id": self.INBOUND_ID_B},
                 )
             )
-            with patch("app.engine.request_tier.Request") as mock_request:
+            with patch("botasaurus.request.Request") as mock_request:
                 mock_request.return_value.get.return_value = type(
                     "Resp",
                     (),
@@ -109,7 +109,7 @@ def slow_enter(self: ScrapeSession) -> ScrapeSession:
         with (
             patch.object(ScrapeSession, "__enter__", slow_enter),
             test_client() as client,
-            patch("app.engine.request_tier.Request") as mock_request,
+            patch("botasaurus.request.Request") as mock_request,
         ):
             mock_request.return_value.get.return_value = type(
                 "Resp",
@@ -162,7 +162,7 @@ def capture_exit(self, exc_type, exc_val, exc_tb):
         with (
             patch.object(ScrapeSession, "__exit__", capture_exit),
             test_client() as client,
-            patch("app.engine.request_tier.Request") as mock_request,
+            patch("botasaurus.request.Request") as mock_request,
         ):
             mock_request.return_value.get.return_value = type(
                 "Resp",

tests/test_timeout_phase.py

@@ -73,11 +73,11 @@ def _execute(payload, *, progress, request_id, **patches):
         with ExitStack() as stack:
             if "Driver" in patches:
                 stack.enter_context(
-                    patch("app.engine.browser_tier.Driver", patches["Driver"])
+                    patch("botasaurus.browser.Driver", patches["Driver"])
                 )
             if "Request" in patches:
                 stack.enter_context(
-                    patch("app.engine.request_tier.Request", patches["Request"])
+                    patch("botasaurus.request.Request", patches["Request"])
                 )
             return engine.execute(payload, request_id=request_id, progress=progress)