kernel: fix spider remote-crawl hang (chrome_intercept) + chromey default

The spider::Website driver hung >60s on every remote-browser crawl because
the gottem-adapters-chrome spider dep lacked the `chrome_intercept` feature.
That feature gates the CDP Network/Fetch domain setup spider uses to detect
page completion; without it compiled in, crawl() falls back to the 120s
request_timeout. Verified against a live Kernel session (examples/
kernel_spider_probe): with the feature, example.com renders in ~1.4s (both
intercept on and off); without it, it times out. The runtime
with_chrome_intercept() call is not required — compiling the feature is.

Also flip the Kernel driver default to raw chromey (drive_cdp) — lean,
fast, direct CDP — with spider::Website now an opt-in per request via
`provider_options.kernel.spider = true` (control flag stripped from the
Kernel create payload).

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: j-mendez

crates/gottem-adapters-chrome/Cargo.toml

@@ -24,7 +24,10 @@ gottem-core  = { path = "../gottem-core", version = "0.1.16" }
 # `Ws(Url(TlsFeatureNotEnabled))`. Cargo unifies chromey's features across the
 # build, so this also covers spider::Website's remote-connection path.
 chromey      = { version = "2", features = ["chrome_tls_connection"] }
-spider       = { workspace = true, features = ["basic", "chrome"] }
+# `chrome_intercept` sets up the CDP Network/Fetch domain spider uses to detect
+# page completion; without it a remote-browser crawl waits on the full
+# request_timeout. Matches spider's own remote-chrome examples.
+spider       = { workspace = true, features = ["basic", "chrome", "chrome_intercept"] }
 async-trait  = { workspace = true }
 bytes        = { workspace = true }
 url          = { workspace = true }

crates/gottem-adapters-chrome/examples/kernel_spider_probe.rs

@@ -0,0 +1,55 @@
+//! Local probe: drive a remote (Kernel) browser via spider::Website over CDP and
+//! time it, to validate the chrome_intercept fix for the slow-crawl hang.
+//!
+//!   cargo run -p gottem-adapters-chrome --example kernel_spider_probe -- <cdp_ws_url> [url]
+//!
+//! Env: INTERCEPT=0 disables chrome_intercept (to reproduce the slow path).
+//!      RUST_LOG=spider=info for spider's own logs.
+
+use spider::features::chrome_common::RequestInterceptConfiguration;
+use spider::tokio;
+use spider::website::Website;
+use std::time::Instant;
+
+#[tokio::main]
+async fn main() {
+    let args: Vec<String> = std::env::args().collect();
+    let cdp = args.get(1).expect("usage: <cdp_ws_url> [url]").clone();
+    let url = args
+        .get(2)
+        .cloned()
+        .unwrap_or_else(|| "https://example.com".to_string());
+    let intercept = std::env::var("INTERCEPT").map(|v| v != "0").unwrap_or(true);
+    eprintln!("intercept={intercept} url={url}");
+
+    let mut b = Website::new(&url);
+    b.with_limit(1)
+        .with_stealth(false)
+        .with_fingerprint(false)
+        .with_chrome_connection(Some(cdp));
+    if intercept {
+        b.with_chrome_intercept(RequestInterceptConfiguration::new(true));
+    }
+    let mut website = b.build().expect("build");
+
+    let mut rx = website.subscribe(16);
+    let t = Instant::now();
+    website.crawl().await;
+    let elapsed = t.elapsed();
+
+    let mut n = 0;
+    let mut bytes = 0usize;
+    let mut status = 0u16;
+    loop {
+        match rx.try_recv() {
+            Ok(p) => {
+                n += 1;
+                bytes = p.get_html_bytes_u8().len();
+                status = p.status_code.as_u16();
+            }
+            Err(spider::tokio::sync::broadcast::error::TryRecvError::Lagged(_)) => continue,
+            Err(_) => break,
+        }
+    }
+    eprintln!("DONE elapsed={elapsed:?} pages={n} status={status} html_bytes={bytes}");
+}

crates/gottem-adapters-chrome/src/lib.rs

@@ -12,8 +12,8 @@
 //!
 //! [`KernelCdpAdapter`] covers Kernel ([onkernel.com](https://kernel.sh)), which
 //! has no *static* endpoint: it `POST`s `/browsers` to mint a per-session
-//! `cdp_ws_url`, drives it (via `spider::Website` by default, or raw chromey
-//! [`drive_cdp`] when `provider_options.kernel.spider_disabled` is set), then
+//! `cdp_ws_url`, drives it (raw chromey [`drive_cdp`] by default, or
+//! `spider::Website` when `provider_options.kernel.spider = true`), then
 //! `DELETE`s the session (Kernel bills per running second). It reports
 //! [`AdapterKind::Custom`]`("kernel_cdp")`.
 //!
@@ -286,10 +286,11 @@ const KERNEL_DEFAULT_CREATE_URL: &str = "https://api.onkernel.com/browsers";
 /// *running* second, so releasing promptly is a cost guarantee, not just hygiene.
 ///
 /// Two drivers, switchable per request:
-/// - **default** — [`spider::Website`] via [`scrape_via_spider`]: crawl() +
-///   subscription (streams pages to the user), stealth/fingerprint off.
-/// - **`provider_options.kernel.spider_disabled = true`** — raw chromey
-///   ([`drive_cdp`]): direct CDP, more control for bespoke automation.
+/// - **default** — raw chromey ([`drive_cdp`]): direct CDP, lean and fast.
+/// - **`provider_options.kernel.spider = true`** — [`spider::Website`] via
+///   [`scrape_via_spider`]: crawl() + subscription (streams pages to the user),
+///   stealth/fingerprint off. Requires spider's `chrome_intercept` feature for
+///   page-completion detection over the remote browser.
 ///
 /// Browser config is tunable per request via `provider_options.kernel` (e.g.
 /// `{ "headless": false, "gpu": true, "proxy_id": "...", "viewport": {...} }`),
@@ -397,12 +398,13 @@ impl Adapter for KernelCdpAdapter {
         )
         .await?;
 
-        // ---- 2. Drive the remote browser. Default path is spider::Website —
-        // its crawl() + subscription is how rendered pages stream back, and it
-        // carries the quality steps. Opt out per request with
-        // `provider_options.kernel.spider_disabled = true` to drive raw chromey
-        // directly (more control for bespoke automation; no spider wrapper).
-        let driven = if kernel_spider_disabled(req) {
+        // ---- 2. Drive the remote browser. Default is raw chromey ([`drive_cdp`])
+        // — direct CDP, lean and fast. Opt into the spider::Website path
+        // (crawl() + subscription, quality steps) per request with
+        // `provider_options.kernel.spider = true`.
+        let driven = if kernel_use_spider(req) {
+            scrape_via_spider(&session.cdp_ws_url, req.url.as_str(), route.timeout(), cancel).await
+        } else {
             drive_cdp(
                 &session.cdp_ws_url,
                 req.url.as_str(),
@@ -411,8 +413,6 @@ impl Adapter for KernelCdpAdapter {
                 cancel,
             )
             .await
-        } else {
-            scrape_via_spider(&session.cdp_ws_url, req.url.as_str(), route.timeout(), cancel).await
         };
 
         // ---- 3. On success, read Kernel's own usage meter for actual-cost
@@ -479,7 +479,7 @@ fn kernel_cost_dollars(uptime_ms: u64, headless: bool, gpu: bool) -> f64 {
 
 /// Keys the adapter consumes from `provider_options.kernel` as its own control
 /// flags — they must NOT be forwarded to Kernel's create-browser API.
-const KERNEL_CONTROL_KEYS: &[&str] = &["spider_disabled"];
+const KERNEL_CONTROL_KEYS: &[&str] = &["spider"];
 
 /// Default create body (cheap + stealthy), with `provider_options.kernel`
 /// layered over the top so callers can flip `headless`, request `gpu`, pin a
@@ -501,12 +501,12 @@ fn build_kernel_create_body(req: &ScrapeRequest) -> serde_json::Value {
     body
 }
 
-/// Whether this request opted out of the spider driver (`provider_options.kernel
-/// .spider_disabled = true`) to drive raw chromey directly instead.
-fn kernel_spider_disabled(req: &ScrapeRequest) -> bool {
+/// Whether this request opted into the spider::Website driver
+/// (`provider_options.kernel.spider = true`). Default is raw chromey.
+fn kernel_use_spider(req: &ScrapeRequest) -> bool {
     req.provider_options
         .get("kernel")
-        .and_then(|o| o.get("spider_disabled"))
+        .and_then(|o| o.get("spider"))
         .and_then(|v| v.as_bool())
         .unwrap_or(false)
 }
@@ -803,25 +803,25 @@ mod tests {
         let mut req = ScrapeRequest::get(Url::parse("https://example.com/").unwrap());
         req.provider_options.insert(
             "kernel".to_string(),
-            serde_json::json!({ "spider_disabled": true, "headless": false }),
+            serde_json::json!({ "spider": true, "headless": false }),
         );
         let body = build_kernel_create_body(&req);
         assert_eq!(body["headless"], serde_json::json!(false)); // real Kernel field passes
-        assert!(body.get("spider_disabled").is_none()); // control flag stripped
+        assert!(body.get("spider").is_none()); // control flag stripped
     }
 
     #[test]
-    fn kernel_spider_disabled_reads_flag() {
+    fn kernel_use_spider_reads_flag() {
         let base = || ScrapeRequest::get(Url::parse("https://example.com/").unwrap());
-        assert!(!kernel_spider_disabled(&base())); // absent → spider (default)
+        assert!(!kernel_use_spider(&base())); // absent → chromey (default)
         let mut on = base();
         on.provider_options
-            .insert("kernel".into(), serde_json::json!({ "spider_disabled": true }));
-        assert!(kernel_spider_disabled(&on));
+            .insert("kernel".into(), serde_json::json!({ "spider": true }));
+        assert!(kernel_use_spider(&on));
         let mut off = base();
         off.provider_options
-            .insert("kernel".into(), serde_json::json!({ "spider_disabled": false }));
-        assert!(!kernel_spider_disabled(&off));
+            .insert("kernel".into(), serde_json::json!({ "spider": false }));
+        assert!(!kernel_use_spider(&off));
     }
 
     #[test]