fix: validate 598/599 as genuine errors before ConnectError, bump to v2.47.3

Only classify 598/599 as ConnectError when the page is empty or has an
error_status set. If a real server spoofs 598/599 with content, it falls
through to ServerError instead. Added 8 tests covering all status branches.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: j-mendez

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "spider"
-version = "2.47.2"
+version = "2.47.3"
 authors = ["j-mendez <jeff@spider.cloud>"]
 description = "A web crawler and scraper, building blocks for data curation workloads."
 repository = "https://github.com/spider-rs/spider"
@@ -120,11 +120,11 @@ features = ["serde", "headers", "dynamic-versions"]
 
 [dependencies.spider_agent_types]
 path = "../spider_agent_types"
-version = "2.47.2"
+version = "2.47.3"
 
 [dependencies.spider_agent]
 path = "../spider_agent"
-version = "2.47.2"
+version = "2.47.3"
 optional = true
 default-features = false
 

spider/Cargo.toml

@@ -2688,17 +2688,19 @@ pub async fn fetch_page_html_chrome_base(
                         let next_event = async { listener.next().await };
 
                         let event = match chunk_idle {
-                            Some(timeout) => match tokio::time::timeout(timeout, next_event).await {
-                                Ok(Some(event)) => event,
-                                Ok(None) => break,
-                                Err(_elapsed) => {
-                                    log::warn!(
+                            Some(timeout) => {
+                                match tokio::time::timeout(timeout, next_event).await {
+                                    Ok(Some(event)) => event,
+                                    Ok(None) => break,
+                                    Err(_elapsed) => {
+                                        log::warn!(
                                         "chrome network idle timeout ({timeout:?}), force-stopping page"
                                     );
-                                    let _ = page_clone.force_stop_all().await;
-                                    break;
+                                        let _ = page_clone.force_stop_all().await;
+                                        break;
+                                    }
                                 }
-                            },
+                            }
                             None => match next_event.await {
                                 Some(event) => event,
                                 None => break,

spider/src/utils/mod.rs

@@ -2461,8 +2461,9 @@ impl Website {
             self.status = CrawlStatus::Blocked;
         } else if page.status_code == reqwest::StatusCode::TOO_MANY_REQUESTS {
             self.status = CrawlStatus::RateLimited;
-        } else if page.status_code == *UNKNOWN_STATUS_ERROR
-            || page.status_code == *CHROME_UNKNOWN_STATUS_ERROR
+        } else if (page.status_code == *UNKNOWN_STATUS_ERROR
+            || page.status_code == *CHROME_UNKNOWN_STATUS_ERROR)
+            && (page.is_empty() || page.error_status.is_some())
         {
             self.status = CrawlStatus::ConnectError;
         } else if page.status_code.is_server_error() {
@@ -10990,6 +10991,96 @@ mod tests {
             "Should not build rotator with no proxies"
         );
     }
+
+    fn make_page(status: reqwest::StatusCode) -> crate::page::Page {
+        let mut page = crate::page::Page::default();
+        page.status_code = status;
+        page
+    }
+
+    #[test]
+    fn test_crawl_status_599_empty_page_is_connect_error() {
+        let mut website = crate::website::Website::new("http://example.com");
+        let page = make_page(*crate::page::UNKNOWN_STATUS_ERROR);
+        let links = hashbrown::HashSet::new();
+        website.set_crawl_initial_status(&page, &links);
+        assert_eq!(*website.get_status(), super::CrawlStatus::ConnectError);
+    }
+
+    #[test]
+    fn test_crawl_status_598_empty_page_is_connect_error() {
+        let mut website = crate::website::Website::new("http://example.com");
+        let page = make_page(*crate::page::CHROME_UNKNOWN_STATUS_ERROR);
+        let links = hashbrown::HashSet::new();
+        website.set_crawl_initial_status(&page, &links);
+        assert_eq!(*website.get_status(), super::CrawlStatus::ConnectError);
+    }
+
+    #[test]
+    fn test_crawl_status_598_with_error_status_is_connect_error() {
+        let mut website = crate::website::Website::new("http://example.com");
+        let mut page = make_page(*crate::page::CHROME_UNKNOWN_STATUS_ERROR);
+        page.html = Some(b"<html><body>some content</body></html>".to_vec().into());
+        page.error_status = Some("Invalid proxy configuration.".into());
+        let links = hashbrown::HashSet::new();
+        website.set_crawl_initial_status(&page, &links);
+        assert_eq!(*website.get_status(), super::CrawlStatus::ConnectError);
+    }
+
+    #[test]
+    fn test_crawl_status_598_with_content_no_error_is_server_error() {
+        let mut website = crate::website::Website::new("http://example.com");
+        let mut page = make_page(*crate::page::CHROME_UNKNOWN_STATUS_ERROR);
+        page.html = Some(
+            b"<html><body>real server content</body></html>"
+                .to_vec()
+                .into(),
+        );
+        let links = hashbrown::HashSet::new();
+        website.set_crawl_initial_status(&page, &links);
+        assert_eq!(*website.get_status(), super::CrawlStatus::ServerError);
+    }
+
+    #[test]
+    fn test_crawl_status_599_with_content_no_error_is_server_error() {
+        let mut website = crate::website::Website::new("http://example.com");
+        let mut page = make_page(*crate::page::UNKNOWN_STATUS_ERROR);
+        page.html = Some(
+            b"<html><body>real server content</body></html>"
+                .to_vec()
+                .into(),
+        );
+        let links = hashbrown::HashSet::new();
+        website.set_crawl_initial_status(&page, &links);
+        assert_eq!(*website.get_status(), super::CrawlStatus::ServerError);
+    }
+
+    #[test]
+    fn test_crawl_status_500_is_server_error() {
+        let mut website = crate::website::Website::new("http://example.com");
+        let page = make_page(reqwest::StatusCode::INTERNAL_SERVER_ERROR);
+        let links = hashbrown::HashSet::new();
+        website.set_crawl_initial_status(&page, &links);
+        assert_eq!(*website.get_status(), super::CrawlStatus::ServerError);
+    }
+
+    #[test]
+    fn test_crawl_status_429_is_rate_limited() {
+        let mut website = crate::website::Website::new("http://example.com");
+        let page = make_page(reqwest::StatusCode::TOO_MANY_REQUESTS);
+        let links = hashbrown::HashSet::new();
+        website.set_crawl_initial_status(&page, &links);
+        assert_eq!(*website.get_status(), super::CrawlStatus::RateLimited);
+    }
+
+    #[test]
+    fn test_crawl_status_empty_page_200_is_empty() {
+        let mut website = crate::website::Website::new("http://example.com");
+        let page = make_page(reqwest::StatusCode::OK);
+        let links = hashbrown::HashSet::new();
+        website.set_crawl_initial_status(&page, &links);
+        assert_eq!(*website.get_status(), super::CrawlStatus::Empty);
+    }
 }
 
 #[tokio::test]

spider/src/website.rs

@@ -1,6 +1,6 @@
 [package]
 name = "spider_agent"
-version = "2.47.2"
+version = "2.47.3"
 authors = ["j-mendez <jeff@spider.cloud>"]
 description = "A concurrent-safe multimodal agent for web automation and research."
 repository = "https://github.com/spider-rs/spider"
@@ -28,8 +28,8 @@ parking_lot = "0.12"
 base64 = "0.22"
 
 # Extracted types and HTML processing
-spider_agent_types = { version = "2.47.2", path = "../spider_agent_types" }
-spider_agent_html = { version = "2.47.2", path = "../spider_agent_html" }
+spider_agent_types = { version = "2.47.3", path = "../spider_agent_types" }
+spider_agent_html = { version = "2.47.3", path = "../spider_agent_html" }
 
 # HTML processing (still needed for engine internals)
 lol_html = "2"

spider_agent/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_agent_html"
-version = "2.47.2"
+version = "2.47.3"
 authors = ["j-mendez <jeff@spider.cloud>"]
 description = "HTML processing utilities for spider_agent — cleaning, content analysis, and diffing."
 repository = "https://github.com/spider-rs/spider"
@@ -24,7 +24,7 @@ serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 
 # Types from our types crate
-spider_agent_types = { version = "2.47.2", path = "../spider_agent_types" }
+spider_agent_types = { version = "2.47.3", path = "../spider_agent_types" }
 
 [dev-dependencies]
 serde_json = "1"

spider_agent_html/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_agent_types"
-version = "2.47.2"
+version = "2.47.3"
 authors = ["j-mendez <jeff@spider.cloud>"]
 description = "Pure data types and constants for spider_agent automation. Zero heavy dependencies."
 repository = "https://github.com/spider-rs/spider"

spider_agent_types/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_cli"
-version = "2.47.2"
+version = "2.47.3"
 authors = ["j-mendez <jeff@spider.cloud>"]
 description = "The fastest web crawler CLI written in Rust."
 repository = "https://github.com/spider-rs/spider"

spider_cli/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_utils"
-version = "2.47.2"
+version = "2.47.3"
 authors = ["j-mendez <jeff@spider.cloud>"]
 description = "Utilities to use for Spider Web Crawler."
 repository = "https://github.com/spider-rs/spider"

spider_utils/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "spider_worker"
-version = "2.47.2"
+version = "2.47.3"
 authors = ["j-mendez <jeff@spider.cloud>"]
 description = "The fastest web crawler as a worker or proxy."
 repository = "https://github.com/spider-rs/spider"