From 8b31d10ab90fd34045626948f58d503647e93f8d Mon Sep 17 00:00:00 2001
From: Timon Vonk <mail@timonv.nl>
Date: Fri, 14 Feb 2025 16:55:05 +0100
Subject: [PATCH] fix(integration): Enable `scrape` again on spider

---
 Cargo.lock                                   | 8 ++++----
 Cargo.toml                                   | 2 +-
 swiftide-integrations/src/scraping/loader.rs | 4 +---
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 50949cfd..1f886284 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -6668,7 +6668,7 @@ version = "0.13.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
 dependencies = [
- "heck 0.5.0",
+ "heck 0.4.1",
  "itertools 0.14.0",
  "log",
  "multimap",
@@ -7899,7 +7899,7 @@ version = "0.8.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "03c3c6b7927ffe7ecaa769ee0e3994da3b8cafc8f444578982c83ecb161af917"
 dependencies = [
- "heck 0.5.0",
+ "heck 0.4.1",
  "proc-macro2",
  "quote",
  "syn 2.0.98",
@@ -7944,9 +7944,9 @@ dependencies = [
 
 [[package]]
 name = "spider"
-version = "2.27.62"
+version = "2.27.63"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a89064a268c22a86833ce57105b03e0771447fb29870f5e6311fb198cb8c7f82"
+checksum = "163cb05e3fcc1144d0de3b404e2068934cefa857afb0a1a3f874a70adeff9d53"
 dependencies = [
  "ahash",
  "aho-corasick",
diff --git a/Cargo.toml b/Cargo.toml
index 10ceca0b..ae8fc963 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -43,7 +43,7 @@ dyn-clone = { version = "1.0" }
 convert_case = "0.7.1"
 
 # Integrations
-spider = { version = "2.27" }
+spider = { version = "2.27.62" }
 async-openai = { version = "0.27.1" }
 qdrant-client = { version = "1.13", default-features = false, features = [
   "serde",
diff --git a/swiftide-integrations/src/scraping/loader.rs b/swiftide-integrations/src/scraping/loader.rs
index 1fc63270..250a6749 100644
--- a/swiftide-integrations/src/scraping/loader.rs
+++ b/swiftide-integrations/src/scraping/loader.rs
@@ -66,9 +66,7 @@ impl Loader for ScrapingLoader {
 
         let _scrape_thread = tokio::spawn(async move {
             tracing::info!("[Spider] Starting scrape loop");
-            // TODO: It would be much nicer if this used `scrape` instead, as it is supposedly
-            // more concurrent
-            spider_website.crawl().await;
+            spider_website.scrape().await;
             tracing::info!("[Spider] Scrape loop finished");
         });