insert in batch and nitpicks

Author: CodeWarriorr

.sqlx/query-600ebc4def7b6cc0fc0ebe17c567ace4e5bb4e7c6f959bcae80304029753d11d.json

@@ -47,7 +47,7 @@ migrate-status:
 	@sqlx migrate info --source migrations --database-url $(DATABASE_URL)
 
 # Init dev environment
-init-dev: run-db 
+init-dev: run-db
 	@until pg_isready -h localhost -p 5434 -U pguser > /dev/null 2>&1; do sleep 1; done
 	@$(MAKE) migrate-up init-dev-db
 # Init dev db (creates DMOB table and seeds data)

Makefile

@@ -1,3 +1,3 @@
--- Remove update_at trigger
+-- Remove updated_at trigger
 DROP TRIGGER IF EXISTS update_storage_providers_updated_at ON storage_providers;
 DROP FUNCTION IF EXISTS update_updated_at_column();

migrations/20251106110627_create_update_at_trigger.down.sql

@@ -41,4 +41,4 @@ uuid = { version = "1.10.0", features = ["v4", "serde"] }
 chrono = { version = "0.4.38", features = ["serde"] }
 regex = "1.11.1"
 moka = { version = "0.12.10", default-features = false, features = ["future"] }
-dotenvy = "0.15.7" 
+dotenvy = "0.15.7"

url_finder/Cargo.toml

@@ -36,15 +36,8 @@ async fn discover_and_sync_providers(
 
     debug!("Found {} distinct providers in dmob", providers.len());
 
-    let mut processed = 0;
-    for provider in &providers {
-        match sp_repo.insert_if_not_exists(provider).await {
-            Ok(_) => processed += 1,
-            Err(e) => error!("Failed to insert provider {}: {:?}", provider, e),
-        }
+    match sp_repo.insert_batch_if_not_exists(&providers).await {
+        Ok(count) => Ok(count),
+        Err(e) => Err(e),
     }
-
-    debug!("Processed {} providers", processed);
-
-    Ok(providers.len())
 }

url_finder/src/background/provider_discovery.rs

@@ -105,7 +105,7 @@ async fn main() -> Result<()> {
         cache,
     });
 
-    // Start the provider discovery the background
+    // Start the provider discovery in the background
     tokio::spawn({
         let sp_repo = sp_repo.clone();
         let deal_repo = deal_repo.clone();

url_finder/src/main.rs

@@ -30,6 +30,7 @@ impl StorageProviderRepository {
         Self { pool }
     }
 
+    #[allow(dead_code)]
     pub async fn insert_if_not_exists(&self, provider_id: &str) -> Result<()> {
         sqlx::query!(
             r#"INSERT INTO
@@ -45,6 +46,26 @@ impl StorageProviderRepository {
         Ok(())
     }
 
+    pub async fn insert_batch_if_not_exists(&self, provider_ids: &[String]) -> Result<usize> {
+        if provider_ids.is_empty() {
+            return Ok(0);
+        }
+
+        let result = sqlx::query!(
+            r#"INSERT INTO
+                    storage_providers (provider_id)
+               SELECT
+                    UNNEST($1::text[])
+               ON CONFLICT DO NOTHING
+            "#,
+            provider_ids
+        )
+        .execute(&self.pool)
+        .await?;
+
+        Ok(result.rows_affected() as usize)
+    }
+
     pub async fn get_by_provider_id(&self, provider_id: &str) -> Result<Option<StorageProvider>> {
         Ok(sqlx::query_as!(
             StorageProvider,