upgrade nightly, upgrade rusqlite to use sqlite3_trace_v2 and log slow queries, upgrade cr-sqlite fixing segfault to upgrade from 0.16

Author: jeromegn

Cargo.lock

@@ -52,7 +52,7 @@ rand = { version = "0.8.5", features = ["small_rng"] }
 rangemap = { version = "1.5.1", features = ["serde1"] }
 rcgen = { version = "0.11.1", features = ["x509-parser"] }
 rhai = { version = "1.15.1", features = ["sync"] }
-rusqlite = { version = "0.30.0", features = ["serde_json", "time", "bundled", "uuid", "array", "load_extension", "column_decltype", "vtab", "functions", "chrono", "series", "trace"] }
+rusqlite = { version = "0.32.1", features = ["serde_json", "time", "bundled", "uuid", "array", "load_extension", "column_decltype", "vtab", "functions", "chrono", "series", "trace"], git = "https://github.com/rusqlite/rusqlite", rev = "c9f77814ddc99825f9c8eb52a0acb8d00f447f17" }
 rustls = { version = "0.21.0", features = ["dangerous_configuration", "quic"] }
 rustls-pemfile = "1.0.2"
 seahash = "4.1.0"

Cargo.toml

@@ -906,7 +906,7 @@ pub async fn handle_sync(
 
         debug!("found {} candidates to synchronize with", candidates.len());
 
-        let desired_count = cmp::max(cmp::min(candidates.len() / 100, 10), 3);
+        let desired_count = (candidates.len() / 100).clamp(3, 10);
         debug!("Selected {desired_count} nodes to sync with");
 
         let mut rng = StdRng::from_entropy();

crates/corro-agent/src/agent/handlers.rs

@@ -462,7 +462,9 @@ async fn execute_schema(agent: &Agent, statements: Vec<String>) -> eyre::Result<
 
     new_schema.constrain()?;
 
-    block_in_place(|| {
+    // conn.trace(Some(|sql| debug!(sql)));
+
+    let apply_res = block_in_place(|| {
         let tx = conn.immediate_transaction()?;
 
         apply_schema(&tx, &schema_write, &mut new_schema)?;
@@ -480,7 +482,11 @@ async fn execute_schema(agent: &Agent, statements: Vec<String>) -> eyre::Result<
         agent.pool().drain_read();
 
         Ok::<_, eyre::Report>(())
-    })?;
+    });
+
+    // conn.trace(None);
+
+    apply_res?;
 
     *schema_write = new_schema;
 

crates/corro-agent/src/api/public/mod.rs

@@ -361,7 +361,9 @@ pub fn apply_schema(
                 schema_to_merge.tables.insert(name.clone(), parsed_table);
             }
 
+            debug!("selecting crsql_as_crr");
             tx.execute_batch(&format!("SELECT crsql_as_crr('{name}'); CREATE INDEX IF NOT EXISTS corro_{name}__crsql_clock_site_id_dbv ON {name}__crsql_clock (site_id, db_version);"))?;
+            debug!("done selecting as crr");
 
             if schema_to_merge.tables.contains_key(name) {
                 // just merged!

crates/corro-types/crsqlite-darwin-aarch64.dylib

@@ -1,13 +1,13 @@
 use std::{
     ops::{Deref, DerefMut},
-    time::Instant,
+    time::{Duration, Instant},
 };
 
 use once_cell::sync::Lazy;
-use rusqlite::{params, Connection, Transaction};
+use rusqlite::{params, trace::TraceEventCodes, Connection, Transaction};
 use sqlite_pool::SqliteConn;
 use tempfile::TempDir;
-use tracing::{error, info, trace};
+use tracing::{error, info, trace, warn};
 
 pub type SqlitePool = sqlite_pool::Pool<CrConn>;
 pub type SqlitePoolError = sqlite_pool::PoolError;
@@ -42,7 +42,19 @@ pub fn rusqlite_to_crsqlite(mut conn: rusqlite::Connection) -> rusqlite::Result<
     init_cr_conn(&mut conn)?;
     setup_conn(&conn)?;
     sqlite_functions::add_to_connection(&conn)?;
-    conn.trace(Some(|sql| trace!("{sql}")));
+
+    const SLOW_THRESHOLD: Duration = Duration::from_secs(1);
+    conn.trace_v2(
+        TraceEventCodes::SQLITE_TRACE_PROFILE,
+        Some(|event| {
+            if let rusqlite::trace::TraceEvent::Profile(stmt_ref, duration) = event {
+                if duration >= SLOW_THRESHOLD {
+                    warn!("SLOW query {duration:?} => {}", stmt_ref.sql());
+                }
+            }
+        }),
+    );
+
     Ok(CrConn(conn))
 }