wip

Author: vincent-herlemont

Cargo.toml

@@ -27,6 +27,8 @@ semver = "1"
 tokio = { version = "1.40.0", features = ["sync"], optional = true }
 # TODO: channels with futures
 # TODO: channels crossbeam
+jemallocator = "0.5.4"
+
 
 [dev-dependencies]
 assert_fs = "1.1.2"
@@ -49,6 +51,7 @@ cc = "1.1.15"
 rusqlite = { version = "0.32.1", features = ["bundled"] }
 concat-idents = "1.1.5"
 
+
 [features]
 default = [ "upgrade_0_5_x", "upgrade_0_7_x" ]
 upgrade_0_5_x = [ "redb1" ]
@@ -63,4 +66,4 @@ name = "seconday_index"
 harness = false
 
 [build-dependencies]
-skeptic = "0.13.7"
+skeptic = "0.13.7"

benches/seconday_index.rs

@@ -11,6 +11,9 @@ use criterion::{
 use native_model::{native_model, Model};
 use rand::Rng;
 
+// #[global_allocator]
+// static GLOBAL: jemallocator::Jemalloc = jemallocator::Jemalloc;
+
 fn bench_insert<T: Default + Item + native_db::ToInput>(c: &mut Criterion, item_name: &str) {
     let mut group = c.benchmark_group(format!("insert_{}", item_name));
     group.plot_config(
@@ -149,41 +152,41 @@ fn bench_select<T: Default + Item + native_db::ToInput + Clone + Debug>(
             let start = std::time::Instant::now();
             let native_db = native_db.r_transaction().unwrap();
             for _ in 0..iters {
-                let from_sk = 1;
-                let to_sk = 100;
-                let items: Vec<T> = native_db.scan().secondary(Item1SK_NUni_NOptKey::sk_1).unwrap().range(from_sk..).unwrap().try_collect().unwrap();
-                println!("from_sk: {:?}, to_sk: {:?}, len: {:?}", from_sk, to_sk, items.len());
+                let from_sk: i64 = rand::thread_rng().gen_range(0..50);
+                let to_sk: i64 = rand::thread_rng().gen_range(50..100);
+                let _items: Vec<T> = native_db.scan().secondary(Item1SK_NUni_NOptKey::sk_1).unwrap().range(from_sk..to_sk).unwrap().try_collect().unwrap();
+                // println!("len: {:?}", _items.len());
             }
             start.elapsed()
         })
     });
 
 
-    // group.bench_function(BenchmarkId::new("random range", "Sqlite"), |b| {
-    //     b.iter_custom(|iters| {
-    //         let sqlite = SqliteBenchDatabase::setup();
-    //         sqlite.insert_bulk_random::<T>(NUMBER_OF_ITEMS);
-    //         let start = std::time::Instant::now();
-    //         for _ in 0..iters {
-    //             let from_sk = rand::thread_rng().gen_range(0..50);
-    //             let to_sk = rand::thread_rng().gen_range(50..100);
-    //             let mut db = sqlite.db().borrow_mut();
-    //             let transaction = db
-    //                 .transaction_with_behavior(TransactionBehavior::Immediate)
-    //                 .unwrap();
-    //             let sql = T::generate_select_range_sk(&"sk_1");
-    //             let mut stmt = transaction.prepare_cached(&sql).unwrap();
-    //             let rows = stmt.query_map(&[(":from_sk", &from_sk), (":to_sk", &to_sk)], |row| {
-    //                 let binary: Vec<u8> = row.get(1)?;
-    //                 let item = T::native_db_bincode_decode_from_slice(&binary).unwrap();
-    //                 Ok(item)
-    //             });
-    //             let out = rows.unwrap().map(|r| r.unwrap()).collect::<Vec<T>>();
-    //             println!("len: {:?}", out.len());
-    //         }
-    //         start.elapsed()
-    //     });
-    // });
+    group.bench_function(BenchmarkId::new("random range", "Sqlite"), |b| {
+        b.iter_custom(|iters| {
+            let sqlite = SqliteBenchDatabase::setup();
+            sqlite.insert_bulk_random::<T>(NUMBER_OF_ITEMS);
+            let start = std::time::Instant::now();
+            for _ in 0..iters {
+                let from_sk: i64 = rand::thread_rng().gen_range(0..50);
+                let to_sk: i64 = rand::thread_rng().gen_range(50..100);
+                let mut db = sqlite.db().borrow_mut();
+                let transaction = db
+                    .transaction_with_behavior(TransactionBehavior::Immediate)
+                    .unwrap();
+                let sql = T::generate_select_range_sk(&"sk_1");
+                let mut stmt = transaction.prepare_cached(&sql).unwrap();
+                let rows = stmt.query_map(&[(":from_sk", &from_sk), (":to_sk", &to_sk)], |row| {
+                    let binary: Vec<u8> = row.get(1)?;
+                    let item = T::native_db_bincode_decode_from_slice(&binary).unwrap();
+                    Ok(item)
+                });
+                let _out = rows.unwrap().map(|r| r.unwrap()).collect::<Vec<T>>();
+                // println!("len: {:?}", _out.len());
+            }
+            start.elapsed()
+        });
+    });
 }
 
 fn first_compare(c: &mut Criterion) {

benches/setup.rs

@@ -200,7 +200,7 @@ impl BenchDatabase for NativeDBBenchDatabase {
     fn setup() -> Self {
         let tmp = TmpFs::new().unwrap();
         let db_path = tmp.path("native_db_bench");
-        let db = Builder::new().create(&MODELS, db_path.clone()).unwrap();
+        let db = Builder::new().set_cache_size(1024 * 1024 * 1024).create(&MODELS, db_path.clone()).unwrap();
         Self { tmp, db }
     }
 

justfile

@@ -74,7 +74,7 @@ bench_build:
     cargo bench --no-run
 
 bench bench_name:
-    CRITERION_DEBUG=1 cargo bench --bench {{bench_name}}; \
+    CRITERION_DEBUG=1 cargo bench --profile release --bench {{bench_name}}; \
     start ./target/criterion/report/index.html
 
 expand test_file_name="util":