Merge pull request #108 from fjall-rs/2.5.0

marvin-j97 · web-flow · commit fa09be928272 · 2025-01-08T19:26:22.000+01:00
2.5.0
diff --git a/Cargo.toml b/Cargo.toml
@@ -2,7 +2,7 @@
 name = "fjall"
 description = "LSM-based key-value storage engine"
 license = "MIT OR Apache-2.0"
-version = "2.4.4"
+version = "2.5.0"
 edition = "2021"
 rust-version = "1.74.0"
 readme = "README.md"
@@ -28,7 +28,7 @@ bytes = ["lsm-tree/bytes"]
 
 [dependencies]
 byteorder = "1.5.0"
-lsm-tree = { version = "2.4.0", default-features = false }
+lsm-tree = { version = "2.5.0", default-features = false }
 log = "0.4.21"
 std-semaphore = "0.1.0"
 tempfile = "3.10.1"
diff --git a/examples/permuterm/src/main.rs b/examples/permuterm/src/main.rs
@@ -80,7 +80,7 @@ fn main() -> Result<()> {
 
     for word in WORDS {
         for term in permuterm(word) {
-            db.insert(format!("{term}#{word}"), word).unwrap();
+            db.insert(format!("{term}#{word}"), *word).unwrap();
         }
     }
 
diff --git a/examples/tx-blob-cas/src/main.rs b/examples/tx-blob-cas/src/main.rs
@@ -125,7 +125,7 @@ impl Cas {
             let rc = self.decrement_ref_count(&mut tx, &prev_content_hash)?;
             if rc == 0 {
                 // No more references
-                tx.remove(&self.blobs, &prev_content_hash);
+                tx.remove(&self.blobs, prev_content_hash);
             }
         }
 
@@ -156,7 +156,7 @@ impl Cas {
             let rc = self.decrement_ref_count(&mut tx, &content_hash)?;
             if rc == 0 {
                 // No more references
-                tx.remove(&self.blobs, &content_hash);
+                tx.remove(&self.blobs, content_hash);
             }
 
             tx.commit()?;
diff --git a/examples/tx-mpmc-queue/src/main.rs b/examples/tx-mpmc-queue/src/main.rs
@@ -63,11 +63,12 @@ fn main() -> fjall::Result<()> {
                     // Something like SingleDelete https://github.com/facebook/rocksdb/wiki/Single-Delete
                     // would be good for this type of workload
                     if let Some((key, _)) = tx.first_key_value(&tasks)? {
-                        tx.remove(&tasks, &key);
+                        let task_id = std::str::from_utf8(&key).unwrap().to_owned();
+
+                        tx.remove(&tasks, key);
 
                         tx.commit()?;
 
-                        let task_id = std::str::from_utf8(&key).unwrap();
                         println!("consumer {idx} completed task {task_id}");
 
                         counter.fetch_add(1, Relaxed);
diff --git a/examples/tx-partition-move/src/main.rs b/examples/tx-partition-move/src/main.rs
@@ -33,12 +33,13 @@ fn main() -> fjall::Result<()> {
                     // Something like SingleDelete https://github.com/facebook/rocksdb/wiki/Single-Delete
                     // would be good for this type of workload
                     if let Some((key, value)) = tx.first_key_value(&src)? {
-                        tx.remove(&src, &key);
-                        tx.insert(&dst, &key, &value);
+                        let task_id = std::str::from_utf8(&key).unwrap().to_owned();
+
+                        tx.remove(&src, key.clone());
+                        tx.insert(&dst, key, value);
 
                         tx.commit()?;
 
-                        let task_id = std::str::from_utf8(&key).unwrap();
                         println!("consumer {idx} moved {task_id}");
 
                         let ms = rng.gen_range(10..100);
diff --git a/examples/tx-ssi-mpmc-queue/src/main.rs b/examples/tx-ssi-mpmc-queue/src/main.rs
@@ -65,13 +65,14 @@ fn main() -> fjall::Result<()> {
                     // Something like SingleDelete https://github.com/facebook/rocksdb/wiki/Single-Delete
                     // would be good for this type of workload
                     if let Some((key, _)) = tx.first_key_value(&tasks)? {
-                        tx.remove(&tasks, &key);
+                        let task_id = std::str::from_utf8(&key).unwrap().to_owned();
+
+                        tx.remove(&tasks, key);
 
                         if tx.commit()?.is_ok() {
                             counter.fetch_add(1, Relaxed);
                         }
 
-                        let task_id = std::str::from_utf8(&key).unwrap();
                         println!("consumer {idx} completed task {task_id}");
 
                         let ms = rng.gen_range(50..200);
diff --git a/examples/tx-ssi-partition-move/src/main.rs b/examples/tx-ssi-partition-move/src/main.rs
@@ -33,12 +33,13 @@ fn main() -> fjall::Result<()> {
                     // Something like SingleDelete https://github.com/facebook/rocksdb/wiki/Single-Delete
                     // would be good for this type of workload
                     if let Some((key, value)) = tx.first_key_value(&src)? {
-                        tx.remove(&src, &key);
-                        tx.insert(&dst, &key, &value);
+                        let task_id = std::str::from_utf8(&key).unwrap().to_owned();
+
+                        tx.remove(&src, key.clone());
+                        tx.insert(&dst, key, value);
 
                         tx.commit()?.ok();
 
-                        let task_id = std::str::from_utf8(&key).unwrap();
                         println!("consumer {idx} moved {task_id}");
 
                         let ms = rng.gen_range(10..100);
diff --git a/src/batch/mod.rs b/src/batch/mod.rs
@@ -6,7 +6,7 @@ pub mod item;
 
 use crate::{Keyspace, PartitionHandle, PersistMode};
 use item::Item;
-use lsm_tree::{AbstractTree, ValueType};
+use lsm_tree::{AbstractTree, UserKey, UserValue, ValueType};
 use std::{collections::HashSet, sync::Arc};
 
 /// Partition key (a.k.a. column family, locality group)
@@ -52,28 +52,20 @@ impl Batch {
     }
 
     /// Inserts a key-value pair into the batch
-    pub fn insert<K: AsRef<[u8]>, V: AsRef<[u8]>>(
+    pub fn insert<K: Into<UserKey>, V: Into<UserValue>>(
         &mut self,
         p: &PartitionHandle,
         key: K,
         value: V,
     ) {
-        self.data.push(Item::new(
-            p.name.clone(),
-            key.as_ref(),
-            value.as_ref(),
-            ValueType::Value,
-        ));
+        self.data
+            .push(Item::new(p.name.clone(), key, value, ValueType::Value));
     }
 
     /// Adds a tombstone marker for a key
-    pub fn remove<K: AsRef<[u8]>>(&mut self, p: &PartitionHandle, key: K) {
-        self.data.push(Item::new(
-            p.name.clone(),
-            key.as_ref(),
-            vec![],
-            ValueType::Tombstone,
-        ));
+    pub fn remove<K: Into<UserKey>>(&mut self, p: &PartitionHandle, key: K) {
+        self.data
+            .push(Item::new(p.name.clone(), key, vec![], ValueType::Tombstone));
     }
 
     /// Commits the batch to the [`Keyspace`] atomically
diff --git a/src/compaction/worker.rs b/src/compaction/worker.rs
@@ -5,9 +5,14 @@
 use super::manager::CompactionManager;
 use crate::snapshot_tracker::SnapshotTracker;
 use lsm_tree::AbstractTree;
+use std::sync::atomic::AtomicUsize;
 
 /// Runs a single run of compaction.
-pub fn run(compaction_manager: &CompactionManager, snapshot_tracker: &SnapshotTracker) {
+pub fn run(
+    compaction_manager: &CompactionManager,
+    snapshot_tracker: &SnapshotTracker,
+    compaction_counter: &AtomicUsize,
+) {
     let Some(item) = compaction_manager.pop() else {
         return;
     };
@@ -21,10 +26,12 @@ pub fn run(compaction_manager: &CompactionManager, snapshot_tracker: &SnapshotTr
 
     // TODO: loop if there's more work to do
 
+    compaction_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
     if let Err(e) = item
         .tree
         .compact(strategy.inner(), snapshot_tracker.get_seqno_safe_to_gc())
     {
         log::error!("Compaction failed: {e:?}");
     };
+    compaction_counter.fetch_sub(1, std::sync::atomic::Ordering::Relaxed);
 }
diff --git a/src/flush/worker.rs b/src/flush/worker.rs
@@ -152,7 +152,10 @@ pub fn run(
                     flush_manager.dequeue_tasks(partition.name.clone(), created_segments.len());
 
                     write_buffer_manager.free(memtables_size);
-                    compaction_manager.notify(partition);
+
+                    for _ in 0..parallelism {
+                        compaction_manager.notify(partition.clone());
+                    }
                 }
             }
             Err(e) => {
diff --git a/src/keyspace.rs b/src/keyspace.rs
@@ -78,6 +78,9 @@ pub struct KeyspaceInner {
     /// True if fsync failed
     pub(crate) is_poisoned: Arc<AtomicBool>,
 
+    /// Active compaction conter
+    pub(crate) active_compaction_count: Arc<AtomicUsize>,
+
     #[doc(hidden)]
     pub snapshot_tracker: SnapshotTracker,
 }
@@ -194,6 +197,14 @@ impl Keyspace {
         self.write_buffer_manager.get()
     }
 
+    /// Returns the number of active compactions currently running.
+    #[doc(hidden)]
+    #[must_use]
+    pub fn active_compactions(&self) -> usize {
+        self.active_compaction_count
+            .load(std::sync::atomic::Ordering::Relaxed)
+    }
+
     /// Returns the amount of journals on disk.
     ///
     /// # Examples
@@ -580,6 +591,7 @@ impl Keyspace {
             write_buffer_manager: WriteBufferManager::default(),
             is_poisoned: Arc::default(),
             snapshot_tracker: SnapshotTracker::default(),
+            active_compaction_count: Arc::default(),
         };
 
         let keyspace = Self(Arc::new(inner));
@@ -713,6 +725,7 @@ impl Keyspace {
             write_buffer_manager: WriteBufferManager::default(),
             is_poisoned: Arc::default(),
             snapshot_tracker: SnapshotTracker::default(),
+            active_compaction_count: Arc::default(),
         };
 
         // NOTE: Lastly, fsync .fjall marker, which contains the version
@@ -792,6 +805,7 @@ impl Keyspace {
         let stop_signal = self.stop_signal.clone();
         let thread_counter = self.active_background_threads.clone();
         let snapshot_tracker = self.snapshot_tracker.clone();
+        let compaction_counter = self.active_compaction_count.clone();
 
         thread_counter.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
 
@@ -802,7 +816,11 @@ impl Keyspace {
                     log::trace!("compaction: waiting for work");
                     compaction_manager.wait_for();
 
-                    crate::compaction::worker::run(&compaction_manager, &snapshot_tracker);
+                    crate::compaction::worker::run(
+                        &compaction_manager,
+                        &snapshot_tracker,
+                        &compaction_counter,
+                    );
                 }
 
                 log::trace!("compaction thread: exiting because keyspace is dropping");
diff --git a/src/partition/mod.rs b/src/partition/mod.rs
@@ -866,15 +866,19 @@ impl PartitionHandle {
     /// # Errors
     ///
     /// Will return `Err` if an IO error occurs.
-    pub fn insert<K: AsRef<[u8]>, V: AsRef<[u8]>>(&self, key: K, value: V) -> crate::Result<()> {
+    pub fn insert<K: Into<UserKey>, V: Into<UserValue>>(
+        &self,
+        key: K,
+        value: V,
+    ) -> crate::Result<()> {
         use std::sync::atomic::Ordering;
 
         if self.is_deleted.load(Ordering::Relaxed) {
             return Err(crate::Error::PartitionDeleted);
         }
 
-        let key = key.as_ref();
-        let value = value.as_ref();
+        let key = key.into();
+        let value = value.into();
 
         let mut journal_writer = self.journal.get_writer();
 
@@ -885,7 +889,7 @@ impl PartitionHandle {
             return Err(crate::Error::Poisoned);
         }
 
-        journal_writer.write_raw(&self.name, key, value, lsm_tree::ValueType::Value, seqno)?;
+        journal_writer.write_raw(&self.name, &key, &value, lsm_tree::ValueType::Value, seqno)?;
 
         if !self.config.manual_journal_persist {
             journal_writer
@@ -943,14 +947,14 @@ impl PartitionHandle {
     /// # Errors
     ///
     /// Will return `Err` if an IO error occurs.
-    pub fn remove<K: AsRef<[u8]>>(&self, key: K) -> crate::Result<()> {
+    pub fn remove<K: Into<UserKey>>(&self, key: K) -> crate::Result<()> {
         use std::sync::atomic::Ordering;
 
         if self.is_deleted.load(Ordering::Relaxed) {
             return Err(crate::Error::PartitionDeleted);
         }
 
-        let key = key.as_ref();
+        let key = key.into();
 
         let mut journal_writer = self.journal.get_writer();
 
@@ -961,7 +965,7 @@ impl PartitionHandle {
             return Err(crate::Error::Poisoned);
         }
 
-        journal_writer.write_raw(&self.name, key, &[], lsm_tree::ValueType::Tombstone, seqno)?;
+        journal_writer.write_raw(&self.name, &key, &[], lsm_tree::ValueType::Tombstone, seqno)?;
 
         if !self.config.manual_journal_persist {
             journal_writer
diff --git a/src/partition/options.rs b/src/partition/options.rs
@@ -234,6 +234,7 @@ impl lsm_tree::coding::Decode for CreateOptions {
                     l0_threshold,
                     target_size,
                     level_ratio,
+                    ..Default::default()
                 })
             }
             1 => {
diff --git a/src/tx/conflict_manager.rs b/src/tx/conflict_manager.rs
@@ -31,18 +31,18 @@ impl ConflictManager {
         }
     }
 
-    pub fn mark_read(&mut self, partition: &PartitionKey, key: &Slice) {
-        self.push_read(partition, Read::Single(key.clone()));
+    pub fn mark_read(&mut self, partition: &PartitionKey, key: Slice) {
+        self.push_read(partition, Read::Single(key));
     }
 
-    pub fn mark_conflict(&mut self, partition: &PartitionKey, key: &[u8]) {
+    pub fn mark_conflict(&mut self, partition: &PartitionKey, key: Slice) {
         if let Some(tbl) = self.conflict_keys.get_mut(partition) {
-            tbl.insert(key.into());
+            tbl.insert(key);
         } else {
             self.conflict_keys
                 .entry(partition.clone())
                 .or_default()
-                .insert(key.into());
+                .insert(key);
         }
     }
 
diff --git a/src/tx/partition.rs b/src/tx/partition.rs
diff --git a/src/tx/write/mod.rs b/src/tx/write/mod.rs
diff --git a/src/tx/write/single_writer.rs b/src/tx/write/single_writer.rs
diff --git a/src/tx/write/ssi.rs b/src/tx/write/ssi.rs

Original file line number	Diff line number	Diff line change
`@@ -80,7 +80,7 @@ fn main() -> Result<()> {`
`80`	`80`
`81`	`81`	`for word in WORDS {`
`82`	`82`	`for term in permuterm(word) {`
`83`		`- db.insert(format!("{term}#{word}"), word).unwrap();`
	`83`	`+ db.insert(format!("{term}#{word}"), *word).unwrap();`
`84`	`84`	`}`
`85`	`85`	`}`
`86`	`86`
Original file line number	Diff line number	Diff line change
`@@ -125,7 +125,7 @@ impl Cas {`
`125`	`125`	`let rc = self.decrement_ref_count(&mut tx, &prev_content_hash)?;`
`126`	`126`	`if rc == 0 {`
`127`	`127`	`// No more references`
`128`		`- tx.remove(&self.blobs, &prev_content_hash);`
	`128`	`+ tx.remove(&self.blobs, prev_content_hash);`
`129`	`129`	`}`
`130`	`130`	`}`
`131`	`131`
`@@ -156,7 +156,7 @@ impl Cas {`
`156`	`156`	`let rc = self.decrement_ref_count(&mut tx, &content_hash)?;`
`157`	`157`	`if rc == 0 {`
`158`	`158`	`// No more references`
`159`		`- tx.remove(&self.blobs, &content_hash);`
	`159`	`+ tx.remove(&self.blobs, content_hash);`
`160`	`160`	`}`
`161`	`161`
`162`	`162`	`tx.commit()?;`
Original file line number	Diff line number	Diff line change
`@@ -152,7 +152,10 @@ pub fn run(`
`152`	`152`	`flush_manager.dequeue_tasks(partition.name.clone(), created_segments.len());`
`153`	`153`
`154`	`154`	`write_buffer_manager.free(memtables_size);`
`155`		`- compaction_manager.notify(partition);`
	`155`	`+`
	`156`	`+ for _ in 0..parallelism {`
	`157`	`+ compaction_manager.notify(partition.clone());`
	`158`	`+ }`
`156`	`159`	`}`
`157`	`160`	`}`
`158`	`161`	`Err(e) => {`