From 2820220ba60c27bd2600ef67eddb67c3a7c7c0d1 Mon Sep 17 00:00:00 2001 From: chad Date: Fri, 1 Aug 2025 16:24:26 -0500 Subject: [PATCH 1/2] chore: dep updates + refactors --- qmdb/Cargo.toml | 32 ++++++++++++++++---------------- qmdb/examples/v1_fuzz/main.rs | 12 ++++++++---- qmdb/src/metadb.rs | 13 +++++++------ qmdb/src/tasks/helpers.rs | 2 +- qmdb/src/utils/changeset.rs | 4 ++-- qmdb/src/utils/mod.rs | 4 +++- rust-toolchain.toml | 2 +- 7 files changed, 38 insertions(+), 31 deletions(-) diff --git a/qmdb/Cargo.toml b/qmdb/Cargo.toml index 0059cf2..e4843a3 100644 --- a/qmdb/Cargo.toml +++ b/qmdb/Cargo.toml @@ -15,28 +15,28 @@ path = "examples/v1_fuzz/main.rs" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -sha2 = { version = "0.10.8", features = ["asm", "asm-aarch64", "sha2-asm"] } -lazy_static = "1.4.0" +sha2 = { version = "0.10.9", features = ["asm", "asm-aarch64", "sha2-asm"] } +lazy_static = "1.5.0" hex = "0.4.3" -anyhow = "1.0.86" +anyhow = "1.0.98" byteorder = "1.5.0" -xxhash-rust = { version = "0.8.10", features = ["xxh32", "xxh3"] } +xxhash-rust = { version = "0.8.15", features = ["xxh32", "xxh3"] } threadpool = "1.8.1" -atomptr = "1.4.1" -rand_core = { version = "0.6.4", features = ["getrandom"] } -time = "0.3.36" +atomptr = "2.0.0" +rand_core = { version = "0.9.3" } +time = "0.3.41" rayon = "1.10.0" -dashmap = "6.0.0-rc.1" -parking_lot = { version = "0.12.1", features = ["arc_lock"] } +dashmap = "6.1.0" +parking_lot = { version = "0.12.4", features = ["arc_lock"] } blake2 = "0.10.6" -serde = {version="1.0.210", features = ["derive"]} -bincode = "1.3.3" +serde = { version = "1.0.219", features = ["derive"] } +bincode = "2.0.1" hpfile = { path = "../hpfile" } -rs_merkle = "1.4" +rs_merkle = "1.5.0" aes-gcm = "0.10.3" aead = "0.5.2" -generic-array = "1.1.1" -log = "0.4" +generic-array = "1.2.0" +log = "0.4.27" [target.'cfg(target_os = "linux")'.dependencies] io-uring = { version = "0.7.2", optional = true } @@ -58,5 +58,5 @@ profiling_hybridindexer = [] tee_cipher = [] directio = ["io-uring"] hpfile_all_in_mem = ["hpfile/all_in_mem"] -in_sp1=[] -slow_hashing=[] +in_sp1 = [] +slow_hashing = [] diff --git a/qmdb/examples/v1_fuzz/main.rs b/qmdb/examples/v1_fuzz/main.rs index 30a0e7a..fd0df91 100644 --- a/qmdb/examples/v1_fuzz/main.rs +++ b/qmdb/examples/v1_fuzz/main.rs @@ -1,6 +1,7 @@ pub mod refdb; use crate::refdb::RefDB; +use bincode::{config::standard, Decode, Encode}; use byteorder::{ByteOrder, LittleEndian}; use hex; use parking_lot::RwLock; @@ -24,12 +25,12 @@ use std::sync::Arc; const BLOCK_COUNT: usize = 100; // each block about 4.5M const BLOCK_NUM_PER_FILE: usize = 100; -#[derive(serde::Serialize, serde::Deserialize)] +#[derive(serde::Serialize, serde::Deserialize, Encode, Decode)] struct TaskLists { lists: Vec, } -#[derive(serde::Serialize, serde::Deserialize)] +#[derive(serde::Serialize, serde::Deserialize, Encode, Decode)] struct TaskList { tasks: Vec, } @@ -56,7 +57,7 @@ fn generate_test_block(test_gen: &mut TestGenV1, start_height: usize, end_height task_lists.push(TaskList { tasks: task_list }); } let task_lists = TaskLists { lists: task_lists }; - let out = bincode::serialize(&task_lists).unwrap(); + let out = bincode::encode_to_vec(&task_lists, standard()).unwrap(); let tasks_file = format!("tasks_for_test_{}.dat", start_height / BLOCK_NUM_PER_FILE); let mut file = File::create_new(tasks_file).unwrap(); file.write_all(out.as_ref()).unwrap(); @@ -89,7 +90,10 @@ fn run_fuzz_single_round( let file_len = file.metadata().unwrap().len(); let mut bz = vec![0u8; file_len as usize]; file.read(&mut bz).unwrap(); - let task_lists = bincode::deserialize::(&bz).unwrap().lists; + let task_lists = bincode::decode_from_slice::(&bz, standard()) + .unwrap() + .0 + .lists; println!( "task_lists len:{:?} which sn is {:?}", task_lists.len(), diff --git a/qmdb/src/metadb.rs b/qmdb/src/metadb.rs index 05b193f..1ef9aab 100644 --- a/qmdb/src/metadb.rs +++ b/qmdb/src/metadb.rs @@ -1,6 +1,7 @@ use crate::def::{NONCE_SIZE, PRUNE_EVERY_NBLOCKS, SHARD_COUNT, TAG_SIZE, TWIG_SHIFT}; use aes_gcm::aead::AeadInPlace; use aes_gcm::Aes256Gcm; +use bincode::{config::standard, Decode, Encode}; use byteorder::{ByteOrder, LittleEndian}; use dashmap::DashMap; use log::warn; @@ -15,7 +16,7 @@ use std::{ os::unix::fs::FileExt, }; -#[derive(serde::Serialize, serde::Deserialize, Clone, Debug)] +#[derive(serde::Serialize, serde::Deserialize, Clone, Debug, Encode, Decode)] pub struct MetaInfo { pub curr_height: i64, pub last_pruned_twig: [(u64, i64); SHARD_COUNT], @@ -100,8 +101,8 @@ impl MetaDB { let size = meta_info_bz.len(); meta_info_bz = meta_info_bz[8..size - TAG_SIZE].to_owned(); } - match bincode::deserialize::(&meta_info_bz[..]) { - Ok(info) => self.info = info, + match bincode::decode_from_slice::(&meta_info_bz[..], standard()) { + Ok((info, _)) => self.info = info, Err(_) => warn!("Failed to deserialize {}, ignore it", name), }; } @@ -113,8 +114,8 @@ impl MetaDB { let size = meta_info_bz.len(); meta_info_bz = meta_info_bz[8..size - TAG_SIZE].to_owned(); } - match bincode::deserialize::(&meta_info_bz[..]) { - Ok(info) => { + match bincode::decode_from_slice::(&meta_info_bz[..], standard()) { + Ok((info, _)) => { if info.curr_height > self.info.curr_height { self.info = info; //pick the latest one } @@ -154,7 +155,7 @@ impl MetaDB { let kv = self.extra_data_map.remove(&self.info.curr_height).unwrap(); self.info.extra_data = kv.1; let name = format!("{}.{}", self.meta_file_name, self.info.curr_height % 2); - let mut bz = bincode::serialize(&self.info).unwrap(); + let mut bz = bincode::encode_to_vec(&self.info, standard()).unwrap(); if self.cipher.is_some() { let cipher = self.cipher.as_ref().unwrap(); let mut nonce_arr = [0u8; NONCE_SIZE]; diff --git a/qmdb/src/tasks/helpers.rs b/qmdb/src/tasks/helpers.rs index c9c7df5..a5df63f 100644 --- a/qmdb/src/tasks/helpers.rs +++ b/qmdb/src/tasks/helpers.rs @@ -3,7 +3,7 @@ use crate::utils::changeset::ChangeSet; use serde::{Deserialize, Serialize}; use std::sync::Arc; -#[derive(Clone, serde::Serialize, serde::Deserialize)] +#[derive(Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)] pub struct SimpleTask { #[serde( serialize_with = "serialize_arc_vec", diff --git a/qmdb/src/utils/changeset.rs b/qmdb/src/utils/changeset.rs index 4c69c21..7bbdcea 100644 --- a/qmdb/src/utils/changeset.rs +++ b/qmdb/src/utils/changeset.rs @@ -2,7 +2,7 @@ use crate::def::{OP_CREATE, OP_DELETE, OP_READ, OP_WRITE, SHARD_COUNT}; use crate::utils::{byte0_to_shard_id, hasher, OpRecord}; use std::cmp::Ordering; -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)] pub struct ChangeSet { pub data: Vec, pub op_list: Vec, @@ -10,7 +10,7 @@ pub struct ChangeSet { shard_op_count: [u32; SHARD_COUNT], } -#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)] pub struct ChangeOp { pub op_type: u8, shard_id: u8, diff --git a/qmdb/src/utils/mod.rs b/qmdb/src/utils/mod.rs index 62b4343..c4986ff 100644 --- a/qmdb/src/utils/mod.rs +++ b/qmdb/src/utils/mod.rs @@ -19,7 +19,9 @@ pub fn byte0_to_shard_id(byte0: u8) -> usize { (byte0 as usize) * SHARD_COUNT / 256 } -#[derive(Debug, PartialEq, Clone, serde::Serialize, serde::Deserialize)] +#[derive( + Debug, PartialEq, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, +)] pub struct OpRecord { pub op_type: u8, pub num_active: usize, diff --git a/rust-toolchain.toml b/rust-toolchain.toml index 7d922f0..6de545e 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,3 +1,3 @@ [toolchain] -channel = "1.84.1" +channel = "1.88.0" components = ["rustc", "cargo", "clippy", "rustfmt", "llvm-tools-preview"] From bc787ea0e449f9a33a9b5df82e4fa28a73bed755 Mon Sep 17 00:00:00 2001 From: chad Date: Fri, 1 Aug 2025 16:34:53 -0500 Subject: [PATCH 2/2] build: update generic-array 1.1.0 --- qmdb/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qmdb/Cargo.toml b/qmdb/Cargo.toml index e4843a3..1307eb2 100644 --- a/qmdb/Cargo.toml +++ b/qmdb/Cargo.toml @@ -35,7 +35,7 @@ hpfile = { path = "../hpfile" } rs_merkle = "1.5.0" aes-gcm = "0.10.3" aead = "0.5.2" -generic-array = "1.2.0" +generic-array = "1.1.0" log = "0.4.27" [target.'cfg(target_os = "linux")'.dependencies]