From 8ad37c231c6a52f1c76575675e57e64f0a19044f Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Sat, 17 Aug 2024 15:53:42 +0200 Subject: [PATCH 01/10] add new trait to encode values to bytes still lots of TODOs --- heed-traits/src/lib.rs | 63 ++++++++++++++++- heed-types/src/bytes.rs | 39 +++++++++-- heed-types/src/integer.rs | 42 ++++++++---- heed-types/src/serde_bincode.rs | 16 +++-- heed-types/src/serde_json.rs | 16 +++-- heed-types/src/serde_rmp.rs | 16 +++-- heed-types/src/str.rs | 21 +++--- heed-types/src/unit.rs | 16 +++-- heed/src/database.rs | 118 ++++++++++++++++++-------------- heed/src/iterator/iter.rs | 15 ++-- 10 files changed, 245 insertions(+), 117 deletions(-) diff --git a/heed-traits/src/lib.rs b/heed-traits/src/lib.rs index 88fb0efe..4676393b 100644 --- a/heed-traits/src/lib.rs +++ b/heed-traits/src/lib.rs @@ -12,12 +12,18 @@ use std::borrow::Cow; use std::cmp::{Ord, Ordering}; use std::error::Error as StdError; +use std::fmt; /// A boxed `Send + Sync + 'static` error. pub type BoxedError = Box; /// A trait that represents an encoding structure. -pub trait BytesEncode<'a> { +#[deprecated = "replaced by `ToBytes` to allow for more optimization"] +#[allow(deprecated)] // deprecated BoxedErrorWrapper is used in a bound +pub trait BytesEncode<'a>: + // TODO are these bound needed? + ToBytes<'a, SelfType = Self::EItem, ReturnBytes = Cow<'a, [u8]>, Error = BoxedErrorWrapper> +{ /// The type to encode. type EItem: ?Sized + 'a; @@ -25,6 +31,61 @@ pub trait BytesEncode<'a> { fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError>; } +/// A trait that represents an encoding structure. +pub trait ToBytes<'a> { + /// The type to encode to bytes. + type SelfType: ?Sized + 'a; + + /// The type containing the encoded bytes. + type ReturnBytes: Into> + AsRef<[u8]> + 'a; + + /// The error type to return when decoding goes wrong. + type Error: StdError + Send + Sync + 'static; + + /// Encode the given item as bytes. + fn to_bytes(item: &'a Self::SelfType) -> Result; +} + +#[allow(deprecated)] +impl<'a, T: BytesEncode<'a>> ToBytes<'a> for T { + type SelfType = >::EItem; + + type ReturnBytes = Cow<'a, [u8]>; + + type Error = BoxedErrorWrapper; + + fn to_bytes(item: &'a Self::SelfType) -> Result { + Self::bytes_encode(item).map_err(BoxedErrorWrapper) + } +} + +/// Wraps the [`BoxedError`] type alias because for complicated reasons it does not implement +/// [`Error`][StdError]. This wrapper forwards [`Debug`][fmt::Debug], [`Display`][fmt::Display] +/// and [`Error`][StdError] through the wrapper and the [`Box`]. +#[deprecated = "this wrapper was added for backwards compatibility of BytesEncode only"] +pub struct BoxedErrorWrapper(BoxedError); + +#[allow(deprecated)] +impl fmt::Debug for BoxedErrorWrapper { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(&self.0, f) + } +} + +#[allow(deprecated)] +impl fmt::Display for BoxedErrorWrapper { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + ::fmt(&self.0, f) + } +} + +#[allow(deprecated)] +impl StdError for BoxedErrorWrapper { + fn source(&self) -> Option<&(dyn StdError + 'static)> { + self.0.source() + } +} + /// A trait that represents a decoding structure. pub trait BytesDecode<'a> { /// The type to decode. diff --git a/heed-types/src/bytes.rs b/heed-types/src/bytes.rs index ae602ba0..a707b825 100644 --- a/heed-types/src/bytes.rs +++ b/heed-types/src/bytes.rs @@ -1,6 +1,6 @@ -use std::borrow::Cow; +use std::convert::Infallible; -use heed_traits::{BoxedError, BytesDecode, BytesEncode}; +use heed_traits::{BoxedError, BytesDecode, ToBytes}; /// Describes a byte slice `[u8]` that is totally borrowed and doesn't depend on /// any [memory alignment]. @@ -8,11 +8,15 @@ use heed_traits::{BoxedError, BytesDecode, BytesEncode}; /// [memory alignment]: std::mem::align_of() pub enum Bytes {} -impl<'a> BytesEncode<'a> for Bytes { - type EItem = [u8]; +impl<'a> ToBytes<'a> for Bytes { + type SelfType = [u8]; - fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { - Ok(Cow::Borrowed(item)) + type ReturnBytes = &'a [u8]; + + type Error = Infallible; + + fn to_bytes(item: &'a Self::SelfType) -> Result { + Ok(item) } } @@ -23,3 +27,26 @@ impl<'a> BytesDecode<'a> for Bytes { Ok(bytes) } } + +/// Like [`Bytes`], but always contains exactly `N` (the generic parameter) bytes. +pub enum FixedSizeBytes {} + +impl<'a, const N: usize> ToBytes<'a> for FixedSizeBytes { + type SelfType = [u8; N]; + + type ReturnBytes = [u8; N]; // TODO &'a [u8; N] or [u8; N] + + type Error = Infallible; + + fn to_bytes(item: &'a Self::SelfType) -> Result { + Ok(*item) + } +} + +impl<'a, const N: usize> BytesDecode<'a> for FixedSizeBytes { + type DItem = [u8; N]; // TODO &'a [u8; N] or [u8; N] + + fn bytes_decode(bytes: &'a [u8]) -> Result { + bytes.try_into().map_err(Into::into) + } +} diff --git a/heed-types/src/integer.rs b/heed-types/src/integer.rs index 71435103..79751acb 100644 --- a/heed-types/src/integer.rs +++ b/heed-types/src/integer.rs @@ -1,18 +1,22 @@ -use std::borrow::Cow; +use std::convert::Infallible; use std::marker::PhantomData; use std::mem::size_of; use byteorder::{ByteOrder, ReadBytesExt}; -use heed_traits::{BoxedError, BytesDecode, BytesEncode}; +use heed_traits::{BoxedError, BytesDecode, ToBytes}; /// Encodable version of [`u8`]. pub struct U8; -impl BytesEncode<'_> for U8 { - type EItem = u8; +impl ToBytes<'_> for U8 { + type SelfType = u8; - fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { - Ok(Cow::from([*item].to_vec())) + type ReturnBytes = [u8; 1]; + + type Error = Infallible; + + fn to_bytes(item: &Self::SelfType) -> Result { + Ok([*item]) } } @@ -27,11 +31,15 @@ impl BytesDecode<'_> for U8 { /// Encodable version of [`i8`]. pub struct I8; -impl BytesEncode<'_> for I8 { - type EItem = i8; +impl ToBytes<'_> for I8 { + type SelfType = i8; + + type ReturnBytes = [u8; 1]; - fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { - Ok(Cow::from([*item as u8].to_vec())) + type Error = Infallible; + + fn to_bytes(item: &Self::SelfType) -> Result { + Ok([*item as u8]) } } @@ -51,13 +59,17 @@ macro_rules! define_type { pub struct $name(PhantomData); - impl BytesEncode<'_> for $name { - type EItem = $native; + impl ToBytes<'_> for $name { + type SelfType = $native; + + type ReturnBytes = [u8; size_of::<$native>()]; + + type Error = Infallible; - fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { - let mut buf = vec![0; size_of::()]; + fn to_bytes(item: &Self::SelfType) -> Result { + let mut buf = [0; size_of::<$native>()]; O::$write_method(&mut buf, *item); - Ok(Cow::from(buf)) + Ok(buf) } } diff --git a/heed-types/src/serde_bincode.rs b/heed-types/src/serde_bincode.rs index 37fefe01..63e85a3a 100644 --- a/heed-types/src/serde_bincode.rs +++ b/heed-types/src/serde_bincode.rs @@ -1,6 +1,4 @@ -use std::borrow::Cow; - -use heed_traits::{BoxedError, BytesDecode, BytesEncode}; +use heed_traits::{BoxedError, BytesDecode, ToBytes}; use serde::{Deserialize, Serialize}; /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `bincode` to do so. @@ -8,14 +6,18 @@ use serde::{Deserialize, Serialize}; /// It can borrow bytes from the original slice. pub struct SerdeBincode(std::marker::PhantomData); -impl<'a, T: 'a> BytesEncode<'a> for SerdeBincode +impl<'a, T: 'a> ToBytes<'a> for SerdeBincode where T: Serialize, { - type EItem = T; + type SelfType = T; + + type ReturnBytes = Vec; + + type Error = bincode::Error; - fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError> { - bincode::serialize(item).map(Cow::Owned).map_err(Into::into) + fn to_bytes(item: &'a Self::SelfType) -> Result { + bincode::serialize(item) } } diff --git a/heed-types/src/serde_json.rs b/heed-types/src/serde_json.rs index f1f1c3be..13045a4a 100644 --- a/heed-types/src/serde_json.rs +++ b/heed-types/src/serde_json.rs @@ -1,6 +1,4 @@ -use std::borrow::Cow; - -use heed_traits::{BoxedError, BytesDecode, BytesEncode}; +use heed_traits::{BoxedError, BytesDecode, ToBytes}; use serde::{Deserialize, Serialize}; /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `serde_json` to do so. @@ -8,14 +6,18 @@ use serde::{Deserialize, Serialize}; /// It can borrow bytes from the original slice. pub struct SerdeJson(std::marker::PhantomData); -impl<'a, T: 'a> BytesEncode<'a> for SerdeJson +impl<'a, T: 'a> ToBytes<'a> for SerdeJson where T: Serialize, { - type EItem = T; + type SelfType = T; + + type ReturnBytes = Vec; + + type Error = serde_json::Error; - fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { - serde_json::to_vec(item).map(Cow::Owned).map_err(Into::into) + fn to_bytes(item: &'a Self::SelfType) -> Result { + serde_json::to_vec(item) } } diff --git a/heed-types/src/serde_rmp.rs b/heed-types/src/serde_rmp.rs index f33c7467..418ead93 100644 --- a/heed-types/src/serde_rmp.rs +++ b/heed-types/src/serde_rmp.rs @@ -1,6 +1,4 @@ -use std::borrow::Cow; - -use heed_traits::{BoxedError, BytesDecode, BytesEncode}; +use heed_traits::{BoxedError, BytesDecode, ToBytes}; use serde::{Deserialize, Serialize}; /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `rmp_serde` to do so. @@ -8,14 +6,18 @@ use serde::{Deserialize, Serialize}; /// It can borrow bytes from the original slice. pub struct SerdeRmp(std::marker::PhantomData); -impl<'a, T: 'a> BytesEncode<'a> for SerdeRmp +impl<'a, T: 'a> ToBytes<'a> for SerdeRmp where T: Serialize, { - type EItem = T; + type SelfType = T; + + type ReturnBytes = Vec; + + type Error = rmp_serde::encode::Error; - fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { - rmp_serde::to_vec(item).map(Cow::Owned).map_err(Into::into) + fn to_bytes(item: &'a Self::SelfType) -> Result { + rmp_serde::to_vec(item) } } diff --git a/heed-types/src/str.rs b/heed-types/src/str.rs index 220306d9..0a0dd30d 100644 --- a/heed-types/src/str.rs +++ b/heed-types/src/str.rs @@ -1,16 +1,19 @@ -use std::borrow::Cow; -use std::str; +use std::convert::Infallible; -use heed_traits::{BoxedError, BytesDecode, BytesEncode}; +use heed_traits::{BoxedError, BytesDecode, ToBytes}; -/// Describes a [`prim@str`]. +/// Describes a [`str`]. pub enum Str {} -impl BytesEncode<'_> for Str { - type EItem = str; +impl<'a> ToBytes<'a> for Str { + type SelfType = str; - fn bytes_encode(item: &Self::EItem) -> Result, BoxedError> { - Ok(Cow::Borrowed(item.as_bytes())) + type ReturnBytes = &'a [u8]; + + type Error = Infallible; + + fn to_bytes(item: &'a Self::SelfType) -> Result { + Ok(item.as_bytes()) } } @@ -18,6 +21,6 @@ impl<'a> BytesDecode<'a> for Str { type DItem = &'a str; fn bytes_decode(bytes: &'a [u8]) -> Result { - str::from_utf8(bytes).map_err(Into::into) + std::str::from_utf8(bytes).map_err(Into::into) } } diff --git a/heed-types/src/unit.rs b/heed-types/src/unit.rs index 4fbb9e2c..69f13fa0 100644 --- a/heed-types/src/unit.rs +++ b/heed-types/src/unit.rs @@ -1,16 +1,20 @@ -use std::borrow::Cow; +use std::convert::Infallible; use std::{error, fmt}; -use heed_traits::{BoxedError, BytesDecode, BytesEncode}; +use heed_traits::{BoxedError, BytesDecode, ToBytes}; /// Describes the unit `()` type. pub enum Unit {} -impl BytesEncode<'_> for Unit { - type EItem = (); +impl ToBytes<'_> for Unit { + type SelfType = (); - fn bytes_encode(_item: &Self::EItem) -> Result, BoxedError> { - Ok(Cow::Borrowed(&[])) + type ReturnBytes = [u8; 0]; + + type Error = Infallible; + + fn to_bytes(_item: &'_ Self::SelfType) -> Result { + Ok([]) } } diff --git a/heed/src/database.rs b/heed/src/database.rs index 3817103f..b0f65e9e 100644 --- a/heed/src/database.rs +++ b/heed/src/database.rs @@ -2,7 +2,7 @@ use std::borrow::Cow; use std::ops::{Bound, RangeBounds}; use std::{any, fmt, marker, mem, ptr}; -use heed_traits::{Comparator, LexicographicComparator}; +use heed_traits::{Comparator, LexicographicComparator, ToBytes}; use types::{DecodeIgnore, LazyDecode}; use crate::cursor::MoveOperation; @@ -339,16 +339,20 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - pub fn get<'a, 'txn>(&self, txn: &'txn RoTxn, key: &'a KC::EItem) -> Result> + pub fn get<'a, 'txn>( + &self, + txn: &'txn RoTxn, + key: &'a KC::SelfType, + ) -> Result> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let mut key_val = unsafe { crate::into_val(&key_bytes) }; + let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut data_val = mem::MaybeUninit::uninit(); let result = unsafe { @@ -1225,31 +1229,32 @@ impl Database { range: &'a R, ) -> Result> where - KC: BytesEncode<'a>, - R: RangeBounds, + KC: ToBytes<'a>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); + // TODO optimize, this might do unnecessary allocations on types that are already 'static let start_bound = match range.start_bound() { Bound::Included(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Included(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Excluded(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, }; let end_bound = match range.end_bound() { Bound::Included(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Included(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Excluded(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, }; @@ -1317,31 +1322,32 @@ impl Database { range: &'a R, ) -> Result> where - KC: BytesEncode<'a>, - R: RangeBounds, + KC: ToBytes<'a>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); + // TODO optimize, this might do unnecessary allocations on types that are already 'static let start_bound = match range.start_bound() { Bound::Included(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Included(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Excluded(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, }; let end_bound = match range.end_bound() { Bound::Included(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Included(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Excluded(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, }; @@ -1399,31 +1405,32 @@ impl Database { range: &'a R, ) -> Result> where - KC: BytesEncode<'a>, - R: RangeBounds, + KC: ToBytes<'a>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); + // TODO optimize, this might do unnecessary allocations on types that are already 'static let start_bound = match range.start_bound() { Bound::Included(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Included(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Excluded(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, }; let end_bound = match range.end_bound() { Bound::Included(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Included(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Excluded(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, }; @@ -1571,16 +1578,17 @@ impl Database { pub fn prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, - prefix: &'a KC::EItem, + prefix: &'a KC::SelfType, ) -> Result> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, C: LexicographicComparator, { assert_eq_env_db_txn!(self, txn); - let prefix_bytes = KC::bytes_encode(prefix).map_err(Error::Encoding)?; - let prefix_bytes = prefix_bytes.into_owned(); + let prefix_bytes = KC::to_bytes(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; + // TODO optimize, this might do unnecessary allocations on types that are already 'static + let prefix_bytes = prefix_bytes.into(); RoCursor::new(txn, self.dbi).map(|cursor| RoPrefix::new(cursor, prefix_bytes)) } @@ -1703,16 +1711,17 @@ impl Database { pub fn rev_prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, - prefix: &'a KC::EItem, + prefix: &'a KC::SelfType, ) -> Result> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, C: LexicographicComparator, { assert_eq_env_db_txn!(self, txn); - let prefix_bytes = KC::bytes_encode(prefix).map_err(Error::Encoding)?; - let prefix_bytes = prefix_bytes.into_owned(); + let prefix_bytes = KC::to_bytes(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; + // TODO optimize, this might do unnecessary allocations on types that are already 'static + let prefix_bytes = prefix_bytes.into(); RoCursor::new(txn, self.dbi).map(|cursor| RoRevPrefix::new(cursor, prefix_bytes)) } @@ -1822,18 +1831,23 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - pub fn put<'a>(&self, txn: &mut RwTxn, key: &'a KC::EItem, data: &'a DC::EItem) -> Result<()> + pub fn put<'a>( + &self, + txn: &mut RwTxn, + key: &'a KC::SelfType, + data: &'a DC::SelfType, + ) -> Result<()> where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; - let mut key_val = unsafe { crate::into_val(&key_bytes) }; - let mut data_val = unsafe { crate::into_val(&data_bytes) }; + let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; + let mut data_val = unsafe { crate::into_val(data_bytes.as_ref()) }; let flags = 0; unsafe { @@ -2471,8 +2485,8 @@ impl Database { /// ``` pub fn delete_range<'a, 'txn, R>(&self, txn: &'txn mut RwTxn, range: &'a R) -> Result where - KC: BytesEncode<'a> + BytesDecode<'txn>, - R: RangeBounds, + KC: ToBytes<'a> + BytesDecode<'txn>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); diff --git a/heed/src/iterator/iter.rs b/heed/src/iterator/iter.rs index 87f67662..7d76a9d6 100644 --- a/heed/src/iterator/iter.rs +++ b/heed/src/iterator/iter.rs @@ -1,6 +1,7 @@ use std::borrow::Cow; use std::marker; +use heed_traits::ToBytes; use types::LazyDecode; use crate::iteration_method::{IterationMethod, MoveBetweenKeys, MoveThroughDuplicateValues}; @@ -325,16 +326,16 @@ impl<'txn, KC, DC, IM> RwIter<'txn, KC, DC, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::EItem, - data: &'a NDC::EItem, + key: &'a KC::SelfType, + data: &'a NDC::SelfType, ) -> Result<()> where - KC: BytesEncode<'a>, - NDC: BytesEncode<'a>, + KC: ToBytes<'a>, + NDC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = NDC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current_with_flags(flags, &key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } /// Move on the first value of keys, ignoring duplicate values. From eb1bd0166b7c024abc1a6d7b010dcce1af254151 Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Sat, 17 Aug 2024 16:38:13 +0200 Subject: [PATCH 02/10] move code from BytesEncode to ToBytes --- heed/src/database.rs | 170 ++++++++++++++++++------------------ heed/src/iterator/iter.rs | 69 ++++++++------- heed/src/iterator/prefix.rs | 83 ++++++++++-------- heed/src/iterator/range.rs | 83 ++++++++++-------- heed/src/lib.rs | 5 +- 5 files changed, 222 insertions(+), 188 deletions(-) diff --git a/heed/src/database.rs b/heed/src/database.rs index b0f65e9e..0889f4d7 100644 --- a/heed/src/database.rs +++ b/heed/src/database.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::ops::{Bound, RangeBounds}; use std::{any, fmt, marker, mem, ptr}; @@ -425,16 +424,16 @@ impl Database { pub fn get_duplicates<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::EItem, + key: &'a KC::SelfType, ) -> Result>> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - if cursor.move_on_key(&key_bytes)? { + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + if cursor.move_on_key(key_bytes.as_ref())? { Ok(Some(RoIter::new(cursor))) } else { Ok(None) @@ -488,17 +487,17 @@ impl Database { pub fn get_lower_than<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::EItem, + key: &'a KC::SelfType, ) -> Result> where - KC: BytesEncode<'a> + BytesDecode<'txn>, + KC: ToBytes<'a> + BytesDecode<'txn>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - cursor.move_on_key_greater_than_or_equal_to(&key_bytes)?; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + cursor.move_on_key_greater_than_or_equal_to(key_bytes.as_ref())?; match cursor.move_on_prev(MoveOperation::NoDup) { Ok(Some((key, data))) => match (KC::bytes_decode(key), DC::bytes_decode(data)) { @@ -557,18 +556,19 @@ impl Database { pub fn get_lower_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::EItem, + key: &'a KC::SelfType, ) -> Result> where - KC: BytesEncode<'a> + BytesDecode<'txn>, + KC: ToBytes<'a> + BytesDecode<'txn>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let result = match cursor.move_on_key_greater_than_or_equal_to(&key_bytes) { - Ok(Some((key, data))) if key == &key_bytes[..] => Ok(Some((key, data))), + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = key_bytes.as_ref(); + let result = match cursor.move_on_key_greater_than_or_equal_to(key_bytes) { + Ok(Some((key, data))) if key == key_bytes => Ok(Some((key, data))), Ok(_) => cursor.move_on_prev(MoveOperation::NoDup), Err(e) => Err(e), }; @@ -630,18 +630,19 @@ impl Database { pub fn get_greater_than<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::EItem, + key: &'a KC::SelfType, ) -> Result> where - KC: BytesEncode<'a> + BytesDecode<'txn>, + KC: ToBytes<'a> + BytesDecode<'txn>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let entry = match cursor.move_on_key_greater_than_or_equal_to(&key_bytes)? { - Some((key, data)) if key > &key_bytes[..] => Some((key, data)), + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = key_bytes.as_ref(); + let entry = match cursor.move_on_key_greater_than_or_equal_to(key_bytes)? { + Some((key, data)) if key > key_bytes => Some((key, data)), Some((_key, _data)) => cursor.move_on_next(MoveOperation::NoDup)?, None => None, }; @@ -702,17 +703,17 @@ impl Database { pub fn get_greater_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::EItem, + key: &'a KC::SelfType, ) -> Result> where - KC: BytesEncode<'a> + BytesDecode<'txn>, + KC: ToBytes<'a> + BytesDecode<'txn>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - match cursor.move_on_key_greater_than_or_equal_to(&key_bytes) { + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + match cursor.move_on_key_greater_than_or_equal_to(key_bytes.as_ref()) { Ok(Some((key, data))) => match (KC::bytes_decode(key), DC::bytes_decode(data)) { (Ok(key), Ok(data)) => Ok(Some((key, data))), (Err(e), _) | (_, Err(e)) => Err(Error::Decoding(e)), @@ -1498,31 +1499,32 @@ impl Database { range: &'a R, ) -> Result> where - KC: BytesEncode<'a>, - R: RangeBounds, + KC: ToBytes<'a>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); + // TODO optimize, this might do unnecessary allocations on types that are already 'static let start_bound = match range.start_bound() { Bound::Included(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Included(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Excluded(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, }; let end_bound = match range.end_bound() { Bound::Included(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Included(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::bytes_encode(bound).map_err(Error::Encoding)?; - Bound::Excluded(bytes.into_owned()) + let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, }; @@ -1650,16 +1652,17 @@ impl Database { pub fn prefix_iter_mut<'a, 'txn>( &self, txn: &'txn mut RwTxn, - prefix: &'a KC::EItem, + prefix: &'a KC::SelfType, ) -> Result> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, C: LexicographicComparator, { assert_eq_env_db_txn!(self, txn); - let prefix_bytes = KC::bytes_encode(prefix).map_err(Error::Encoding)?; - let prefix_bytes = prefix_bytes.into_owned(); + let prefix_bytes = KC::to_bytes(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; + // TODO optimize, this might do unnecessary allocations on types that are already 'static + let prefix_bytes = prefix_bytes.into(); RwCursor::new(txn, self.dbi).map(|cursor| RwPrefix::new(cursor, prefix_bytes)) } @@ -1783,16 +1786,17 @@ impl Database { pub fn rev_prefix_iter_mut<'a, 'txn>( &self, txn: &'txn mut RwTxn, - prefix: &'a KC::EItem, + prefix: &'a KC::SelfType, ) -> Result> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, C: LexicographicComparator, { assert_eq_env_db_txn!(self, txn); - let prefix_bytes = KC::bytes_encode(prefix).map_err(Error::Encoding)?; - let prefix_bytes = prefix_bytes.into_owned(); + let prefix_bytes = KC::to_bytes(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; + // TODO optimize, this might do unnecessary allocations on types that are already 'static + let prefix_bytes = prefix_bytes.into(); RwCursor::new(txn, self.dbi).map(|cursor| RwRevPrefix::new(cursor, prefix_bytes)) } @@ -1896,18 +1900,18 @@ impl Database { pub fn put_reserved<'a, F>( &self, txn: &mut RwTxn, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result<()> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { assert_eq_env_db_txn!(self, txn); - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let mut key_val = unsafe { crate::into_val(&key_bytes) }; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut reserved = ffi::reserve_size_val(data_size); let flags = ffi::MDB_RESERVE; @@ -1987,20 +1991,20 @@ impl Database { &self, txn: &mut RwTxn, flags: PutFlags, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result<()> where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; - let mut key_val = unsafe { crate::into_val(&key_bytes) }; - let mut data_val = unsafe { crate::into_val(&data_bytes) }; + let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; + let mut data_val = unsafe { crate::into_val(data_bytes.as_ref()) }; let flags = flags.bits(); unsafe { @@ -2046,12 +2050,12 @@ impl Database { pub fn get_or_put<'a, 'txn>( &'txn self, txn: &mut RwTxn, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result> where - KC: BytesEncode<'a>, - DC: BytesEncode<'a> + BytesDecode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a> + BytesDecode<'a>, { self.get_or_put_with_flags(txn, PutFlags::empty(), key, data) } @@ -2094,20 +2098,20 @@ impl Database { &'txn self, txn: &mut RwTxn, flags: PutFlags, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result> where - KC: BytesEncode<'a>, - DC: BytesEncode<'a> + BytesDecode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a> + BytesDecode<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; - let mut key_val = unsafe { crate::into_val(&key_bytes) }; - let mut data_val = unsafe { crate::into_val(&data_bytes) }; + let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; + let mut data_val = unsafe { crate::into_val(data_bytes.as_ref()) }; let flags = (flags | PutFlags::NO_OVERWRITE).bits(); let result = unsafe { @@ -2178,12 +2182,12 @@ impl Database { pub fn get_or_put_reserved<'a, 'txn, F>( &'txn self, txn: &mut RwTxn, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, DC: BytesDecode<'a>, { @@ -2243,20 +2247,20 @@ impl Database { &'txn self, txn: &mut RwTxn, flags: PutFlags, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result> where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, DC: BytesDecode<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let mut key_val = unsafe { crate::into_val(&key_bytes) }; + let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut reserved = ffi::reserve_size_val(data_size); let flags = (flags | PutFlags::NO_OVERWRITE).bits() | lmdb_master_sys::MDB_RESERVE; @@ -2328,14 +2332,14 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - pub fn delete<'a>(&self, txn: &mut RwTxn, key: &'a KC::EItem) -> Result + pub fn delete<'a>(&self, txn: &mut RwTxn, key: &'a KC::SelfType) -> Result where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let mut key_val = unsafe { crate::into_val(&key_bytes) }; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let result = unsafe { mdb_result(ffi::mdb_del(txn.txn.txn, self.dbi, &mut key_val, ptr::null_mut())) @@ -2410,19 +2414,19 @@ impl Database { pub fn delete_one_duplicate<'a>( &self, txn: &mut RwTxn, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; - let mut key_val = unsafe { crate::into_val(&key_bytes) }; - let mut data_val = unsafe { crate::into_val(&data_bytes) }; + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; + let mut data_val = unsafe { crate::into_val(data_bytes.as_ref()) }; let result = unsafe { mdb_result(ffi::mdb_del(txn.txn.txn, self.dbi, &mut key_val, &mut data_val)) }; diff --git a/heed/src/iterator/iter.rs b/heed/src/iterator/iter.rs index 7d76a9d6..95bf1aa6 100644 --- a/heed/src/iterator/iter.rs +++ b/heed/src/iterator/iter.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::marker; use heed_traits::ToBytes; @@ -266,16 +265,16 @@ impl<'txn, KC, DC, IM> RwIter<'txn, KC, DC, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current(&key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } /// Write a new value to the current entry. The entry is written with the specified flags. @@ -294,16 +293,21 @@ impl<'txn, KC, DC, IM> RwIter<'txn, KC, DC, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - self.cursor.put_current_reserved_with_flags(flags, &key_bytes, data_size, write_func) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_reserved_with_flags( + flags, + key_bytes.as_ref(), + data_size, + write_func, + ) } /// Insert a key-value pair in this database. The entry is written with the specified flags and data codec. @@ -615,16 +619,16 @@ impl<'txn, KC, DC, IM> RwRevIter<'txn, KC, DC, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current(&key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } /// Write a new value to the current entry. The entry is written with the specified flags. @@ -643,16 +647,21 @@ impl<'txn, KC, DC, IM> RwRevIter<'txn, KC, DC, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - self.cursor.put_current_reserved_with_flags(flags, &key_bytes, data_size, write_func) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_reserved_with_flags( + flags, + key_bytes.as_ref(), + data_size, + write_func, + ) } /// Insert a key-value pair in this database. The entry is written with the specified flags and data codec. @@ -675,16 +684,16 @@ impl<'txn, KC, DC, IM> RwRevIter<'txn, KC, DC, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::EItem, - data: &'a NDC::EItem, + key: &'a KC::SelfType, + data: &'a NDC::SelfType, ) -> Result<()> where - KC: BytesEncode<'a>, - NDC: BytesEncode<'a>, + KC: ToBytes<'a>, + NDC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = NDC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current_with_flags(flags, &key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } /// Move on the first value of keys, ignoring duplicate values. diff --git a/heed/src/iterator/prefix.rs b/heed/src/iterator/prefix.rs index fabd2f49..15fb513a 100644 --- a/heed/src/iterator/prefix.rs +++ b/heed/src/iterator/prefix.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::marker; use heed_traits::LexicographicComparator; @@ -258,16 +257,16 @@ impl<'txn, KC, DC, C, IM> RwPrefix<'txn, KC, DC, C, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current(&key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } /// Write a new value to the current entry. The entry is written with the specified flags. @@ -286,16 +285,21 @@ impl<'txn, KC, DC, C, IM> RwPrefix<'txn, KC, DC, C, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - self.cursor.put_current_reserved_with_flags(flags, &key_bytes, data_size, write_func) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_reserved_with_flags( + flags, + key_bytes.as_ref(), + data_size, + write_func, + ) } /// Insert a key-value pair in this database. The entry is written with the specified flags and data codec. @@ -318,16 +322,16 @@ impl<'txn, KC, DC, C, IM> RwPrefix<'txn, KC, DC, C, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::EItem, - data: &'a NDC::EItem, + key: &'a KC::SelfType, + data: &'a NDC::SelfType, ) -> Result<()> where - KC: BytesEncode<'a>, - NDC: BytesEncode<'a>, + KC: ToBytes<'a>, + NDC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = NDC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current_with_flags(flags, &key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } /// Move on the first value of keys, ignoring duplicate values. @@ -648,16 +652,16 @@ impl<'txn, KC, DC, C, IM> RwRevPrefix<'txn, KC, DC, C, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current(&key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } /// Write a new value to the current entry. The entry is written with the specified flags. @@ -676,16 +680,21 @@ impl<'txn, KC, DC, C, IM> RwRevPrefix<'txn, KC, DC, C, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - self.cursor.put_current_reserved_with_flags(flags, &key_bytes, data_size, write_func) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_reserved_with_flags( + flags, + key_bytes.as_ref(), + data_size, + write_func, + ) } /// Insert a key-value pair in this database. The entry is written with the specified flags and data codec. @@ -708,16 +717,16 @@ impl<'txn, KC, DC, C, IM> RwRevPrefix<'txn, KC, DC, C, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::EItem, - data: &'a NDC::EItem, + key: &'a KC::SelfType, + data: &'a NDC::SelfType, ) -> Result<()> where - KC: BytesEncode<'a>, - NDC: BytesEncode<'a>, + KC: ToBytes<'a>, + NDC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = NDC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current_with_flags(flags, &key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } /// Move on the first value of keys, ignoring duplicate values. diff --git a/heed/src/iterator/range.rs b/heed/src/iterator/range.rs index c6dc99c4..05adf154 100644 --- a/heed/src/iterator/range.rs +++ b/heed/src/iterator/range.rs @@ -1,4 +1,3 @@ -use std::borrow::Cow; use std::marker; use std::ops::Bound; @@ -267,16 +266,16 @@ impl<'txn, KC, DC, IM> RwRange<'txn, KC, DC, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current(&key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } /// Write a new value to the current entry. The entry is written with the specified flags. @@ -295,16 +294,21 @@ impl<'txn, KC, DC, IM> RwRange<'txn, KC, DC, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - self.cursor.put_current_reserved_with_flags(flags, &key_bytes, data_size, write_func) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_reserved_with_flags( + flags, + key_bytes.as_ref(), + data_size, + write_func, + ) } /// Insert a key-value pair in this database. The entry is written with the specified flags and data codec. @@ -327,16 +331,16 @@ impl<'txn, KC, DC, IM> RwRange<'txn, KC, DC, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::EItem, - data: &'a NDC::EItem, + key: &'a KC::SelfType, + data: &'a NDC::SelfType, ) -> Result<()> where - KC: BytesEncode<'a>, - NDC: BytesEncode<'a>, + KC: ToBytes<'a>, + NDC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = NDC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current_with_flags(flags, &key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } /// Move on the first value of keys, ignoring duplicate values. @@ -704,16 +708,16 @@ impl<'txn, KC, DC, IM> RwRevRange<'txn, KC, DC, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::EItem, - data: &'a DC::EItem, + key: &'a KC::SelfType, + data: &'a DC::SelfType, ) -> Result where - KC: BytesEncode<'a>, - DC: BytesEncode<'a>, + KC: ToBytes<'a>, + DC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = DC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current(&key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } /// Write a new value to the current entry. The entry is written with the specified flags. @@ -732,16 +736,21 @@ impl<'txn, KC, DC, IM> RwRevRange<'txn, KC, DC, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::EItem, + key: &'a KC::SelfType, data_size: usize, write_func: F, ) -> Result where - KC: BytesEncode<'a>, + KC: ToBytes<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - self.cursor.put_current_reserved_with_flags(flags, &key_bytes, data_size, write_func) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_reserved_with_flags( + flags, + key_bytes.as_ref(), + data_size, + write_func, + ) } /// Insert a key-value pair in this database. The entry is written with the specified flags and data codec. @@ -764,16 +773,16 @@ impl<'txn, KC, DC, IM> RwRevRange<'txn, KC, DC, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::EItem, - data: &'a NDC::EItem, + key: &'a KC::SelfType, + data: &'a NDC::SelfType, ) -> Result<()> where - KC: BytesEncode<'a>, - NDC: BytesEncode<'a>, + KC: ToBytes<'a>, + NDC: ToBytes<'a>, { - let key_bytes: Cow<[u8]> = KC::bytes_encode(key).map_err(Error::Encoding)?; - let data_bytes: Cow<[u8]> = NDC::bytes_encode(data).map_err(Error::Encoding)?; - self.cursor.put_current_with_flags(flags, &key_bytes, &data_bytes) + let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } /// Move on the first value of keys, ignoring duplicate values. diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 83a5d15d..cef0904b 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -92,7 +92,10 @@ pub use self::mdb::error::Error as MdbError; use self::mdb::ffi::{from_val, into_val}; pub use self::mdb::flags::{DatabaseFlags, EnvFlags, PutFlags}; pub use self::reserved_space::ReservedSpace; -pub use self::traits::{BoxedError, BytesDecode, BytesEncode, Comparator, LexicographicComparator}; +#[allow(deprecated)] // re-exports BytesEncode +pub use self::traits::{ + BoxedError, BytesDecode, BytesEncode, Comparator, LexicographicComparator, ToBytes, +}; pub use self::txn::{RoTxn, RwTxn}; /// The underlying LMDB library version information. From 2978538b3c67ed9d6e05723084ed7c0b36764d87 Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Sat, 17 Aug 2024 16:52:35 +0200 Subject: [PATCH 03/10] fix cookbook --- heed/src/cookbook.rs | 46 ++++++++++++++++++++++++++------------------ 1 file changed, 27 insertions(+), 19 deletions(-) diff --git a/heed/src/cookbook.rs b/heed/src/cookbook.rs index f2cfd905..4bae4d50 100644 --- a/heed/src/cookbook.rs +++ b/heed/src/cookbook.rs @@ -147,14 +147,15 @@ //! //! ``` //! use std::borrow::Cow; +//! use std::convert::Infallible; //! use std::error::Error; //! use std::fs; //! use std::path::Path; //! //! use heed::types::*; -//! use heed::{BoxedError, BytesDecode, BytesEncode, Database, EnvOpenOptions}; +//! use heed::{BoxedError, BytesDecode, ToBytes, Database, EnvOpenOptions}; //! -//! #[derive(Debug, PartialEq, Eq)] +//! #[derive(Debug, Clone, Copy, PartialEq, Eq)] //! pub enum Level { //! Debug, //! Warn, @@ -169,21 +170,23 @@ //! //! pub struct LogKeyCodec; //! -//! impl<'a> BytesEncode<'a> for LogKeyCodec { -//! type EItem = LogKey; +//! impl<'a> ToBytes<'a> for LogKeyCodec { +//! type SelfType = LogKey; +//! +//! type ReturnBytes = [u8; 5]; +//! +//! type Error = Infallible; //! //! /// Encodes the u32 timestamp in big endian followed by the log level with a single byte. -//! fn bytes_encode(log: &Self::EItem) -> Result, BoxedError> { -//! let (timestamp_bytes, level_byte) = match log { -//! LogKey { timestamp, level: Level::Debug } => (timestamp.to_be_bytes(), 0), -//! LogKey { timestamp, level: Level::Warn } => (timestamp.to_be_bytes(), 1), -//! LogKey { timestamp, level: Level::Error } => (timestamp.to_be_bytes(), 2), -//! }; +//! fn to_bytes(log: &'a Self::SelfType) -> Result { +//! let mut output = [0; 5]; //! -//! let mut output = Vec::new(); -//! output.extend_from_slice(×tamp_bytes); -//! output.push(level_byte); -//! Ok(Cow::Owned(output)) +//! let [timestamp @ .., level] = &mut output; +//! +//! *timestamp = log.timestamp.to_be_bytes(); +//! *level = log.level as u8; +//! +//! Ok(output) //! } //! } //! @@ -215,12 +218,17 @@ //! /// the logs that appeared during a, rather long, period. //! pub struct LogAtHalfTimestampCodec; //! -//! impl<'a> BytesEncode<'a> for LogAtHalfTimestampCodec { -//! type EItem = u32; +//! impl<'a> ToBytes<'a> for LogAtHalfTimestampCodec { +//! type SelfType = u32; +//! +//! type ReturnBytes = [u8; 2]; +//! +//! type Error = Infallible; //! //! /// This method encodes only the prefix of the keys in this particular case, the timestamp. -//! fn bytes_encode(half_timestamp: &Self::EItem) -> Result, BoxedError> { -//! Ok(Cow::Owned(half_timestamp.to_be_bytes()[..2].to_vec())) +//! fn to_bytes(half_timestamp: &'a Self::SelfType) -> Result { +//! let [bytes @ .., _, _] = half_timestamp.to_be_bytes(); +//! Ok(bytes) //! } //! } //! @@ -449,4 +457,4 @@ // To let cargo generate doc links #![allow(unused_imports)] -use crate::{BytesDecode, BytesEncode, Database, EnvOpenOptions}; +use crate::{BytesDecode, Database, EnvOpenOptions, ToBytes}; From 6a0c28896d9f3cf9c72919bc1fcabfa20f3687fa Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Sat, 17 Aug 2024 17:16:02 +0200 Subject: [PATCH 04/10] remove deprecated BytesEncode and add ToBytes::bytes_encode_into_writer --- heed-traits/src/lib.rs | 62 +++++---------------------------- heed-types/src/serde_bincode.rs | 2 +- heed-types/src/serde_json.rs | 2 +- heed-types/src/serde_rmp.rs | 2 +- heed-types/src/unit.rs | 2 +- heed/src/lib.rs | 5 +-- 6 files changed, 13 insertions(+), 62 deletions(-) diff --git a/heed-traits/src/lib.rs b/heed-traits/src/lib.rs index 4676393b..a11837f0 100644 --- a/heed-traits/src/lib.rs +++ b/heed-traits/src/lib.rs @@ -9,28 +9,12 @@ #![warn(missing_docs)] -use std::borrow::Cow; use std::cmp::{Ord, Ordering}; use std::error::Error as StdError; -use std::fmt; /// A boxed `Send + Sync + 'static` error. pub type BoxedError = Box; -/// A trait that represents an encoding structure. -#[deprecated = "replaced by `ToBytes` to allow for more optimization"] -#[allow(deprecated)] // deprecated BoxedErrorWrapper is used in a bound -pub trait BytesEncode<'a>: - // TODO are these bound needed? - ToBytes<'a, SelfType = Self::EItem, ReturnBytes = Cow<'a, [u8]>, Error = BoxedErrorWrapper> -{ - /// The type to encode. - type EItem: ?Sized + 'a; - - /// Encode the given item as bytes. - fn bytes_encode(item: &'a Self::EItem) -> Result, BoxedError>; -} - /// A trait that represents an encoding structure. pub trait ToBytes<'a> { /// The type to encode to bytes. @@ -44,45 +28,15 @@ pub trait ToBytes<'a> { /// Encode the given item as bytes. fn to_bytes(item: &'a Self::SelfType) -> Result; -} - -#[allow(deprecated)] -impl<'a, T: BytesEncode<'a>> ToBytes<'a> for T { - type SelfType = >::EItem; - - type ReturnBytes = Cow<'a, [u8]>; - - type Error = BoxedErrorWrapper; - - fn to_bytes(item: &'a Self::SelfType) -> Result { - Self::bytes_encode(item).map_err(BoxedErrorWrapper) - } -} - -/// Wraps the [`BoxedError`] type alias because for complicated reasons it does not implement -/// [`Error`][StdError]. This wrapper forwards [`Debug`][fmt::Debug], [`Display`][fmt::Display] -/// and [`Error`][StdError] through the wrapper and the [`Box`]. -#[deprecated = "this wrapper was added for backwards compatibility of BytesEncode only"] -pub struct BoxedErrorWrapper(BoxedError); - -#[allow(deprecated)] -impl fmt::Debug for BoxedErrorWrapper { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ::fmt(&self.0, f) - } -} - -#[allow(deprecated)] -impl fmt::Display for BoxedErrorWrapper { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - ::fmt(&self.0, f) - } -} -#[allow(deprecated)] -impl StdError for BoxedErrorWrapper { - fn source(&self) -> Option<&(dyn StdError + 'static)> { - self.0.source() + /// Encode the given item as bytes and write it into the writer. This function by default + /// forwards to `to_bytes`. + fn bytes_encode_into_writer( + item: &'a Self::SelfType, + writer: &mut Vec, + ) -> Result<(), Self::Error> { + writer.extend_from_slice(Self::to_bytes(item)?.as_ref()); + Ok(()) } } diff --git a/heed-types/src/serde_bincode.rs b/heed-types/src/serde_bincode.rs index 63e85a3a..0b027f1e 100644 --- a/heed-types/src/serde_bincode.rs +++ b/heed-types/src/serde_bincode.rs @@ -16,7 +16,7 @@ where type Error = bincode::Error; - fn to_bytes(item: &'a Self::SelfType) -> Result { + fn to_bytes(item: &Self::SelfType) -> Result { bincode::serialize(item) } } diff --git a/heed-types/src/serde_json.rs b/heed-types/src/serde_json.rs index 13045a4a..95531890 100644 --- a/heed-types/src/serde_json.rs +++ b/heed-types/src/serde_json.rs @@ -16,7 +16,7 @@ where type Error = serde_json::Error; - fn to_bytes(item: &'a Self::SelfType) -> Result { + fn to_bytes(item: &Self::SelfType) -> Result { serde_json::to_vec(item) } } diff --git a/heed-types/src/serde_rmp.rs b/heed-types/src/serde_rmp.rs index 418ead93..15698689 100644 --- a/heed-types/src/serde_rmp.rs +++ b/heed-types/src/serde_rmp.rs @@ -16,7 +16,7 @@ where type Error = rmp_serde::encode::Error; - fn to_bytes(item: &'a Self::SelfType) -> Result { + fn to_bytes(item: &Self::SelfType) -> Result { rmp_serde::to_vec(item) } } diff --git a/heed-types/src/unit.rs b/heed-types/src/unit.rs index 69f13fa0..97bebe3d 100644 --- a/heed-types/src/unit.rs +++ b/heed-types/src/unit.rs @@ -13,7 +13,7 @@ impl ToBytes<'_> for Unit { type Error = Infallible; - fn to_bytes(_item: &'_ Self::SelfType) -> Result { + fn to_bytes(&(): &Self::SelfType) -> Result { Ok([]) } } diff --git a/heed/src/lib.rs b/heed/src/lib.rs index cef0904b..950b6ec1 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -92,10 +92,7 @@ pub use self::mdb::error::Error as MdbError; use self::mdb::ffi::{from_val, into_val}; pub use self::mdb::flags::{DatabaseFlags, EnvFlags, PutFlags}; pub use self::reserved_space::ReservedSpace; -#[allow(deprecated)] // re-exports BytesEncode -pub use self::traits::{ - BoxedError, BytesDecode, BytesEncode, Comparator, LexicographicComparator, ToBytes, -}; +pub use self::traits::{BoxedError, BytesDecode, Comparator, LexicographicComparator, ToBytes}; pub use self::txn::{RoTxn, RwTxn}; /// The underlying LMDB library version information. From 2a1b9025e3e93d3ffccc9b8663061ec936208d59 Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Sat, 17 Aug 2024 20:30:57 +0200 Subject: [PATCH 05/10] rename the new trait and associated function back --- heed-traits/src/lib.rs | 14 +- heed-types/src/bytes.rs | 14 +- heed-types/src/integer.rs | 20 +-- heed-types/src/serde_bincode.rs | 8 +- heed-types/src/serde_json.rs | 8 +- heed-types/src/serde_rmp.rs | 8 +- heed-types/src/str.rs | 8 +- heed-types/src/unit.rs | 8 +- heed/src/cookbook.rs | 17 ++- heed/src/database.rs | 219 +++++++++++++++++--------------- heed/src/iterator/iter.rs | 62 ++++----- heed/src/iterator/prefix.rs | 60 ++++----- heed/src/iterator/range.rs | 60 ++++----- heed/src/lib.rs | 2 +- 14 files changed, 259 insertions(+), 249 deletions(-) diff --git a/heed-traits/src/lib.rs b/heed-traits/src/lib.rs index a11837f0..06acfa93 100644 --- a/heed-traits/src/lib.rs +++ b/heed-traits/src/lib.rs @@ -16,9 +16,9 @@ use std::error::Error as StdError; pub type BoxedError = Box; /// A trait that represents an encoding structure. -pub trait ToBytes<'a> { - /// The type to encode to bytes. - type SelfType: ?Sized + 'a; +pub trait BytesEncode<'a> { + /// The type to encode. + type EItem: ?Sized + 'a; /// The type containing the encoded bytes. type ReturnBytes: Into> + AsRef<[u8]> + 'a; @@ -27,15 +27,15 @@ pub trait ToBytes<'a> { type Error: StdError + Send + Sync + 'static; /// Encode the given item as bytes. - fn to_bytes(item: &'a Self::SelfType) -> Result; + fn bytes_encode(item: &'a Self::EItem) -> Result; /// Encode the given item as bytes and write it into the writer. This function by default - /// forwards to `to_bytes`. + /// forwards to `bytes_encode`. fn bytes_encode_into_writer( - item: &'a Self::SelfType, + item: &'a Self::EItem, writer: &mut Vec, ) -> Result<(), Self::Error> { - writer.extend_from_slice(Self::to_bytes(item)?.as_ref()); + writer.extend_from_slice(Self::bytes_encode(item)?.as_ref()); Ok(()) } } diff --git a/heed-types/src/bytes.rs b/heed-types/src/bytes.rs index a707b825..934357bd 100644 --- a/heed-types/src/bytes.rs +++ b/heed-types/src/bytes.rs @@ -1,6 +1,6 @@ use std::convert::Infallible; -use heed_traits::{BoxedError, BytesDecode, ToBytes}; +use heed_traits::{BoxedError, BytesDecode, BytesEncode}; /// Describes a byte slice `[u8]` that is totally borrowed and doesn't depend on /// any [memory alignment]. @@ -8,14 +8,14 @@ use heed_traits::{BoxedError, BytesDecode, ToBytes}; /// [memory alignment]: std::mem::align_of() pub enum Bytes {} -impl<'a> ToBytes<'a> for Bytes { - type SelfType = [u8]; +impl<'a> BytesEncode<'a> for Bytes { + type EItem = [u8]; type ReturnBytes = &'a [u8]; type Error = Infallible; - fn to_bytes(item: &'a Self::SelfType) -> Result { + fn bytes_encode(item: &'a Self::EItem) -> Result { Ok(item) } } @@ -31,14 +31,14 @@ impl<'a> BytesDecode<'a> for Bytes { /// Like [`Bytes`], but always contains exactly `N` (the generic parameter) bytes. pub enum FixedSizeBytes {} -impl<'a, const N: usize> ToBytes<'a> for FixedSizeBytes { - type SelfType = [u8; N]; +impl<'a, const N: usize> BytesEncode<'a> for FixedSizeBytes { + type EItem = [u8; N]; type ReturnBytes = [u8; N]; // TODO &'a [u8; N] or [u8; N] type Error = Infallible; - fn to_bytes(item: &'a Self::SelfType) -> Result { + fn bytes_encode(item: &'a Self::EItem) -> Result { Ok(*item) } } diff --git a/heed-types/src/integer.rs b/heed-types/src/integer.rs index 79751acb..fd9a67de 100644 --- a/heed-types/src/integer.rs +++ b/heed-types/src/integer.rs @@ -3,19 +3,19 @@ use std::marker::PhantomData; use std::mem::size_of; use byteorder::{ByteOrder, ReadBytesExt}; -use heed_traits::{BoxedError, BytesDecode, ToBytes}; +use heed_traits::{BoxedError, BytesDecode, BytesEncode}; /// Encodable version of [`u8`]. pub struct U8; -impl ToBytes<'_> for U8 { - type SelfType = u8; +impl BytesEncode<'_> for U8 { + type EItem = u8; type ReturnBytes = [u8; 1]; type Error = Infallible; - fn to_bytes(item: &Self::SelfType) -> Result { + fn bytes_encode(item: &Self::EItem) -> Result { Ok([*item]) } } @@ -31,14 +31,14 @@ impl BytesDecode<'_> for U8 { /// Encodable version of [`i8`]. pub struct I8; -impl ToBytes<'_> for I8 { - type SelfType = i8; +impl BytesEncode<'_> for I8 { + type EItem = i8; type ReturnBytes = [u8; 1]; type Error = Infallible; - fn to_bytes(item: &Self::SelfType) -> Result { + fn bytes_encode(item: &Self::EItem) -> Result { Ok([*item as u8]) } } @@ -59,14 +59,14 @@ macro_rules! define_type { pub struct $name(PhantomData); - impl ToBytes<'_> for $name { - type SelfType = $native; + impl BytesEncode<'_> for $name { + type EItem = $native; type ReturnBytes = [u8; size_of::<$native>()]; type Error = Infallible; - fn to_bytes(item: &Self::SelfType) -> Result { + fn bytes_encode(item: &Self::EItem) -> Result { let mut buf = [0; size_of::<$native>()]; O::$write_method(&mut buf, *item); Ok(buf) diff --git a/heed-types/src/serde_bincode.rs b/heed-types/src/serde_bincode.rs index 0b027f1e..a55a0d03 100644 --- a/heed-types/src/serde_bincode.rs +++ b/heed-types/src/serde_bincode.rs @@ -1,4 +1,4 @@ -use heed_traits::{BoxedError, BytesDecode, ToBytes}; +use heed_traits::{BoxedError, BytesDecode, BytesEncode}; use serde::{Deserialize, Serialize}; /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `bincode` to do so. @@ -6,17 +6,17 @@ use serde::{Deserialize, Serialize}; /// It can borrow bytes from the original slice. pub struct SerdeBincode(std::marker::PhantomData); -impl<'a, T: 'a> ToBytes<'a> for SerdeBincode +impl<'a, T: 'a> BytesEncode<'a> for SerdeBincode where T: Serialize, { - type SelfType = T; + type EItem = T; type ReturnBytes = Vec; type Error = bincode::Error; - fn to_bytes(item: &Self::SelfType) -> Result { + fn bytes_encode(item: &Self::EItem) -> Result { bincode::serialize(item) } } diff --git a/heed-types/src/serde_json.rs b/heed-types/src/serde_json.rs index 95531890..3d58421a 100644 --- a/heed-types/src/serde_json.rs +++ b/heed-types/src/serde_json.rs @@ -1,4 +1,4 @@ -use heed_traits::{BoxedError, BytesDecode, ToBytes}; +use heed_traits::{BoxedError, BytesDecode, BytesEncode}; use serde::{Deserialize, Serialize}; /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `serde_json` to do so. @@ -6,17 +6,17 @@ use serde::{Deserialize, Serialize}; /// It can borrow bytes from the original slice. pub struct SerdeJson(std::marker::PhantomData); -impl<'a, T: 'a> ToBytes<'a> for SerdeJson +impl<'a, T: 'a> BytesEncode<'a> for SerdeJson where T: Serialize, { - type SelfType = T; + type EItem = T; type ReturnBytes = Vec; type Error = serde_json::Error; - fn to_bytes(item: &Self::SelfType) -> Result { + fn bytes_encode(item: &Self::EItem) -> Result { serde_json::to_vec(item) } } diff --git a/heed-types/src/serde_rmp.rs b/heed-types/src/serde_rmp.rs index 15698689..94e0c9f7 100644 --- a/heed-types/src/serde_rmp.rs +++ b/heed-types/src/serde_rmp.rs @@ -1,4 +1,4 @@ -use heed_traits::{BoxedError, BytesDecode, ToBytes}; +use heed_traits::{BoxedError, BytesDecode, BytesEncode}; use serde::{Deserialize, Serialize}; /// Describes a type that is [`Serialize`]/[`Deserialize`] and uses `rmp_serde` to do so. @@ -6,17 +6,17 @@ use serde::{Deserialize, Serialize}; /// It can borrow bytes from the original slice. pub struct SerdeRmp(std::marker::PhantomData); -impl<'a, T: 'a> ToBytes<'a> for SerdeRmp +impl<'a, T: 'a> BytesEncode<'a> for SerdeRmp where T: Serialize, { - type SelfType = T; + type EItem = T; type ReturnBytes = Vec; type Error = rmp_serde::encode::Error; - fn to_bytes(item: &Self::SelfType) -> Result { + fn bytes_encode(item: &Self::EItem) -> Result { rmp_serde::to_vec(item) } } diff --git a/heed-types/src/str.rs b/heed-types/src/str.rs index 0a0dd30d..454ec6fd 100644 --- a/heed-types/src/str.rs +++ b/heed-types/src/str.rs @@ -1,18 +1,18 @@ use std::convert::Infallible; -use heed_traits::{BoxedError, BytesDecode, ToBytes}; +use heed_traits::{BoxedError, BytesDecode, BytesEncode}; /// Describes a [`str`]. pub enum Str {} -impl<'a> ToBytes<'a> for Str { - type SelfType = str; +impl<'a> BytesEncode<'a> for Str { + type EItem = str; type ReturnBytes = &'a [u8]; type Error = Infallible; - fn to_bytes(item: &'a Self::SelfType) -> Result { + fn bytes_encode(item: &'a Self::EItem) -> Result { Ok(item.as_bytes()) } } diff --git a/heed-types/src/unit.rs b/heed-types/src/unit.rs index 97bebe3d..a644ca4c 100644 --- a/heed-types/src/unit.rs +++ b/heed-types/src/unit.rs @@ -1,19 +1,19 @@ use std::convert::Infallible; use std::{error, fmt}; -use heed_traits::{BoxedError, BytesDecode, ToBytes}; +use heed_traits::{BoxedError, BytesDecode, BytesEncode}; /// Describes the unit `()` type. pub enum Unit {} -impl ToBytes<'_> for Unit { - type SelfType = (); +impl BytesEncode<'_> for Unit { + type EItem = (); type ReturnBytes = [u8; 0]; type Error = Infallible; - fn to_bytes(&(): &Self::SelfType) -> Result { + fn bytes_encode(&(): &Self::EItem) -> Result { Ok([]) } } diff --git a/heed/src/cookbook.rs b/heed/src/cookbook.rs index 4bae4d50..cb3b8e63 100644 --- a/heed/src/cookbook.rs +++ b/heed/src/cookbook.rs @@ -146,14 +146,13 @@ //! to create codecs to encode prefixes when possible instead of using a slice of bytes. //! //! ``` -//! use std::borrow::Cow; //! use std::convert::Infallible; //! use std::error::Error; //! use std::fs; //! use std::path::Path; //! //! use heed::types::*; -//! use heed::{BoxedError, BytesDecode, ToBytes, Database, EnvOpenOptions}; +//! use heed::{BoxedError, BytesDecode, BytesEncode, Database, EnvOpenOptions}; //! //! #[derive(Debug, Clone, Copy, PartialEq, Eq)] //! pub enum Level { @@ -170,15 +169,15 @@ //! //! pub struct LogKeyCodec; //! -//! impl<'a> ToBytes<'a> for LogKeyCodec { -//! type SelfType = LogKey; +//! impl<'a> BytesEncode<'a> for LogKeyCodec { +//! type EItem = LogKey; //! //! type ReturnBytes = [u8; 5]; //! //! type Error = Infallible; //! //! /// Encodes the u32 timestamp in big endian followed by the log level with a single byte. -//! fn to_bytes(log: &'a Self::SelfType) -> Result { +//! fn bytes_encode(log: &Self::EItem) -> Result { //! let mut output = [0; 5]; //! //! let [timestamp @ .., level] = &mut output; @@ -218,15 +217,15 @@ //! /// the logs that appeared during a, rather long, period. //! pub struct LogAtHalfTimestampCodec; //! -//! impl<'a> ToBytes<'a> for LogAtHalfTimestampCodec { -//! type SelfType = u32; +//! impl<'a> BytesEncode<'a> for LogAtHalfTimestampCodec { +//! type EItem = u32; //! //! type ReturnBytes = [u8; 2]; //! //! type Error = Infallible; //! //! /// This method encodes only the prefix of the keys in this particular case, the timestamp. -//! fn to_bytes(half_timestamp: &'a Self::SelfType) -> Result { +//! fn bytes_encode(half_timestamp: &Self::EItem) -> Result { //! let [bytes @ .., _, _] = half_timestamp.to_be_bytes(); //! Ok(bytes) //! } @@ -457,4 +456,4 @@ // To let cargo generate doc links #![allow(unused_imports)] -use crate::{BytesDecode, Database, EnvOpenOptions, ToBytes}; +use crate::{BytesDecode, BytesEncode, Database, EnvOpenOptions}; diff --git a/heed/src/database.rs b/heed/src/database.rs index 0889f4d7..a2e303b2 100644 --- a/heed/src/database.rs +++ b/heed/src/database.rs @@ -1,7 +1,7 @@ use std::ops::{Bound, RangeBounds}; use std::{any, fmt, marker, mem, ptr}; -use heed_traits::{Comparator, LexicographicComparator, ToBytes}; +use heed_traits::{BytesEncode, Comparator, LexicographicComparator}; use types::{DecodeIgnore, LazyDecode}; use crate::cursor::MoveOperation; @@ -338,18 +338,14 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - pub fn get<'a, 'txn>( - &self, - txn: &'txn RoTxn, - key: &'a KC::SelfType, - ) -> Result> + pub fn get<'a, 'txn>(&self, txn: &'txn RoTxn, key: &'a KC::EItem) -> Result> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut data_val = mem::MaybeUninit::uninit(); @@ -424,15 +420,15 @@ impl Database { pub fn get_duplicates<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::SelfType, + key: &'a KC::EItem, ) -> Result>> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; if cursor.move_on_key(key_bytes.as_ref())? { Ok(Some(RoIter::new(cursor))) } else { @@ -487,16 +483,16 @@ impl Database { pub fn get_lower_than<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::SelfType, + key: &'a KC::EItem, ) -> Result> where - KC: ToBytes<'a> + BytesDecode<'txn>, + KC: BytesEncode<'a> + BytesDecode<'txn>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; cursor.move_on_key_greater_than_or_equal_to(key_bytes.as_ref())?; match cursor.move_on_prev(MoveOperation::NoDup) { @@ -556,16 +552,16 @@ impl Database { pub fn get_lower_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::SelfType, + key: &'a KC::EItem, ) -> Result> where - KC: ToBytes<'a> + BytesDecode<'txn>, + KC: BytesEncode<'a> + BytesDecode<'txn>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; let key_bytes = key_bytes.as_ref(); let result = match cursor.move_on_key_greater_than_or_equal_to(key_bytes) { Ok(Some((key, data))) if key == key_bytes => Ok(Some((key, data))), @@ -630,16 +626,16 @@ impl Database { pub fn get_greater_than<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::SelfType, + key: &'a KC::EItem, ) -> Result> where - KC: ToBytes<'a> + BytesDecode<'txn>, + KC: BytesEncode<'a> + BytesDecode<'txn>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; let key_bytes = key_bytes.as_ref(); let entry = match cursor.move_on_key_greater_than_or_equal_to(key_bytes)? { Some((key, data)) if key > key_bytes => Some((key, data)), @@ -703,16 +699,16 @@ impl Database { pub fn get_greater_than_or_equal_to<'a, 'txn>( &self, txn: &'txn RoTxn, - key: &'a KC::SelfType, + key: &'a KC::EItem, ) -> Result> where - KC: ToBytes<'a> + BytesDecode<'txn>, + KC: BytesEncode<'a> + BytesDecode<'txn>, DC: BytesDecode<'txn>, { assert_eq_env_db_txn!(self, txn); let mut cursor = RoCursor::new(txn, self.dbi)?; - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; match cursor.move_on_key_greater_than_or_equal_to(key_bytes.as_ref()) { Ok(Some((key, data))) => match (KC::bytes_decode(key), DC::bytes_decode(data)) { (Ok(key), Ok(data)) => Ok(Some((key, data))), @@ -1230,19 +1226,21 @@ impl Database { range: &'a R, ) -> Result> where - KC: ToBytes<'a>, - R: RangeBounds, + KC: BytesEncode<'a>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); // TODO optimize, this might do unnecessary allocations on types that are already 'static let start_bound = match range.start_bound() { Bound::Included(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, @@ -1250,11 +1248,13 @@ impl Database { let end_bound = match range.end_bound() { Bound::Included(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, @@ -1323,19 +1323,21 @@ impl Database { range: &'a R, ) -> Result> where - KC: ToBytes<'a>, - R: RangeBounds, + KC: BytesEncode<'a>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); // TODO optimize, this might do unnecessary allocations on types that are already 'static let start_bound = match range.start_bound() { Bound::Included(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, @@ -1343,11 +1345,13 @@ impl Database { let end_bound = match range.end_bound() { Bound::Included(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, @@ -1406,19 +1410,21 @@ impl Database { range: &'a R, ) -> Result> where - KC: ToBytes<'a>, - R: RangeBounds, + KC: BytesEncode<'a>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); // TODO optimize, this might do unnecessary allocations on types that are already 'static let start_bound = match range.start_bound() { Bound::Included(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, @@ -1426,11 +1432,13 @@ impl Database { let end_bound = match range.end_bound() { Bound::Included(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, @@ -1499,19 +1507,21 @@ impl Database { range: &'a R, ) -> Result> where - KC: ToBytes<'a>, - R: RangeBounds, + KC: BytesEncode<'a>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); // TODO optimize, this might do unnecessary allocations on types that are already 'static let start_bound = match range.start_bound() { Bound::Included(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, @@ -1519,11 +1529,13 @@ impl Database { let end_bound = match range.end_bound() { Bound::Included(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Included(bytes.into()) } Bound::Excluded(bound) => { - let bytes = KC::to_bytes(bound).map_err(|err| Error::Encoding(Box::new(err)))?; + let bytes = + KC::bytes_encode(bound).map_err(|err| Error::Encoding(Box::new(err)))?; Bound::Excluded(bytes.into()) } Bound::Unbounded => Bound::Unbounded, @@ -1580,15 +1592,16 @@ impl Database { pub fn prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, - prefix: &'a KC::SelfType, + prefix: &'a KC::EItem, ) -> Result> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, C: LexicographicComparator, { assert_eq_env_db_txn!(self, txn); - let prefix_bytes = KC::to_bytes(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; + let prefix_bytes = + KC::bytes_encode(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; // TODO optimize, this might do unnecessary allocations on types that are already 'static let prefix_bytes = prefix_bytes.into(); RoCursor::new(txn, self.dbi).map(|cursor| RoPrefix::new(cursor, prefix_bytes)) @@ -1652,15 +1665,16 @@ impl Database { pub fn prefix_iter_mut<'a, 'txn>( &self, txn: &'txn mut RwTxn, - prefix: &'a KC::SelfType, + prefix: &'a KC::EItem, ) -> Result> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, C: LexicographicComparator, { assert_eq_env_db_txn!(self, txn); - let prefix_bytes = KC::to_bytes(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; + let prefix_bytes = + KC::bytes_encode(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; // TODO optimize, this might do unnecessary allocations on types that are already 'static let prefix_bytes = prefix_bytes.into(); RwCursor::new(txn, self.dbi).map(|cursor| RwPrefix::new(cursor, prefix_bytes)) @@ -1714,15 +1728,16 @@ impl Database { pub fn rev_prefix_iter<'a, 'txn>( &self, txn: &'txn RoTxn, - prefix: &'a KC::SelfType, + prefix: &'a KC::EItem, ) -> Result> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, C: LexicographicComparator, { assert_eq_env_db_txn!(self, txn); - let prefix_bytes = KC::to_bytes(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; + let prefix_bytes = + KC::bytes_encode(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; // TODO optimize, this might do unnecessary allocations on types that are already 'static let prefix_bytes = prefix_bytes.into(); RoCursor::new(txn, self.dbi).map(|cursor| RoRevPrefix::new(cursor, prefix_bytes)) @@ -1786,15 +1801,16 @@ impl Database { pub fn rev_prefix_iter_mut<'a, 'txn>( &self, txn: &'txn mut RwTxn, - prefix: &'a KC::SelfType, + prefix: &'a KC::EItem, ) -> Result> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, C: LexicographicComparator, { assert_eq_env_db_txn!(self, txn); - let prefix_bytes = KC::to_bytes(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; + let prefix_bytes = + KC::bytes_encode(prefix).map_err(|err| Error::Encoding(Box::new(err)))?; // TODO optimize, this might do unnecessary allocations on types that are already 'static let prefix_bytes = prefix_bytes.into(); RwCursor::new(txn, self.dbi).map(|cursor| RwRevPrefix::new(cursor, prefix_bytes)) @@ -1835,20 +1851,15 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - pub fn put<'a>( - &self, - txn: &mut RwTxn, - key: &'a KC::SelfType, - data: &'a DC::SelfType, - ) -> Result<()> + pub fn put<'a>(&self, txn: &mut RwTxn, key: &'a KC::EItem, data: &'a DC::EItem) -> Result<()> where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut data_val = unsafe { crate::into_val(data_bytes.as_ref()) }; @@ -1900,17 +1911,17 @@ impl Database { pub fn put_reserved<'a, F>( &self, txn: &mut RwTxn, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result<()> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { assert_eq_env_db_txn!(self, txn); - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut reserved = ffi::reserve_size_val(data_size); let flags = ffi::MDB_RESERVE; @@ -1991,17 +2002,17 @@ impl Database { &self, txn: &mut RwTxn, flags: PutFlags, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result<()> where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut data_val = unsafe { crate::into_val(data_bytes.as_ref()) }; @@ -2050,12 +2061,12 @@ impl Database { pub fn get_or_put<'a, 'txn>( &'txn self, txn: &mut RwTxn, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result> where - KC: ToBytes<'a>, - DC: ToBytes<'a> + BytesDecode<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a> + BytesDecode<'a>, { self.get_or_put_with_flags(txn, PutFlags::empty(), key, data) } @@ -2098,17 +2109,17 @@ impl Database { &'txn self, txn: &mut RwTxn, flags: PutFlags, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result> where - KC: ToBytes<'a>, - DC: ToBytes<'a> + BytesDecode<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a> + BytesDecode<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut data_val = unsafe { crate::into_val(data_bytes.as_ref()) }; @@ -2182,12 +2193,12 @@ impl Database { pub fn get_or_put_reserved<'a, 'txn, F>( &'txn self, txn: &mut RwTxn, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, DC: BytesDecode<'a>, { @@ -2247,18 +2258,18 @@ impl Database { &'txn self, txn: &mut RwTxn, flags: PutFlags, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result> where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, DC: BytesDecode<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut reserved = ffi::reserve_size_val(data_size); @@ -2332,13 +2343,13 @@ impl Database { /// wtxn.commit()?; /// # Ok(()) } /// ``` - pub fn delete<'a>(&self, txn: &mut RwTxn, key: &'a KC::SelfType) -> Result + pub fn delete<'a>(&self, txn: &mut RwTxn, key: &'a KC::EItem) -> Result where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let result = unsafe { @@ -2414,17 +2425,17 @@ impl Database { pub fn delete_one_duplicate<'a>( &self, txn: &mut RwTxn, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { assert_eq_env_db_txn!(self, txn); - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; let mut key_val = unsafe { crate::into_val(key_bytes.as_ref()) }; let mut data_val = unsafe { crate::into_val(data_bytes.as_ref()) }; @@ -2489,8 +2500,8 @@ impl Database { /// ``` pub fn delete_range<'a, 'txn, R>(&self, txn: &'txn mut RwTxn, range: &'a R) -> Result where - KC: ToBytes<'a> + BytesDecode<'txn>, - R: RangeBounds, + KC: BytesEncode<'a> + BytesDecode<'txn>, + R: RangeBounds, { assert_eq_env_db_txn!(self, txn); diff --git a/heed/src/iterator/iter.rs b/heed/src/iterator/iter.rs index 95bf1aa6..a8da8198 100644 --- a/heed/src/iterator/iter.rs +++ b/heed/src/iterator/iter.rs @@ -1,6 +1,6 @@ use std::marker; -use heed_traits::ToBytes; +use heed_traits::BytesEncode; use types::LazyDecode; use crate::iteration_method::{IterationMethod, MoveBetweenKeys, MoveThroughDuplicateValues}; @@ -265,15 +265,15 @@ impl<'txn, KC, DC, IM> RwIter<'txn, KC, DC, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } @@ -293,15 +293,15 @@ impl<'txn, KC, DC, IM> RwIter<'txn, KC, DC, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_reserved_with_flags( flags, key_bytes.as_ref(), @@ -330,15 +330,15 @@ impl<'txn, KC, DC, IM> RwIter<'txn, KC, DC, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, - data: &'a NDC::SelfType, + key: &'a KC::EItem, + data: &'a NDC::EItem, ) -> Result<()> where - KC: ToBytes<'a>, - NDC: ToBytes<'a>, + KC: BytesEncode<'a>, + NDC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } @@ -619,15 +619,15 @@ impl<'txn, KC, DC, IM> RwRevIter<'txn, KC, DC, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } @@ -647,15 +647,15 @@ impl<'txn, KC, DC, IM> RwRevIter<'txn, KC, DC, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_reserved_with_flags( flags, key_bytes.as_ref(), @@ -684,15 +684,15 @@ impl<'txn, KC, DC, IM> RwRevIter<'txn, KC, DC, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, - data: &'a NDC::SelfType, + key: &'a KC::EItem, + data: &'a NDC::EItem, ) -> Result<()> where - KC: ToBytes<'a>, - NDC: ToBytes<'a>, + KC: BytesEncode<'a>, + NDC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } diff --git a/heed/src/iterator/prefix.rs b/heed/src/iterator/prefix.rs index 15fb513a..5d85a828 100644 --- a/heed/src/iterator/prefix.rs +++ b/heed/src/iterator/prefix.rs @@ -257,15 +257,15 @@ impl<'txn, KC, DC, C, IM> RwPrefix<'txn, KC, DC, C, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } @@ -285,15 +285,15 @@ impl<'txn, KC, DC, C, IM> RwPrefix<'txn, KC, DC, C, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_reserved_with_flags( flags, key_bytes.as_ref(), @@ -322,15 +322,15 @@ impl<'txn, KC, DC, C, IM> RwPrefix<'txn, KC, DC, C, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, - data: &'a NDC::SelfType, + key: &'a KC::EItem, + data: &'a NDC::EItem, ) -> Result<()> where - KC: ToBytes<'a>, - NDC: ToBytes<'a>, + KC: BytesEncode<'a>, + NDC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } @@ -652,15 +652,15 @@ impl<'txn, KC, DC, C, IM> RwRevPrefix<'txn, KC, DC, C, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } @@ -680,15 +680,15 @@ impl<'txn, KC, DC, C, IM> RwRevPrefix<'txn, KC, DC, C, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_reserved_with_flags( flags, key_bytes.as_ref(), @@ -717,15 +717,15 @@ impl<'txn, KC, DC, C, IM> RwRevPrefix<'txn, KC, DC, C, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, - data: &'a NDC::SelfType, + key: &'a KC::EItem, + data: &'a NDC::EItem, ) -> Result<()> where - KC: ToBytes<'a>, - NDC: ToBytes<'a>, + KC: BytesEncode<'a>, + NDC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } diff --git a/heed/src/iterator/range.rs b/heed/src/iterator/range.rs index 05adf154..cdd5751a 100644 --- a/heed/src/iterator/range.rs +++ b/heed/src/iterator/range.rs @@ -266,15 +266,15 @@ impl<'txn, KC, DC, IM> RwRange<'txn, KC, DC, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } @@ -294,15 +294,15 @@ impl<'txn, KC, DC, IM> RwRange<'txn, KC, DC, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_reserved_with_flags( flags, key_bytes.as_ref(), @@ -331,15 +331,15 @@ impl<'txn, KC, DC, IM> RwRange<'txn, KC, DC, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, - data: &'a NDC::SelfType, + key: &'a KC::EItem, + data: &'a NDC::EItem, ) -> Result<()> where - KC: ToBytes<'a>, - NDC: ToBytes<'a>, + KC: BytesEncode<'a>, + NDC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } @@ -708,15 +708,15 @@ impl<'txn, KC, DC, IM> RwRevRange<'txn, KC, DC, IM> { /// [undefined behavior]: https://doc.rust-lang.org/reference/behavior-considered-undefined.html pub unsafe fn put_current<'a>( &mut self, - key: &'a KC::SelfType, - data: &'a DC::SelfType, + key: &'a KC::EItem, + data: &'a DC::EItem, ) -> Result where - KC: ToBytes<'a>, - DC: ToBytes<'a>, + KC: BytesEncode<'a>, + DC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = DC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = DC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current(key_bytes.as_ref(), data_bytes.as_ref()) } @@ -736,15 +736,15 @@ impl<'txn, KC, DC, IM> RwRevRange<'txn, KC, DC, IM> { pub unsafe fn put_current_reserved_with_flags<'a, F>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, + key: &'a KC::EItem, data_size: usize, write_func: F, ) -> Result where - KC: ToBytes<'a>, + KC: BytesEncode<'a>, F: FnOnce(&mut ReservedSpace) -> io::Result<()>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_reserved_with_flags( flags, key_bytes.as_ref(), @@ -773,15 +773,15 @@ impl<'txn, KC, DC, IM> RwRevRange<'txn, KC, DC, IM> { pub unsafe fn put_current_with_options<'a, NDC>( &mut self, flags: PutFlags, - key: &'a KC::SelfType, - data: &'a NDC::SelfType, + key: &'a KC::EItem, + data: &'a NDC::EItem, ) -> Result<()> where - KC: ToBytes<'a>, - NDC: ToBytes<'a>, + KC: BytesEncode<'a>, + NDC: BytesEncode<'a>, { - let key_bytes = KC::to_bytes(key).map_err(|err| Error::Encoding(Box::new(err)))?; - let data_bytes = NDC::to_bytes(data).map_err(|err| Error::Encoding(Box::new(err)))?; + let key_bytes = KC::bytes_encode(key).map_err(|err| Error::Encoding(Box::new(err)))?; + let data_bytes = NDC::bytes_encode(data).map_err(|err| Error::Encoding(Box::new(err)))?; self.cursor.put_current_with_flags(flags, key_bytes.as_ref(), data_bytes.as_ref()) } diff --git a/heed/src/lib.rs b/heed/src/lib.rs index 950b6ec1..83a5d15d 100644 --- a/heed/src/lib.rs +++ b/heed/src/lib.rs @@ -92,7 +92,7 @@ pub use self::mdb::error::Error as MdbError; use self::mdb::ffi::{from_val, into_val}; pub use self::mdb::flags::{DatabaseFlags, EnvFlags, PutFlags}; pub use self::reserved_space::ReservedSpace; -pub use self::traits::{BoxedError, BytesDecode, Comparator, LexicographicComparator, ToBytes}; +pub use self::traits::{BoxedError, BytesDecode, BytesEncode, Comparator, LexicographicComparator}; pub use self::txn::{RoTxn, RwTxn}; /// The underlying LMDB library version information. From 620152269d4bac4390943946e8f61bfd10776eb7 Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Sun, 18 Aug 2024 19:56:53 +0200 Subject: [PATCH 06/10] allow all std::io::Write impls in bytes_encode_into_writer instead of only Vec, use Either for the 2 different errors --- heed-traits/Cargo.toml | 1 + heed-traits/src/lib.rs | 28 +++++++++++++++++++++------- 2 files changed, 22 insertions(+), 7 deletions(-) diff --git a/heed-traits/Cargo.toml b/heed-traits/Cargo.toml index 3745a08c..c80501c8 100644 --- a/heed-traits/Cargo.toml +++ b/heed-traits/Cargo.toml @@ -9,3 +9,4 @@ readme = "../README.md" edition = "2021" [dependencies] +either = "1.13.0" diff --git a/heed-traits/src/lib.rs b/heed-traits/src/lib.rs index 06acfa93..5e87a2dc 100644 --- a/heed-traits/src/lib.rs +++ b/heed-traits/src/lib.rs @@ -11,6 +11,9 @@ use std::cmp::{Ord, Ordering}; use std::error::Error as StdError; +use std::io; + +use either::Either; /// A boxed `Send + Sync + 'static` error. pub type BoxedError = Box; @@ -29,14 +32,25 @@ pub trait BytesEncode<'a> { /// Encode the given item as bytes. fn bytes_encode(item: &'a Self::EItem) -> Result; - /// Encode the given item as bytes and write it into the writer. This function by default - /// forwards to `bytes_encode`. - fn bytes_encode_into_writer( + /// Encode the given item as bytes and write it into the writer. Returns the amount of bytes + /// that were written. This function by default forwards to + /// [`bytes_encode`][BytesEncode::bytes_encode]. + /// + /// # Errors + /// + /// [`Either`] is used to handle the 2 different errors this function can return. + /// [`Either::Left`] is used for errors from [`bytes_encode`][BytesEncode::bytes_encode] and + /// [`Either::Right`] is used for errors from the writer (I/O errors). + fn bytes_encode_into_writer( item: &'a Self::EItem, - writer: &mut Vec, - ) -> Result<(), Self::Error> { - writer.extend_from_slice(Self::bytes_encode(item)?.as_ref()); - Ok(()) + writer: &mut W, + ) -> Result> { + let bytes = Self::bytes_encode(item).map_err(Either::Left)?; + let bytes = bytes.as_ref(); + + writer.write_all(bytes).map_err(Either::Right)?; + + Ok(bytes.len()) } } From 562797cf1c272eded71af001c6100ffb7730e86f Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Tue, 20 Aug 2024 22:53:41 +0200 Subject: [PATCH 07/10] return BoxedError from bytes_encode_into_writer for now --- heed-traits/Cargo.toml | 1 - heed-traits/src/lib.rs | 14 +++----------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/heed-traits/Cargo.toml b/heed-traits/Cargo.toml index c80501c8..3745a08c 100644 --- a/heed-traits/Cargo.toml +++ b/heed-traits/Cargo.toml @@ -9,4 +9,3 @@ readme = "../README.md" edition = "2021" [dependencies] -either = "1.13.0" diff --git a/heed-traits/src/lib.rs b/heed-traits/src/lib.rs index 5e87a2dc..c8ea5099 100644 --- a/heed-traits/src/lib.rs +++ b/heed-traits/src/lib.rs @@ -13,8 +13,6 @@ use std::cmp::{Ord, Ordering}; use std::error::Error as StdError; use std::io; -use either::Either; - /// A boxed `Send + Sync + 'static` error. pub type BoxedError = Box; @@ -35,20 +33,14 @@ pub trait BytesEncode<'a> { /// Encode the given item as bytes and write it into the writer. Returns the amount of bytes /// that were written. This function by default forwards to /// [`bytes_encode`][BytesEncode::bytes_encode]. - /// - /// # Errors - /// - /// [`Either`] is used to handle the 2 different errors this function can return. - /// [`Either::Left`] is used for errors from [`bytes_encode`][BytesEncode::bytes_encode] and - /// [`Either::Right`] is used for errors from the writer (I/O errors). fn bytes_encode_into_writer( item: &'a Self::EItem, writer: &mut W, - ) -> Result> { - let bytes = Self::bytes_encode(item).map_err(Either::Left)?; + ) -> Result { + let bytes = Self::bytes_encode(item)?; let bytes = bytes.as_ref(); - writer.write_all(bytes).map_err(Either::Right)?; + writer.write_all(bytes)?; Ok(bytes.len()) } From 147c6245a6bf36335c88df72ae86825037c03dfa Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Tue, 20 Aug 2024 23:08:44 +0200 Subject: [PATCH 08/10] add zero_copy method --- heed-traits/src/lib.rs | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/heed-traits/src/lib.rs b/heed-traits/src/lib.rs index c8ea5099..a6dcc344 100644 --- a/heed-traits/src/lib.rs +++ b/heed-traits/src/lib.rs @@ -27,12 +27,33 @@ pub trait BytesEncode<'a> { /// The error type to return when decoding goes wrong. type Error: StdError + Send + Sync + 'static; + /// This function can be used to hint callers of the + /// [`bytes_encode`][BytesEncode::bytes_encode] function to use + /// [`bytes_encode_into_writer`][BytesEncode::bytes_encode_into_writer] instead, if the latter + /// runs faster (for example if it needs less heap allocations). + /// + /// The default implementation returns `true` because the default implementation of + /// [`bytes_encode_into_writer`][BytesEncode::bytes_encode_into_writer] is to forward to + /// [`bytes_encode`][BytesEncode::bytes_encode]. + fn zero_copy(item: &Self::EItem) -> bool { + // This is preferred to renaming the function parameter (to _item) because IDEs can + // autofill trait implementations, which will default the paramter name to _item then and + // this could probably also mess with clippy's renamed_function_params lint. + let _ = item; + + true + } + /// Encode the given item as bytes. fn bytes_encode(item: &'a Self::EItem) -> Result; /// Encode the given item as bytes and write it into the writer. Returns the amount of bytes - /// that were written. This function by default forwards to - /// [`bytes_encode`][BytesEncode::bytes_encode]. + /// that were written. + /// + /// When implementing this, also take a look at [`zero_copy`][BytesEncode::zero_copy]'s + /// documentation. + /// + /// The default implementation forwards to [`bytes_encode`][BytesEncode::bytes_encode]. fn bytes_encode_into_writer( item: &'a Self::EItem, writer: &mut W, From f6a4a054139658c1df5e6ce44c133a4b23ff3d16 Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Tue, 20 Aug 2024 23:21:11 +0200 Subject: [PATCH 09/10] hint to using writers for serde serialization types, remove requirement of returning written bytes amount --- heed-traits/src/lib.rs | 10 ++++------ heed-types/src/serde_bincode.rs | 12 ++++++++++++ heed-types/src/serde_json.rs | 12 ++++++++++++ heed-types/src/serde_rmp.rs | 12 ++++++++++++ 4 files changed, 40 insertions(+), 6 deletions(-) diff --git a/heed-traits/src/lib.rs b/heed-traits/src/lib.rs index a6dcc344..50efad2c 100644 --- a/heed-traits/src/lib.rs +++ b/heed-traits/src/lib.rs @@ -47,8 +47,7 @@ pub trait BytesEncode<'a> { /// Encode the given item as bytes. fn bytes_encode(item: &'a Self::EItem) -> Result; - /// Encode the given item as bytes and write it into the writer. Returns the amount of bytes - /// that were written. + /// Encode the given item as bytes and write it into the writer. /// /// When implementing this, also take a look at [`zero_copy`][BytesEncode::zero_copy]'s /// documentation. @@ -57,13 +56,12 @@ pub trait BytesEncode<'a> { fn bytes_encode_into_writer( item: &'a Self::EItem, writer: &mut W, - ) -> Result { + ) -> Result<(), BoxedError> { let bytes = Self::bytes_encode(item)?; - let bytes = bytes.as_ref(); - writer.write_all(bytes)?; + writer.write_all(bytes.as_ref())?; - Ok(bytes.len()) + Ok(()) } } diff --git a/heed-types/src/serde_bincode.rs b/heed-types/src/serde_bincode.rs index a55a0d03..dfb555d9 100644 --- a/heed-types/src/serde_bincode.rs +++ b/heed-types/src/serde_bincode.rs @@ -16,9 +16,21 @@ where type Error = bincode::Error; + fn zero_copy(_item: &Self::EItem) -> bool { + false + } + fn bytes_encode(item: &Self::EItem) -> Result { bincode::serialize(item) } + + fn bytes_encode_into_writer( + item: &'a Self::EItem, + writer: &mut W, + ) -> Result<(), BoxedError> { + bincode::serialize_into(writer, item)?; + Ok(()) + } } impl<'a, T: 'a> BytesDecode<'a> for SerdeBincode diff --git a/heed-types/src/serde_json.rs b/heed-types/src/serde_json.rs index 3d58421a..b093b147 100644 --- a/heed-types/src/serde_json.rs +++ b/heed-types/src/serde_json.rs @@ -16,9 +16,21 @@ where type Error = serde_json::Error; + fn zero_copy(_item: &Self::EItem) -> bool { + false + } + fn bytes_encode(item: &Self::EItem) -> Result { serde_json::to_vec(item) } + + fn bytes_encode_into_writer( + item: &'a Self::EItem, + writer: &mut W, + ) -> Result<(), BoxedError> { + serde_json::to_writer(writer, item)?; + Ok(()) + } } impl<'a, T: 'a> BytesDecode<'a> for SerdeJson diff --git a/heed-types/src/serde_rmp.rs b/heed-types/src/serde_rmp.rs index 94e0c9f7..f866519b 100644 --- a/heed-types/src/serde_rmp.rs +++ b/heed-types/src/serde_rmp.rs @@ -16,9 +16,21 @@ where type Error = rmp_serde::encode::Error; + fn zero_copy(_item: &Self::EItem) -> bool { + false + } + fn bytes_encode(item: &Self::EItem) -> Result { rmp_serde::to_vec(item) } + + fn bytes_encode_into_writer( + item: &'a Self::EItem, + writer: &mut W, + ) -> Result<(), BoxedError> { + rmp_serde::encode::write(writer, item)?; + Ok(()) + } } impl<'a, T: 'a> BytesDecode<'a> for SerdeRmp From 3519c51aa9478268c5200b66f6089f1c06109898 Mon Sep 17 00:00:00 2001 From: Antoni Spaanderman <56turtle56@gmail.com> Date: Tue, 20 Aug 2024 23:23:06 +0200 Subject: [PATCH 10/10] allow any writer, not only mutable references to writers --- heed-traits/src/lib.rs | 2 +- heed-types/src/serde_bincode.rs | 2 +- heed-types/src/serde_json.rs | 2 +- heed-types/src/serde_rmp.rs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/heed-traits/src/lib.rs b/heed-traits/src/lib.rs index 50efad2c..adf0c14a 100644 --- a/heed-traits/src/lib.rs +++ b/heed-traits/src/lib.rs @@ -55,7 +55,7 @@ pub trait BytesEncode<'a> { /// The default implementation forwards to [`bytes_encode`][BytesEncode::bytes_encode]. fn bytes_encode_into_writer( item: &'a Self::EItem, - writer: &mut W, + mut writer: W, ) -> Result<(), BoxedError> { let bytes = Self::bytes_encode(item)?; diff --git a/heed-types/src/serde_bincode.rs b/heed-types/src/serde_bincode.rs index dfb555d9..d65ce05f 100644 --- a/heed-types/src/serde_bincode.rs +++ b/heed-types/src/serde_bincode.rs @@ -26,7 +26,7 @@ where fn bytes_encode_into_writer( item: &'a Self::EItem, - writer: &mut W, + writer: W, ) -> Result<(), BoxedError> { bincode::serialize_into(writer, item)?; Ok(()) diff --git a/heed-types/src/serde_json.rs b/heed-types/src/serde_json.rs index b093b147..80624ff2 100644 --- a/heed-types/src/serde_json.rs +++ b/heed-types/src/serde_json.rs @@ -26,7 +26,7 @@ where fn bytes_encode_into_writer( item: &'a Self::EItem, - writer: &mut W, + writer: W, ) -> Result<(), BoxedError> { serde_json::to_writer(writer, item)?; Ok(()) diff --git a/heed-types/src/serde_rmp.rs b/heed-types/src/serde_rmp.rs index f866519b..00127131 100644 --- a/heed-types/src/serde_rmp.rs +++ b/heed-types/src/serde_rmp.rs @@ -26,9 +26,9 @@ where fn bytes_encode_into_writer( item: &'a Self::EItem, - writer: &mut W, + mut writer: W, ) -> Result<(), BoxedError> { - rmp_serde::encode::write(writer, item)?; + rmp_serde::encode::write(&mut writer, item)?; Ok(()) } }