feat: adding the feature upgrade_0_5_x for auto-upgrading from version 0.5.x to the higher version 0.6.x

Author: vincent-herlemont

.gitignore

@@ -7,4 +7,5 @@ package.json
 *.expanded.rs
 
 # Related to [Why do binaries have Cargo.lock in version control, but not libraries?](https://doc.rust-lang.org/cargo/faq.html#why-do-binaries-have-cargolock-in-version-control-but-not-libraries)
-Cargo.lock
+Cargo.lock
+tests/data/db_x_x_x

Cargo.toml

@@ -16,6 +16,7 @@ members = ["native_db_macro"]
 
 [dependencies]
 redb = "2.1.0"
+redb1 = { version = "=1.5.1", package = "redb", optional = true }
 native_db_macro = { version = "0.5.3", path = "native_db_macro" }
 thiserror = "1.0"
 serde = { version = "1.0" }
@@ -42,9 +43,11 @@ uuid = { version = "1", features = ["serde", "v4"] }
 chrono = { version = "0.4", features = ["serde"] }
 rand = "0.8"
 once_cell = "1.19"
+dinghy-test = "0.7.1"
 
 [features]
-default = []
+default = [ "upgrade_0_5_x" ]
+upgrade_0_5_x = [ "redb1" ]
 
 [[bench]]
 name = "overhead_data_size"

justfile

@@ -6,6 +6,7 @@ default:
 build_no_default *args:
     cargo build --no-default-features {{args}}
 
+# E.g. just build_default --test modules breaking_release_migration::from_0_5_x_to_0_6_x
 build_default *args:
     cargo build {{args}}
 

renovate.json

@@ -5,6 +5,10 @@
   "semanticCommitScope": "deps",
   "platformAutomerge": true,
   "packageRules": [
+    {
+      "excludePackageNames": ["native_db_0_5_3", "redb1"],
+      "enabled": false
+    },
     {
       "description": "Automerge non-major updates",
       "matchUpdateTypes": ["minor", "patch"],

src/database.rs

@@ -1,4 +1,5 @@
 use crate::database_builder::ModelBuilder;
+use crate::database_instance::DatabaseInstance;
 use crate::db_type::Result;
 use crate::stats::{Stats, StatsTable};
 use crate::table_definition::PrimaryTableDefinition;
@@ -32,7 +33,7 @@ use std::u64;
 ///    Ok(())
 /// }
 pub struct Database<'a> {
-    pub(crate) instance: redb::Database,
+    pub(crate) instance: DatabaseInstance,
     pub(crate) primary_table_definitions: HashMap<String, PrimaryTableDefinition<'a>>,
     pub(crate) watchers: Arc<RwLock<watch::Watchers>>,
     pub(crate) watchers_counter_id: AtomicU64,
@@ -41,7 +42,7 @@ pub struct Database<'a> {
 impl Database<'_> {
     /// Creates a new read-write transaction.
     pub fn rw_transaction(&self) -> Result<RwTransaction> {
-        let rw = self.instance.begin_write()?;
+        let rw = self.instance.redb_database()?.begin_write()?;
         let write_txn = RwTransaction {
             watcher: &self.watchers,
             batch: RefCell::new(watch::Batch::new()),
@@ -55,7 +56,7 @@ impl Database<'_> {
 
     /// Creates a new read-only transaction.
     pub fn r_transaction(&self) -> Result<RTransaction> {
-        let txn = self.instance.begin_read()?;
+        let txn = self.instance.redb_database()?.begin_read()?;
         let read_txn = RTransaction {
             internal: InternalRTransaction {
                 redb_transaction: txn,
@@ -95,7 +96,7 @@ impl<'a> Database<'a> {
         let mut primary_table_definition: PrimaryTableDefinition =
             (model_builder, main_table_definition).into();
 
-        let rw = self.instance.begin_write()?;
+        let rw = self.instance.redb_database()?.begin_write()?;
         rw.open_table(primary_table_definition.redb.clone())?;
 
         for secondary_key in model_builder.model.secondary_keys.iter() {
@@ -120,7 +121,7 @@ impl<'a> Database<'a> {
     }
 
     pub fn redb_stats(&self) -> Result<Stats> {
-        let rx = self.instance.begin_read()?;
+        let rx = self.instance.redb_database()?.begin_read()?;
         let mut stats_primary_tables = vec![];
         for primary_table in self.primary_table_definitions.values() {
             let result_table_open = rx.open_table(primary_table.redb.clone());

src/database_builder.rs

@@ -1,30 +1,48 @@
-use crate::db_type::Result;
+use crate::database_instance::DatabaseInstance;
+use crate::db_type::{Error, Result};
 use crate::table_definition::NativeModelOptions;
-use crate::{watch, Database, DatabaseModel, Input};
+use crate::{upgrade, watch, Database, DatabaseModel, Input};
 use std::collections::HashMap;
 use std::path::Path;
 use std::sync::atomic::AtomicU64;
 use std::sync::{Arc, RwLock};
 
-/// Builder that allows you to create a [`Database`](crate::Database) instance via [`create`](Self::create) or [`open`](Self::open) etc. and [define](Self::define) models.
 #[derive(Debug)]
-pub struct DatabaseBuilder {
-    cache_size_bytes: Option<usize>,
-    models_builder: HashMap<String, ModelBuilder>,
+pub(crate) struct DatabaseConfiguration {
+    pub(crate) cache_size_bytes: Option<usize>,
 }
 
-impl DatabaseBuilder {
-    fn new_rdb_builder(&self) -> redb::Builder {
+impl DatabaseConfiguration {
+    pub(crate) fn new_rdb_builder(&self) -> redb::Builder {
         let mut redb_builder = redb::Builder::new();
         if let Some(cache_size_bytes) = self.cache_size_bytes {
             redb_builder.set_cache_size(cache_size_bytes);
         }
         redb_builder
     }
+}
 
-    fn init<'a>(&'a self, redb_database: redb::Database) -> Result<Database<'a>> {
+#[cfg(feature = "redb1")]
+impl DatabaseConfiguration {
+    pub(crate) fn redb1_new_rdb1_builder(&self) -> redb1::Builder {
+        let mut redb_builder = redb1::Builder::new();
+        if let Some(cache_size_bytes) = self.cache_size_bytes {
+            redb_builder.set_cache_size(cache_size_bytes);
+        }
+        redb_builder
+    }
+}
+/// Builder that allows you to create a [`Database`](crate::Database) instance via [`create`](Self::create) or [`open`](Self::open) etc. and [define](Self::define) models.
+#[derive(Debug)]
+pub struct DatabaseBuilder {
+    database_configuration: DatabaseConfiguration,
+    models_builder: HashMap<String, ModelBuilder>,
+}
+
+impl DatabaseBuilder {
+    fn init<'a>(&'a self, database_instance: DatabaseInstance) -> Result<Database<'a>> {
         let mut database = Database {
-            instance: redb_database,
+            instance: database_instance,
             primary_table_definitions: HashMap::new(),
             watchers: Arc::new(RwLock::new(watch::Watchers::new())),
             watchers_counter_id: AtomicU64::new(0),
@@ -42,40 +60,47 @@ impl DatabaseBuilder {
     /// Similar to [redb::Builder::new()](https://docs.rs/redb/latest/redb/struct.Builder.html#method.new).
     pub fn new() -> Self {
         Self {
-            cache_size_bytes: None,
+            database_configuration: DatabaseConfiguration {
+                cache_size_bytes: None,
+            },
             models_builder: HashMap::new(),
         }
     }
 
     /// Similar to [redb::Builder::set_cache_size()](https://docs.rs/redb/latest/redb/struct.Builder.html#method.set_cache_size).
     pub fn set_cache_size(&mut self, bytes: usize) -> &mut Self {
-        self.cache_size_bytes = Some(bytes);
+        self.database_configuration.cache_size_bytes = Some(bytes);
         self
     }
 
     /// Creates a new `Db` instance using the given path.
     ///
     /// Similar to [redb::Builder.create(...)](https://docs.rs/redb/latest/redb/struct.Builder.html#method.create)
     pub fn create(&self, path: impl AsRef<Path>) -> Result<Database> {
-        let db = self.new_rdb_builder().create(path)?;
-        // Ok(Self::from_redb(db))
-        self.init(db)
+        let builder = self.database_configuration.new_rdb_builder();
+        let database_instance = DatabaseInstance::create_on_disk(builder, path)?;
+        self.init(database_instance)
     }
 
     /// Similar to [redb::Builder::open(...)](https://docs.rs/redb/latest/redb/struct.Builder.html#method.open)
+    /// But it also upgrades the database if needed if 
     pub fn open(&self, path: impl AsRef<Path>) -> Result<Database> {
-        let db = self.new_rdb_builder().open(path)?;
-        // Ok(Self::from_redb(db))
-        self.init(db)
+        let builder = self.database_configuration.new_rdb_builder();
+        let database_instance = match DatabaseInstance::open_on_disk(builder, &path) {
+            Err(Error::RedbDatabaseError(redb::DatabaseError::UpgradeRequired(_))) => {
+                upgrade::upgrade(&self.database_configuration, &path, &self.models_builder)
+            }
+            Err(error) => return Err(error),
+            Ok(database_instance) => Ok(database_instance),
+        }?;
+        self.init(database_instance)
     }
 
     /// Creates a new [`Database`](crate::Database) instance in memory.
     pub fn create_in_memory(&self) -> Result<Database> {
-        let in_memory_backend = redb::backends::InMemoryBackend::new();
-        let db = self.new_rdb_builder();
-        let db = db.create_with_backend(in_memory_backend)?;
-        // Ok(Self::from_redb(db))
-        self.init(db)
+        let builder = self.database_configuration.new_rdb_builder();
+        let database_instance = DatabaseInstance::create_in_memory(builder)?;
+        self.init(database_instance)
     }
 
     /// Defines a table using the given model.

src/database_instance.rs

@@ -0,0 +1,65 @@
+use crate::db_type::Result;
+use redb::Builder;
+use std::path::Path;
+use std::path::PathBuf;
+
+
+pub(crate) struct DatabaseInstance {
+    kind: DatabaseInstanceKind,
+}
+
+impl DatabaseInstance {
+    pub(crate) fn create_on_disk(builder: Builder, path: impl AsRef<Path>) -> Result<Self> {
+        let db = builder.create(path.as_ref())?;
+        Ok(Self {
+            kind: DatabaseInstanceKind::OnDisk {
+                redb_database: db,
+                path: path.as_ref().to_path_buf(),
+            },
+        })
+    }
+
+    pub(crate) fn open_on_disk(builder: Builder, path: impl AsRef<Path>) -> Result<Self> {
+        let db = builder.open(path.as_ref())?;
+        Ok(Self {
+            kind: DatabaseInstanceKind::OnDisk {
+                redb_database: db,
+                path: path.as_ref().to_path_buf(),
+            },
+        })
+    }
+
+    pub(crate) fn create_in_memory(builder: Builder) -> Result<Self> {
+        let in_memory_backend = redb::backends::InMemoryBackend::new();
+        let db = builder.create_with_backend(in_memory_backend)?;
+        Ok(Self {
+            kind: DatabaseInstanceKind::InMemory {
+                redb_database: db,
+            },
+        })
+    }
+
+    pub(crate) fn redb_database(&self) -> Result<&redb::Database> {
+        self.kind.redb_database()
+    }
+}
+
+enum DatabaseInstanceKind {
+    InMemory {
+        redb_database: redb::Database,
+    },
+    OnDisk {
+        redb_database: redb::Database,
+        #[allow(dead_code)]
+        path: PathBuf,
+    }
+}
+
+impl DatabaseInstanceKind {
+    pub(crate) fn redb_database(&self) -> Result<&redb::Database> {
+        match self {
+            DatabaseInstanceKind::InMemory { redb_database } => Ok(redb_database),
+            DatabaseInstanceKind::OnDisk { redb_database, .. } => Ok(redb_database)
+        }
+    }
+}

src/db_type/error.rs

@@ -9,9 +9,17 @@ pub enum Error {
     #[error("Redb database error")]
     RedbDatabaseError(#[from] redb::DatabaseError),
 
+    #[cfg(feature = "redb1")]
+    #[error("Legacy redb1 database error")]
+    LegacyRedb1DatabaseError(#[from] redb1::DatabaseError),
+
     #[error("Redb transaction error")]
     RedbTransactionError(#[from] redb::TransactionError),
 
+    #[cfg(feature = "redb1")]
+    #[error("Redb redb1 transaction error")]
+    Redb1TransactionError(#[from] redb1::TransactionError),
+
     #[error("Redb storage error")]
     RedbStorageError(#[from] redb::StorageError),
 
@@ -21,6 +29,12 @@ pub enum Error {
     #[error("Redb commit error")]
     RedbCommitError(#[from] redb::CommitError),
 
+    #[error("Redb compaction error")]
+    RedbCompactionError(#[from] redb::CompactionError),
+
+    #[error("Database instance need upgrade")]
+    DatabaseInstanceNeedUpgrade(u8),
+
     #[error("IO error")]
     Io(#[from] std::io::Error),
 

src/db_type/key/inner_key_value.rs

@@ -6,7 +6,7 @@ use std::ops::{Bound, Range, RangeBounds, RangeFrom, RangeInclusive, RangeTo, Ra
 pub struct DatabaseInnerKeyValue(Vec<u8>);
 
 impl DatabaseInnerKeyValue {
-    fn new(data: Vec<u8>) -> Self {
+    pub(crate) fn new(data: Vec<u8>) -> Self {
         Self(data)
     }