Add SQLite KVStore implementation

bench/Cargo.toml

@@ -13,7 +13,7 @@ hashbrown = ["lightning/hashbrown"]
 
 [dependencies]
 lightning = { path = "../lightning", features = ["_test_utils", "criterion"] }
-lightning-persister = { path = "../lightning-persister", features = ["criterion"] }
+lightning-persister = { path = "../lightning-persister", features = ["criterion", "sqlite-bundled"] }
 lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync", features = ["criterion"] }
 criterion = { version = "0.4", default-features = false }
 

bench/benches/bench.rs

@@ -19,6 +19,7 @@ criterion_group!(benches,
 	lightning::sign::benches::bench_get_secure_random_bytes,
 	lightning::ln::channelmanager::bench::bench_sends,
 	lightning_persister::fs_store::bench::bench_sends,
+	lightning_persister::sqlite_store::bench::bench_sends,
 	lightning_rapid_gossip_sync::bench::bench_reading_full_graph_from_file,
 	lightning::routing::gossip::benches::read_network_graph,
 	lightning::routing::gossip::benches::write_network_graph);

lightning-persister/Cargo.toml

@@ -13,9 +13,14 @@ edition = "2018"
 all-features = true
 rustdoc-args = ["--cfg", "docsrs"]
 
+[features]
+sqlite = ["rusqlite"]
+sqlite-bundled = ["sqlite", "rusqlite/bundled"]
+
 [dependencies]
 bitcoin = "0.29.0"
 lightning = { version = "0.0.117-alpha2", path = "../lightning" }
+rusqlite = { version = "0.28.0", optional = true, default-features = false}
 
 [target.'cfg(windows)'.dependencies]
 windows-sys = { version = "0.48.0", default-features = false, features = ["Win32_Storage_FileSystem", "Win32_Foundation"] }
@@ -26,3 +31,5 @@ criterion = { version = "0.4", optional = true, default-features = false }
 [dev-dependencies]
 lightning = { version = "0.0.117-alpha2", path = "../lightning", features = ["_test_utils"] }
 bitcoin = { version = "0.29.0", default-features = false }
+rusqlite = { version = "0.28.0", default-features = false, features = ["bundled"]}
+rand = "0.8.5"

lightning-persister/src/lib.rs

@@ -12,6 +12,9 @@
 
 pub mod fs_store;
 
+#[cfg(any(test, feature = "sqlite"))]
+pub mod sqlite_store;
+
 mod utils;
 
 #[cfg(test)]

lightning-persister/src/sqlite_store/migrations.rs

@@ -0,0 +1,164 @@
+use rusqlite::Connection;
+
+use lightning::io;
+
+pub(super) fn migrate_schema(
+	connection: &mut Connection, kv_table_name: &str, from_version: u16, to_version: u16,
+) -> io::Result<()> {
+	assert!(from_version < to_version);
+	if from_version == 1 && to_version == 2 {
+		let tx = connection.transaction().map_err(|e| {
+			let msg = format!(
+				"Failed to migrate table {} from user_version {} to {}: {}",
+				kv_table_name, from_version, to_version, e
+			);
+			io::Error::new(io::ErrorKind::Other, msg)
+		})?;
+
+		// Rename 'namespace' column to 'primary_namespace'
+		let sql = format!(
+			"ALTER TABLE {}
+				RENAME COLUMN namespace TO primary_namespace;",
+			kv_table_name
+		);
+
+		tx.execute(&sql, []).map_err(|e| {
+			let msg = format!(
+				"Failed to migrate table {} from user_version {} to {}: {}",
+				kv_table_name, from_version, to_version, e
+			);
+			io::Error::new(io::ErrorKind::Other, msg)
+		})?;
+
+		// Add new 'secondary_namespace' column
+		let sql = format!(
+			"ALTER TABLE {}
+				ADD secondary_namespace TEXT DEFAULT \"\" NOT NULL;",
+			kv_table_name
+		);
+
+		tx.execute(&sql, []).map_err(|e| {
+			let msg = format!(
+				"Failed to migrate table {} from user_version {} to {}: {}",
+				kv_table_name, from_version, to_version, e
+			);
+			io::Error::new(io::ErrorKind::Other, msg)
+		})?;
+
+		// Update user_version
+		tx.pragma(Some(rusqlite::DatabaseName::Main), "user_version", to_version, |_| Ok(()))
+			.map_err(|e| {
+				let msg = format!(
+					"Failed to upgrade user_version from {} to {}: {}",
+					from_version, to_version, e
+				);
+				io::Error::new(io::ErrorKind::Other, msg)
+			})?;
+
+		tx.commit().map_err(|e| {
+			let msg = format!(
+				"Failed to migrate table {} from user_version {} to {}: {}",
+				kv_table_name, from_version, to_version, e
+			);
+			io::Error::new(io::ErrorKind::Other, msg)
+		})?;
+	}
+	Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+	use crate::sqlite_store::SqliteStore;
+	use crate::test_utils::{do_read_write_remove_list_persist, random_storage_path};
+
+	use lightning::util::persist::KVStore;
+
+	use rusqlite::{named_params, Connection};
+
+	use std::fs;
+
+	#[test]
+	fn rwrl_post_schema_1_migration() {
+		let old_schema_version = 1;
+
+		let mut temp_path = random_storage_path();
+		temp_path.push("rwrl_post_schema_1_migration");
+
+		let db_file_name = "test_db".to_string();
+		let kv_table_name = "test_table".to_string();
+
+		let test_namespace = "testspace".to_string();
+		let test_key = "testkey".to_string();
+		let test_data = [42u8; 32];
+
+		{
+			// We create a database with a SCHEMA_VERSION 1 table
+			fs::create_dir_all(temp_path.clone()).unwrap();
+			let mut db_file_path = temp_path.clone();
+			db_file_path.push(db_file_name.clone());
+
+			let connection = Connection::open(db_file_path.clone()).unwrap();
+
+			connection
+				.pragma(
+					Some(rusqlite::DatabaseName::Main),
+					"user_version",
+					old_schema_version,
+					|_| Ok(()),
+				)
+				.unwrap();
+
+			let sql = format!(
+				"CREATE TABLE IF NOT EXISTS {} (
+					namespace TEXT NOT NULL,
+					key TEXT NOT NULL CHECK (key <> ''),
+					value BLOB, PRIMARY KEY ( namespace, key )
+					);",
+				kv_table_name
+			);
+
+			connection.execute(&sql, []).unwrap();
+
+			// We write some data to to the table
+			let sql = format!(
+				"INSERT OR REPLACE INTO {} (namespace, key, value) VALUES (:namespace, :key, :value);",
+				kv_table_name
+				);
+			let mut stmt = connection.prepare_cached(&sql).unwrap();
+
+			stmt.execute(named_params! {
+				":namespace": test_namespace,
+				":key": test_key,
+				":value": test_data,
+			})
+			.unwrap();
+
+			// We read the just written data back to assert it happened.
+			let sql = format!(
+				"SELECT value FROM {} WHERE namespace=:namespace AND key=:key;",
+				kv_table_name
+			);
+			let mut stmt = connection.prepare_cached(&sql).unwrap();
+
+			let res: Vec<u8> = stmt
+				.query_row(
+					named_params! {
+						":namespace": test_namespace,
+						":key": test_key,
+					},
+					|row| row.get(0),
+				)
+				.unwrap();
+
+			assert_eq!(res, test_data);
+		}
+
+		// Check we migrate the db just fine without losing our written data.
+		let store = SqliteStore::new(temp_path, Some(db_file_name), Some(kv_table_name)).unwrap();
+		let res = store.read(&test_namespace, "", &test_key).unwrap();
+		assert_eq!(res, test_data);
+
+		// Check we can continue to use the store just fine.
+		do_read_write_remove_list_persist(&store);
+	}
+}