Merge #328 #348

328: export sqlite3 dump r=MarinPostma a=MarinPostma

This PR ports the dumping loginc from sqlite to sqld. It allows the dumping of sqld tables to a sqlite compatible format.

sqld now exposes a CLI interface to export dumps, eg:

- print the dump for the database at the default path (`data.sqld`) to stdout:
```
sqld dump
```

- print the dump for the database at the path specified by `--db-path`:
```
sqld --db-path some/path/to/database dump
```

- write the dump for the database at the path specified by `--db-path` at the path specified by path:

```
sqld --db-path some/path/to/database dump --path write/dump/here
```

`dump` is a subcommand, so any argument passed after the `dump` have dump as subject, and any argument passed before `dump` have `sqld` as subject. Any argument with `sqld` as subject other than `--db-path` is ignored.


348: whitelist specific pragmas r=MarinPostma a=MarinPostma

whilelist `foreign_keys` and `writable_schema` pragmas


Co-authored-by: ad hoc <[email protected]>

Author: bors[bot]

sqld/src/database/dump/exporter.rs

@@ -6,7 +6,7 @@ use anyhow::anyhow;
 use sqld_libsql_bindings::wal_hook::WalHook;
 use tokio::sync::{mpsc, oneshot};
 
-use super::libsql::open_db;
+use super::super::libsql::open_db;
 
 type OpMsg = Box<dyn FnOnce(&rusqlite::Connection) + 'static + Send + Sync>;
 

sqld/src/database/dump_loader.rs sqld/src/database/dump/loader.rssqld/src/database/dump_loader.rs renamed to sqld/src/database/dump/loader.rs

@@ -0,0 +1,2 @@
+pub mod exporter;
+pub mod loader;

sqld/src/database/dump/mod.rs

@@ -5,7 +5,7 @@ use crate::query::{Params, Query, QueryResult};
 use crate::query_analysis::{State, Statement};
 use crate::Result;
 
-pub mod dump_loader;
+pub mod dump;
 pub mod factory;
 pub mod libsql;
 pub mod write_proxy;

sqld/src/database/mod.rs

@@ -110,7 +110,7 @@ async fn handle_msg(conn: &mut Conn, client_msg: tungstenite::Message) -> Result
             let client_msg: proto::ClientMsg = match serde_json::from_str(&client_msg) {
                 Ok(client_msg) => client_msg,
                 Err(err) => {
-                    let error_msg = format!("Invalid format of client message: {}", err);
+                    let error_msg = format!("Invalid format of client message: {err}");
                     close(conn, CloseCode::Invalid, &error_msg).await;
                     tracing::warn!("{}", error_msg);
                     return Ok(false);

sqld/src/hrana/conn.rs

@@ -6,7 +6,7 @@ use std::sync::Mutex;
 use std::time::Duration;
 
 use anyhow::Context as AnyhowContext;
-use database::dump_loader::DumpLoader;
+use database::dump::loader::DumpLoader;
 use database::factory::DbFactory;
 use database::libsql::LibSqlDb;
 use database::write_proxy::WriteProxyDbFactory;
@@ -30,7 +30,7 @@ use sha256::try_digest;
 pub use sqld_libsql_bindings as libsql;
 
 mod auth;
-mod database;
+pub mod database;
 mod error;
 mod hrana;
 mod http;

sqld/src/lib.rs

@@ -1,9 +1,16 @@
-use std::{env, fs, net::SocketAddr, path::PathBuf, time::Duration};
+use std::{
+    env,
+    fs::{self, OpenOptions},
+    io::{stdout, Write},
+    net::SocketAddr,
+    path::{Path, PathBuf},
+    time::Duration,
+};
 
 use anyhow::{bail, Context as _, Result};
 use clap::Parser;
 use mimalloc::MiMalloc;
-use sqld::Config;
+use sqld::{database::dump::exporter::export_dump, Config};
 use tracing_subscriber::filter::LevelFilter;
 
 #[global_allocator]
@@ -123,6 +130,18 @@ struct Cli {
     /// defaults to 200MB.
     #[clap(long, env = "SQLD_MAX_LOG_SIZE", default_value = "200")]
     max_log_size: u64,
+
+    #[clap(subcommand)]
+    utils: Option<UtilsSubcommands>,
+}
+
+#[derive(clap::Subcommand, Debug)]
+enum UtilsSubcommands {
+    Dump {
+        #[clap(long)]
+        /// Path at which to write the dump
+        path: Option<PathBuf>,
+    },
 }
 
 impl Cli {
@@ -162,7 +181,7 @@ impl Cli {
         };
         eprintln!("\t- database path: {}", self.db_path.display());
         let extensions_str = self.extensions_path.clone().map_or("<disabled>".to_string(), |x| x.display().to_string());
-        eprintln!("\t- extensions path: {}", extensions_str);
+        eprintln!("\t- extensions path: {extensions_str}");
         eprintln!("\t- listening for HTTP requests on: {}", self.http_listen_addr);
         if let Some(ref addr) = self.pg_listen_addr {
             eprintln!("\t- listening for PostgreSQL wire on: {addr}");
@@ -215,6 +234,25 @@ fn config_from_args(args: Cli) -> Result<Config> {
     })
 }
 
+fn perform_dump(dump_path: Option<&Path>, db_path: &Path) -> anyhow::Result<()> {
+    let out: Box<dyn Write> = match dump_path {
+        Some(path) => {
+            let f = OpenOptions::new()
+                .create_new(true)
+                .write(true)
+                .open(path)
+                .with_context(|| format!("file `{}` already exists", path.display()))?;
+            Box::new(f)
+        }
+        None => Box::new(stdout()),
+    };
+    let conn = rusqlite::Connection::open(db_path.join("data"))?;
+
+    export_dump(conn, out)?;
+
+    Ok(())
+}
+
 #[tokio::main]
 async fn main() -> Result<()> {
     tracing_subscriber::fmt()
@@ -226,24 +264,38 @@ async fn main() -> Result<()> {
         )
         .init();
     let args = Cli::parse();
-    args.print_welcome_message();
 
-    #[cfg(feature = "mwal_backend")]
-    match (&args.backend, args.mwal_addr.is_some()) {
-        (sqld::Backend::Mwal, false) => {
-            anyhow::bail!("--mwal-addr parameter must be present with mwal backend")
-        }
-        (backend, true) if backend != &sqld::Backend::Mwal => {
-            anyhow::bail!(
-                "--mwal-addr parameter conflicts with backend {:?}",
-                args.backend
-            )
+    match args.utils {
+        Some(UtilsSubcommands::Dump { path }) => {
+            if let Some(ref path) = path {
+                eprintln!(
+                    "Dumping database {} to {}",
+                    args.db_path.display(),
+                    path.display()
+                );
+            }
+            perform_dump(path.as_deref(), &args.db_path)
         }
-        _ => (),
-    }
+        None => {
+            args.print_welcome_message();
+            #[cfg(feature = "mwal_backend")]
+            match (&args.backend, args.mwal_addr.is_some()) {
+                (sqld::Backend::Mwal, false) => {
+                    anyhow::bail!("--mwal-addr parameter must be present with mwal backend")
+                }
+                (backend, true) if backend != &sqld::Backend::Mwal => {
+                    anyhow::bail!(
+                        "--mwal-addr parameter conflicts with backend {:?}",
+                        args.backend
+                    )
+                }
+                _ => (),
+            }
 
-    let config = config_from_args(args)?;
-    sqld::run_server(config).await?;
+            let config = config_from_args(args)?;
+            sqld::run_server(config).await?;
 
-    Ok(())
+            Ok(())
+        }
+    }
 }

sqld/src/main.rs

@@ -1,7 +1,7 @@
 use anyhow::Result;
 use fallible_iterator::FallibleIterator;
 use sqlite3_parser::{
-    ast::{Cmd, Stmt},
+    ast::{Cmd, PragmaBody, QualifiedName, Stmt},
     lexer::sql::{Parser, ParserError},
 };
 
@@ -56,12 +56,20 @@ impl StmtKind {
                 | Stmt::CreateView { temporary: false, .. }
             ) => Some(Self::Write),
             Cmd::Stmt(Stmt::Select { .. }) => Some(Self::Read),
-            Cmd::Stmt(Stmt::Pragma { .. }) => Some(Self::Other),
+            Cmd::Stmt(Stmt::Pragma(name, body)) if is_pragma_allowed(name, body.as_ref()) => Some(Self::Write),
             _ => None,
         }
     }
 }
 
+fn is_pragma_allowed(name: &QualifiedName, _body: Option<&PragmaBody>) -> bool {
+    matches!(name, QualifiedName {
+            db_name: None,
+            name,
+            alias: None,
+        } if name.0 == "writable_schema" || name.0 == "foreign_keys")
+}
+
 /// The state of a transaction for a series of statement
 #[derive(Debug, PartialEq, Eq, Clone, Copy)]
 pub enum State {