feat: implement database vacuuming.

This change attempts to resolve
#2927 by introducing the ability
to VACUUM the database by:

 - giving atuin-server-database the `vacuum` method. This simply calls
   sqlx::query("VACUUM"). I also added a test, as we're using sqlx &
   sqlite this test can be executed in memory. Sadly the same cannot
   happen for postgres, but the test is kind of trivial.

 - exposing a vacuum endpoint on the server. This handler calls vacuum.
   This can be useful for users who wish to manually vacuum a db, but
   this is not exposed via the CLI, instead...

 - adding `auto_vacuum_threshold` option to the server configuration. If
   `auto_vacuum_threshold` is > 0, then the server will increment the
   `delete_count` atomic counter, and when this counter reached the
   threshold it will call vacuum and reset the counter.

The default for `auto_vacuum_threshold` is `0` meaning "never vacuum",
but users can set this to a value which will come down to their
preference, guided by:

 - How long they anticipate their server running for. For example
   setting it to a high-ish number like 1000 would probably be appropriate
   for a server running on a machine which has long uptimes. For self hosting
   for a single user, this might be a little long though, keeping in
   mind that if the server goes down (or the mahcine goes down) the counter will
   reset to 0.

 - For users who _really care_ about disk space and don't really care
   about latency on deletes, setting to `1` will will vacuum on every
   delete.

I considered adding some kind of "recommended" number but I don't think
there is a good number to unilaterally recommend for this.

Author: keithamus

crates/atuin-server-database/src/lib.rs

@@ -142,6 +142,8 @@ pub trait Database: Sized + Clone + Send + Sync + 'static {
 
     async fn oldest_history(&self, user: &User) -> DbResult<History>;
 
+    async fn vacuum(&self) -> DbResult<()>;
+
     #[instrument(skip_all)]
     async fn calendar(
         &self,

crates/atuin-server-postgres/src/lib.rs

@@ -672,6 +672,16 @@ impl Database for Postgres {
 
         Ok(status)
     }
+
+    #[instrument(skip_all)]
+    async fn vacuum(&self) -> DbResult<()> {
+        sqlx::query("VACUUM")
+            .execute(&self.pool)
+            .await
+            .map_err(fix_error)?;
+
+        Ok(())
+    }
 }
 
 fn into_utc(x: OffsetDateTime) -> PrimitiveDateTime {

crates/atuin-server-sqlite/src/lib.rs

@@ -544,9 +544,38 @@ impl Database for Sqlite {
         .map_err(fix_error)
         .map(|DbHistory(h)| h)
     }
+
+    #[instrument(skip_all)]
+    async fn vacuum(&self) -> DbResult<()> {
+        sqlx::query("VACUUM")
+            .execute(&self.pool)
+            .await
+            .map_err(fix_error)?;
+
+        Ok(())
+    }
 }
 
 fn into_utc(x: OffsetDateTime) -> PrimitiveDateTime {
     let x = x.to_offset(UtcOffset::UTC);
     PrimitiveDateTime::new(x.date(), x.time())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[sqlx::test]
+    async fn test_vacuum() {
+        let db_uri = "sqlite::memory:";
+        let settings = DbSettings {
+            db_uri: db_uri.to_string(),
+        };
+
+        let db = Sqlite::new(&settings).await.expect("failed to create database");
+
+        // VACUUM should succeed without errors
+        let result = db.vacuum().await;
+        assert!(result.is_ok(), "VACUUM should succeed");
+    }
+}

crates/atuin-server/server.toml

@@ -33,3 +33,8 @@
 # enable = false
 # cert_path = ""
 # pkey_path = ""
+
+# [vacuum]
+# Automatically vacuum the database after this many delete operations
+# Set to 0 to disable automatic vacuuming
+# auto_vacuum_threshold = 0

crates/atuin-server/src/handlers/history.rs

@@ -106,17 +106,38 @@ pub async fn delete<DB: Database>(
     state: State<AppState<DB>>,
     Json(req): Json<DeleteHistoryRequest>,
 ) -> Result<Json<MessageResponse>, ErrorResponseStatus<'static>> {
-    let db = &state.0.database;
+    let State(AppState {
+        database,
+        settings,
+        delete_count,
+    }) = state;
 
     // user_id is the ID of the history, as set by the user (the server has its own ID)
-    let deleted = db.delete_history(&user, req.client_id).await;
+    let deleted = database.delete_history(&user, req.client_id).await;
 
     if let Err(e) = deleted {
         error!("failed to delete history: {}", e);
         return Err(ErrorResponse::reply("failed to delete history")
             .with_status(StatusCode::INTERNAL_SERVER_ERROR));
     }
 
+    // Trigger vacuum if threshold is met
+    if settings.vacuum.auto_vacuum_threshold > 0 {
+        use std::sync::atomic::Ordering;
+        let count = delete_count.fetch_add(1, Ordering::Relaxed) + 1;
+
+        if count >= settings.vacuum.auto_vacuum_threshold {
+            debug!("triggering automatic vacuum after {} deletes", count);
+            tokio::spawn(async move {
+                if let Err(e) = database.vacuum().await {
+                    error!("failed to vacuum database: {}", e);
+                } else {
+                    delete_count.store(0, Ordering::Relaxed);
+                }
+            });
+        }
+    }
+
     Ok(Json(MessageResponse {
         message: String::from("deleted OK"),
     }))
@@ -128,7 +149,9 @@ pub async fn add<DB: Database>(
     state: State<AppState<DB>>,
     Json(req): Json<Vec<AddHistoryRequest>>,
 ) -> Result<(), ErrorResponseStatus<'static>> {
-    let State(AppState { database, settings }) = state;
+    let State(AppState {
+        database, settings, ..
+    }) = state;
 
     debug!("request to add {} history items", req.len());
     counter!("atuin_history_uploaded", req.len() as u64);
@@ -235,3 +258,21 @@ pub async fn calendar<DB: Database>(
 
     Ok(Json(focus))
 }
+
+#[instrument(skip_all, fields(user.id = user.id))]
+pub async fn vacuum<DB: Database>(
+    UserAuth(user): UserAuth,
+    state: State<AppState<DB>>,
+) -> Result<Json<MessageResponse>, ErrorResponseStatus<'static>> {
+    let db = &state.0.database;
+
+    if let Err(e) = db.vacuum().await {
+        error!("failed to vacuum database: {}", e);
+        return Err(ErrorResponse::reply("failed to vacuum database")
+            .with_status(StatusCode::INTERNAL_SERVER_ERROR));
+    }
+
+    Ok(Json(MessageResponse {
+        message: String::from("vacuum completed successfully"),
+    }))
+}

crates/atuin-server/src/handlers/v0/record.rs

@@ -17,7 +17,9 @@ pub async fn post<DB: Database>(
     state: State<AppState<DB>>,
     Json(records): Json<Vec<Record<EncryptedData>>>,
 ) -> Result<(), ErrorResponseStatus<'static>> {
-    let State(AppState { database, settings }) = state;
+    let State(AppState {
+        database, settings, ..
+    }) = state;
 
     tracing::debug!(
         count = records.len(),
@@ -58,6 +60,7 @@ pub async fn index<DB: Database>(
     let State(AppState {
         database,
         settings: _,
+        ..
     }) = state;
 
     let record_index = match database.status(&user).await {
@@ -92,6 +95,7 @@ pub async fn next<DB: Database>(
     let State(AppState {
         database,
         settings: _,
+        ..
     }) = state;
     let params = params.0;
 

crates/atuin-server/src/handlers/v0/store.rs

@@ -21,6 +21,7 @@ pub async fn delete<DB: Database>(
     let State(AppState {
         database,
         settings: _,
+        ..
     }) = state;
 
     if let Err(e) = database.delete_store(&user).await {

crates/atuin-server/src/router.rs

@@ -1,3 +1,5 @@
+use std::sync::{Arc, atomic::AtomicU64};
+
 use async_trait::async_trait;
 use atuin_common::api::{ATUIN_CARGO_VERSION, ATUIN_HEADER_VERSION, ErrorResponse};
 use axum::{
@@ -106,6 +108,7 @@ async fn semver(request: Request, next: Next) -> Response {
 pub struct AppState<DB: Database> {
     pub database: DB,
     pub settings: Settings,
+    pub delete_count: Arc<AtomicU64>,
 }
 
 pub fn router<DB: Database>(database: DB, settings: Settings) -> Router {
@@ -118,6 +121,7 @@ pub fn router<DB: Database>(database: DB, settings: Settings) -> Router {
         .route("/sync/status", get(handlers::status::status))
         .route("/history", post(handlers::history::add))
         .route("/history", delete(handlers::history::delete))
+        .route("/history/vacuum", post(handlers::history::vacuum))
         .route("/user/:username", get(handlers::user::get))
         .route("/account", delete(handlers::user::delete))
         .route("/account/password", patch(handlers::user::change_password))
@@ -144,7 +148,11 @@ pub fn router<DB: Database>(database: DB, settings: Settings) -> Router {
         Router::new().nest(path, routes)
     }
     .fallback(teapot)
-    .with_state(AppState { database, settings })
+    .with_state(AppState {
+        database,
+        settings,
+        delete_count: Arc::new(AtomicU64::new(0)),
+    })
     .layer(
         ServiceBuilder::new()
             .layer(axum::middleware::from_fn(clacks_overhead))

crates/atuin-server/src/settings.rs

@@ -53,6 +53,22 @@ impl Default for Metrics {
     }
 }
 
+#[derive(Clone, Debug, Deserialize, Serialize)]
+pub struct Vacuum {
+    /// Automatically vacuum after delete operations when the number of deleted
+    /// rows exceeds this threshold. Set to 0 to disable automatic vacuum on delete.
+    /// The manual vacuum endpoint is always available regardless of this setting.
+    pub auto_vacuum_threshold: u64,
+}
+
+impl Default for Vacuum {
+    fn default() -> Self {
+        Self {
+            auto_vacuum_threshold: 0,
+        }
+    }
+}
+
 #[derive(Clone, Debug, Deserialize, Serialize)]
 pub struct Settings {
     pub host: String,
@@ -67,6 +83,7 @@ pub struct Settings {
     pub metrics: Metrics,
     pub tls: Tls,
     pub mail: Mail,
+    pub vacuum: Vacuum,
 
     /// Advertise a version that is not what we are _actually_ running
     /// Many clients compare their version with api.atuin.sh, and if they differ, notify the user
@@ -109,6 +126,7 @@ impl Settings {
             .set_default("tls.enable", false)?
             .set_default("tls.cert_path", "")?
             .set_default("tls.pkey_path", "")?
+            .set_default("vacuum.auto_vacuum_threshold", 0)?
             .add_source(
                 Environment::with_prefix("atuin")
                     .prefix_separator("_")