re: New higher-performance rnote file format, and atomic file saving

Cargo.toml

@@ -28,7 +28,9 @@ base64 = "0.22.1"
 cairo-rs = { version = "0.21.1", features = ["v1_18", "png", "svg", "pdf"] }
 chrono = "0.4.41"
 clap = { version = "4.5", features = ["derive"] }
+crc32fast = { version = "1.5" }
 dialoguer = "0.12.0"
+deranged = { version = "0.5", features = ["serde"] }
 flate2 = "1.1"
 fs_extra = "1.3"
 futures = "0.3.31"
@@ -86,13 +88,15 @@ slotmap = { version = "1.0", features = ["serde"] }
 smol = "2.0"
 svg = "0.18.0"
 thiserror = "2.0.12"
+thread_local = "1.1"
 tracing = "0.1.41"
 tracing-subscriber = { version = "0.3.19", features = ["env-filter"] }
 unicode-segmentation = "1.12"
 url = "2.5"
 usvg = "0.45.1"
 winresource = "0.1.20"
 xmlwriter = "0.1.0"
+zstd = "0.13"
 
 # Enabling feature > v20_9 causes linker errors on mingw
 poppler-rs = { version = "0.25.0", features = ["v20_9"] }

crates/rnote-cli/src/cli.rs

@@ -1,5 +1,5 @@
 // Imports
-use crate::{export, import, test, thumbnail};
+use crate::{export, import, set_compression, test, thumbnail};
 use anyhow::Context;
 use clap::Parser;
 use rnote_compose::SplitOrder;
@@ -79,6 +79,16 @@ pub(crate) enum Command {
         /// Output path of the thumbnail
         output: PathBuf,
     },
+    /// Modify the compression method or level of one or more `.rnote` files.{n}
+    /// Please note that the compression method (+ level) will be reset to defaults on the next save within the Rnote application.{n}
+    SetCompression {
+        #[arg(short, visible_short_alias = 'i', long, num_args = 1..)]
+        rnote_files: Vec<PathBuf>,
+        #[arg(long, visible_alias = "cm", value_parser = ["zstd", "Zstd", "none", "None"], default_value = "zstd")]
+        compression_method: String,
+        #[arg(long, visible_alias = "cl")]
+        compression_level: Option<i32>,
+    },
 }
 
 #[derive(clap::ValueEnum, Debug, Clone, Copy, Default)]
@@ -260,8 +270,19 @@ pub(crate) async fn run() -> anyhow::Result<()> {
             println!("Thumbnail...");
             thumbnail::run_thumbnail(rnote_file, size, output).await?;
         }
+        Command::SetCompression {
+            rnote_files,
+            compression_method,
+            compression_level,
+        } => {
+            set_compression::run_set_compression(
+                rnote_files,
+                compression_method,
+                compression_level,
+            )
+            .await?;
+        }
     }
-
     Ok(())
 }
 

crates/rnote-cli/src/main.rs

@@ -6,6 +6,7 @@
 pub(crate) mod cli;
 pub(crate) mod export;
 pub(crate) mod import;
+pub(crate) mod set_compression;
 pub(crate) mod test;
 pub(crate) mod thumbnail;
 pub(crate) mod validators;

crates/rnote-cli/src/meson.build

@@ -4,6 +4,7 @@ rnote_cli_sources = files(
     'export.rs',
     'import.rs',
     'main.rs',
+    'set_compression.rs',
     'test.rs',
     'validators.rs',
 )

crates/rnote-cli/src/set_compression.rs

@@ -0,0 +1,60 @@
+use indicatif::ProgressIterator;
+use rnote_engine::fileformats::rnoteformat::{
+    self, CompressionMethod, DEFAULT_ZSTD_COMPRESSION_INTEGER,
+};
+use smol::{fs::OpenOptions, io::AsyncReadExt};
+use std::path::PathBuf;
+
+pub(crate) async fn run_set_compression(
+    rnote_files: Vec<PathBuf>,
+    compression_method: String,
+    compression_level: Option<i32>,
+) -> anyhow::Result<()> {
+    let mut compression_method = match compression_method.as_str() {
+        "zstd" | "Zstd" => CompressionMethod::Zstd(DEFAULT_ZSTD_COMPRESSION_INTEGER),
+        "none" | "None" => CompressionMethod::None,
+        _ => unreachable!(),
+    };
+
+    if let Some(compression_level) = compression_level
+        && !matches!(compression_method, CompressionMethod::None)
+    {
+        compression_method.update_compression_integer(compression_level)?;
+    }
+
+    let spinner = indicatif::ProgressBar::new_spinner().with_style(
+        indicatif::ProgressStyle::default_spinner()
+            .tick_chars("⊶⊷✔")
+            .template("{spinner:.green} [{elapsed_precise}] ({pos}/{len}) Mutating '{msg}'")
+            .unwrap(),
+    );
+    spinner.set_length(rnote_files.len() as u64);
+    spinner.enable_steady_tick(std::time::Duration::from_millis(250));
+
+    for filepath in rnote_files.iter().progress_with(spinner.clone()) {
+        spinner.set_message(format!("{}", filepath.display()));
+        let file_read_operation = async {
+            let mut read_file = OpenOptions::new().read(true).open(filepath).await?;
+            let mut bytes: Vec<u8> = {
+                match read_file.metadata().await {
+                    Ok(metadata) => {
+                        Vec::with_capacity(usize::try_from(metadata.len()).unwrap_or(usize::MAX))
+                    }
+                    Err(err) => {
+                        eprintln!("Failed to read file metadata, '{err}'");
+                        Vec::new()
+                    }
+                }
+            };
+            read_file.read_to_end(&mut bytes).await?;
+            Ok::<Vec<u8>, anyhow::Error>(bytes)
+        };
+
+        let mut bytes = file_read_operation.await?;
+        let engine_snapshot = rnoteformat::load_engine_snapshot_from_bytes(&bytes)?;
+        bytes = rnoteformat::save_engine_snapshot_to_bytes(engine_snapshot, compression_method)?;
+        rnote_engine::utils::atomic_save_to_file(filepath, &bytes).await?
+    }
+
+    Ok(())
+}

crates/rnote-engine/Cargo.toml

@@ -13,10 +13,13 @@ rnote-compose = { workspace = true }
 
 anyhow = { workspace = true }
 approx = { workspace = true }
+async-fs = { workspace = true }
 base64 = { workspace = true }
 cairo-rs = { workspace = true }
 chrono = { workspace = true }
 clap = { workspace = true, optional = true }
+crc32fast = { workspace = true }
+deranged = { workspace = true, features = ["serde"] }
 flate2 = { workspace = true }
 futures = { workspace = true }
 geo = { workspace = true }
@@ -51,10 +54,13 @@ serde_json = { workspace = true }
 slotmap = { workspace = true }
 svg = { workspace = true }
 thiserror = { workspace = true }
+thread_local = { workspace = true }
 tracing = { workspace = true }
 unicode-segmentation = { workspace = true }
 usvg = { workspace = true }
 xmlwriter = { workspace = true }
+zstd = { workspace = true }
+
 # the long-term plan is to remove the gtk4 dependency entirely after switching to another renderer.
 gtk4 = { workspace = true, optional = true }
 

crates/rnote-engine/src/engine/export.rs

@@ -1,6 +1,6 @@
 // Imports
 use super::{Engine, StrokeContent};
-use crate::fileformats::rnoteformat::RnoteFile;
+use crate::fileformats::rnoteformat;
 use crate::fileformats::{FileFormatSaver, xoppformat};
 use anyhow::Context;
 use futures::channel::oneshot;
@@ -9,6 +9,7 @@ use rnote_compose::SplitOrder;
 use rnote_compose::transform::Transformable;
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
+use std::time::Instant;
 use tracing::error;
 
 /// Document export format.
@@ -315,19 +316,20 @@ impl Engine {
     /// The used image scale-factor for any strokes that are converted to bitmap images on export.
     pub const STROKE_EXPORT_IMAGE_SCALE: f64 = 1.8;
 
-    /// Save the current document as a .rnote file.
+    /// Save the current document as a `.rnote` file.
+    #[allow(unused_variables)]
     pub fn save_as_rnote_bytes(
         &self,
         file_name: String,
     ) -> oneshot::Receiver<anyhow::Result<Vec<u8>>> {
         let (oneshot_sender, oneshot_receiver) = oneshot::channel::<anyhow::Result<Vec<u8>>>();
         let engine_snapshot = self.take_snapshot();
         rayon::spawn(move || {
+            #[rustfmt::skip]
             let result = || -> anyhow::Result<Vec<u8>> {
-                let rnote_file = RnoteFile {
-                    engine_snapshot: ijson::to_value(&engine_snapshot)?,
-                };
-                rnote_file.save_as_bytes(&file_name)
+                let start = Instant::now();
+                rnoteformat::save_engine_snapshot_to_bytes(engine_snapshot, rnoteformat::CompressionMethod::default())
+                    .inspect(|_| {tracing::debug!("Going from `EngineSnapshot` to bytes took {} ms", Instant::now().duration_since(start).as_millis())})
             };
             if oneshot_sender.send(result()).is_err() {
                 error!(

crates/rnote-engine/src/engine/snapshot.rs

@@ -5,11 +5,11 @@ use crate::fileformats::{FileFormatLoader, rnoteformat, xoppformat};
 use crate::store::{ChronoComponent, StrokeKey};
 use crate::strokes::Stroke;
 use crate::{Camera, Document, Engine};
-use anyhow::Context;
 use futures::channel::oneshot;
 use serde::{Deserialize, Serialize};
 use slotmap::{HopSlotMap, SecondaryMap};
 use std::sync::Arc;
+use std::time::Instant;
 use tracing::error;
 
 /// Trait for types which hold configuration needed for engine snapshots
@@ -53,10 +53,11 @@ impl EngineSnapshot {
         let (snapshot_sender, snapshot_receiver) = oneshot::channel::<anyhow::Result<Self>>();
 
         rayon::spawn(move || {
+            #[rustfmt::skip]
             let result = || -> anyhow::Result<Self> {
-                let rnote_file = rnoteformat::RnoteFile::load_from_bytes(&bytes)
-                    .context("loading RnoteFile from bytes failed.")?;
-                Ok(ijson::from_value(&rnote_file.engine_snapshot)?)
+                let start = Instant::now();
+                rnoteformat::load_engine_snapshot_from_bytes(&bytes)
+                    .inspect(|_| {tracing::debug!("Going from bytes to `EngineSnapshot` took {} ms", Instant::now().duration_since(start).as_millis())})
             };
 
             if let Err(_data) = snapshot_sender.send(result()) {

crates/rnote-engine/src/fileformats/rnoteformat/bcursor.rs

@@ -0,0 +1,40 @@
+/// Simple cursor struct for bytes, somewhat similar to `std::io::Cursor`
+/// but with nicer methods for our use case (no read/write abstraction pain)
+#[derive(Debug)]
+pub struct BCursor<'a> {
+    inner: &'a [u8],
+    pos: usize,
+}
+
+impl<'a> BCursor<'a> {
+    /// Creates a new byte-cursor by wrapping borrowed bytes and setting the cursor position to zero.
+    pub fn new(bytes: &'a [u8]) -> Self {
+        Self {
+            inner: bytes,
+            pos: 0,
+        }
+    }
+
+    /// Attempts to advance the position of the cursor by a specified amount, capturing the bytes in between.
+    /// Returns an error if and only if we try to cross out of bounds.
+    pub fn try_capture(&mut self, by: usize) -> anyhow::Result<&'a [u8]> {
+        self.inner
+            .get(self.pos..self.pos + by)
+            .inspect(|_| self.pos += by)
+            .ok_or_else(|| anyhow::anyhow!("Failed to capture {by} bytes, out of bounds"))
+    }
+
+    /// Similar to [Self::try_capture], except the position of the cursor isn't updated.
+    pub fn try_seek(&mut self, by: usize) -> anyhow::Result<&'a [u8]> {
+        self.inner
+            .get(self.pos..self.pos + by)
+            .ok_or_else(|| anyhow::anyhow!("Failed to seek {by} bytes, out of bounds"))
+    }
+
+    /// Similar to [Self::try_capture], except we specify the amount to try advancing the cursor with using a compile-time constant generic, in order to get a known-size array in return.
+    pub fn try_capture_exact<const BY: usize>(&mut self) -> anyhow::Result<[u8; BY]> {
+        let mut bytes_exact: [u8; BY] = [0; BY];
+        bytes_exact.copy_from_slice(self.try_capture(BY)?);
+        Ok(bytes_exact)
+    }
+}