parallelise walker

Author: noahbald

Cargo.lock

@@ -30,4 +30,4 @@ config = { version = "0.15", features = ["json", "json5"] }
 log = { workspace = true }
 serde = { workspace = true }
 serde_json = { workspace = true }
-walkdir = "2.5.0"
+ignore = "0.4"

crates/oxvg/Cargo.toml

@@ -1,17 +1,18 @@
 use std::{
+    cell::RefCell,
     ffi::OsStr,
     io::{IsTerminal, Read, Write},
     path::PathBuf,
     str::FromStr,
 };
 
 use anyhow::anyhow;
+use ignore::{WalkBuilder, WalkState};
 use oxvg_ast::{
     implementations::markup5ever::{Element5Ever, Node5Ever},
     visitor::Info,
 };
 use oxvg_optimiser::Jobs;
-use walkdir::WalkDir;
 
 use crate::{args::RunCommand, config::Config};
 
@@ -29,8 +30,15 @@ pub struct Optimise {
     /// If no config is specified the current config will be printed instead.
     #[clap(long, short, num_args(0..=1))]
     pub config: Option<Vec<PathBuf>>,
+    /// If the path is a directory, whether to walk through and optimise it's subdirectories
     #[clap(long, short, default_value = "false")]
     pub recursive: bool,
+    /// Search through hidden files and directories
+    #[clap(long, short = '.', default_value = "false")]
+    pub hidden: bool,
+    /// Sets the approximate number of threads to use. A value of 0 (default) will automatically determine the appropriate number
+    #[clap(long, short, default_value = "0")]
+    pub threads: usize,
 }
 
 impl RunCommand for Optimise {
@@ -39,9 +47,11 @@ impl RunCommand for Optimise {
         let Some(config) = config else {
             return Ok(());
         };
-        let jobs = config.optimise.unwrap_or_default();
+        if let Some(jobs) = config.optimise {
+            LOADED_JOBS.set(jobs);
+        }
 
-        self.handle_paths(&jobs)
+        self.handle_paths()
     }
 }
 
@@ -52,7 +62,7 @@ impl Optimise {
         dom.serialize_into(wr)
     }
 
-    fn handle_stdin(&self, jobs: &Jobs<Element5Ever>) -> anyhow::Result<()> {
+    fn handle_stdin(&self, jobs: Jobs<Element5Ever>) -> anyhow::Result<()> {
         use oxvg_ast::parse::Node;
 
         let mut string = String::new();
@@ -63,7 +73,7 @@ impl Optimise {
             path: None,
             multipass_count: 0,
         };
-        jobs.clone().run(&dom, &info)?;
+        jobs.run(&dom, &info)?;
 
         if let Some(output) = &self.output.as_ref().and_then(|o| {
             eprintln!("Warning: Using empty `-o,--output` with stdin will print to stdout, you can instead omit `-o,--output`.");
@@ -124,7 +134,7 @@ impl Optimise {
         }
     }
 
-    fn handle_path(&self, jobs: &Jobs<Element5Ever>, path: &PathBuf) -> anyhow::Result<()> {
+    fn handle_path(&self, path: &PathBuf) {
         let output_path = |input: &PathBuf| {
             let Some(output) = self.output.as_ref() else {
                 return Ok(None);
@@ -134,34 +144,44 @@ impl Optimise {
             };
             input.strip_prefix(path).map(|p| Some(output.join(p)))
         };
-        let mut walker = WalkDir::new(path);
-        if !self.recursive {
-            walker = walker.max_depth(1);
-        }
-        for file in walker.follow_links(true) {
-            let file = file?;
-            if !file.file_type().is_file() {
-                continue;
-            }
-            let path = file.into_path();
-            if path.extension().and_then(OsStr::to_str) != Some("svg") {
-                continue;
-            }
-            Self::handle_file(jobs, &path, output_path(&path)?.as_ref())?;
-        }
-
-        Ok(())
+        WalkBuilder::new(path)
+            .max_depth(if self.recursive { None } else { Some(1) })
+            .follow_links(true)
+            .threads(self.threads)
+            .build_parallel()
+            .run(|| {
+                Box::new(move |path| {
+                    let jobs = LOADED_JOBS.with_borrow(Clone::clone);
+                    let Ok(path) = path else {
+                        return WalkState::Continue;
+                    };
+                    if !path.file_type().is_some_and(|f| f.is_file()) {
+                        return WalkState::Continue;
+                    }
+                    let path = path.into_path();
+                    if path.extension().and_then(OsStr::to_str) != Some("svg") {
+                        return WalkState::Continue;
+                    }
+                    let Ok(output_path) = output_path(&path) else {
+                        return WalkState::Continue;
+                    };
+                    if let Err(err) = Self::handle_file(&jobs, &path, output_path.as_ref()) {
+                        eprintln!("{err}");
+                    };
+                    WalkState::Continue
+                })
+            });
     }
 
-    fn handle_paths(&self, jobs: &Jobs<Element5Ever>) -> anyhow::Result<()> {
+    fn handle_paths(&self) -> anyhow::Result<()> {
         if !std::io::stdin().is_terminal()
             && self.paths.len() <= 1
             && self
                 .paths
                 .first()
                 .is_none_or(|path| path == &PathBuf::from_str(".").unwrap())
         {
-            return self.handle_stdin(jobs);
+            return LOADED_JOBS.with(|jobs| self.handle_stdin(jobs.take()));
         }
         if self.paths.is_empty() {
             return Err(anyhow!(
@@ -170,7 +190,7 @@ impl Optimise {
         }
 
         for path in &self.paths {
-            self.handle_path(jobs, path)?;
+            self.handle_path(path);
         }
         Ok(())
     }
@@ -199,3 +219,7 @@ impl Optimise {
         }
     }
 }
+
+thread_local! {
+    static LOADED_JOBS: RefCell<Jobs<Element5Ever>> = RefCell::new(Jobs::default());
+}

crates/oxvg/src/optimise.rs

@@ -143,7 +143,6 @@ impl RemoveDeprecatedAttrs {
         group_deprecated_unsafe: Option<&phf::Set<&'static str>>,
         attributes_in_stylesheet: &AttrStylesheet,
     ) {
-        dbg!(&group_deprecated_safe, &group_deprecated_unsafe);
         if let Some(deprecated) = group_deprecated_safe {
             for deprecated in deprecated {
                 let (prefix, local_name) = match deprecated.split_once(':') {