chore(performance): Use simd for base64 encoding/decoding

Author: JakubOnderka

Cargo.lock

@@ -64,7 +64,7 @@ stdlib = [
   "dep:aes-siv",
   "dep:base16",
   "dep:base62",
-  "dep:base64",
+  "dep:base64-simd",
   "dep:cbc",
   "dep:cfb-mode",
   "dep:chacha20poly1305",
@@ -77,7 +77,6 @@ stdlib = [
   "dep:crypto_secretbox",
   "dep:csv",
   "dep:ctr",
-  "dep:data-encoding",
   "dep:digest",
   "dep:domain",
   "dep:dns-lookup",
@@ -129,7 +128,7 @@ ansi_term = { version = "0.12", optional = true }
 arbitrary = { version = "1", optional = true, features = ["derive"] }
 base16 = { version = "0.2", optional = true }
 base62 = { version = "2.2.1", optional = true }
-base64 = { version = "0.22", optional = true }
+base64-simd = { version = "0.8", optional = true }
 bytes = { version = "1", default-features = false, optional = true }
 charset = { version = "0.1", optional = true }
 encoding_rs = { version = "0.8.35", optional = true }
@@ -142,7 +141,6 @@ clap.workspace = true
 codespan-reporting = { version = "0.12", optional = true }
 convert_case = { version = "0.7.1", optional = true }
 crc = { version = "3.2.1", optional = true }
-data-encoding = { version = "2", optional = true }
 digest = { version = "0.10", optional = true }
 dyn-clone = { version = "1", default-features = false, optional = true }
 exitcode = { version = "1", optional = true }

Cargo.toml

@@ -21,6 +21,7 @@ base16,https://github.com/thomcc/rust-base16,CC0-1.0,Thom Chiovoloni <tchiovolon
 base62,https://github.com/fbernier/base62,MIT,"François Bernier <[email protected]>, Chai T. Rex <[email protected]>"
 base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,"Alice Maz <[email protected]>, Marshall Pierce <[email protected]>"
 base64,https://github.com/marshallpierce/rust-base64,MIT OR Apache-2.0,Marshall Pierce <[email protected]>
+base64-simd,https://github.com/Nugine/simd,MIT,The base64-simd Authors
 bit-set,https://github.com/contain-rs/bit-set,Apache-2.0 OR MIT,Alexis Beingessner <[email protected]>
 bit-vec,https://github.com/contain-rs/bit-vec,Apache-2.0 OR MIT,Alexis Beingessner <[email protected]>
 bitflags,https://github.com/bitflags/bitflags,MIT OR Apache-2.0,The Rust Project Developers
@@ -65,7 +66,6 @@ crypto-common,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto
 crypto_secretbox,https://github.com/RustCrypto/nacl-compat/tree/master/crypto_secretbox,Apache-2.0 OR MIT,RustCrypto Developers
 csv,https://github.com/BurntSushi/rust-csv,Unlicense OR MIT,Andrew Gallant <[email protected]>
 ctr,https://github.com/RustCrypto/block-modes,MIT OR Apache-2.0,RustCrypto Developers
-data-encoding,https://github.com/ia0/data-encoding,MIT,Julien Cretin <[email protected]>
 dbl,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers
 deranged,https://github.com/jhpratt/deranged,MIT OR Apache-2.0,Jacob Pratt <[email protected]>
 digest,https://github.com/RustCrypto/traits,MIT OR Apache-2.0,RustCrypto Developers
@@ -162,6 +162,7 @@ once_cell,https://github.com/matklad/once_cell,MIT OR Apache-2.0,Aleksey Kladov
 onig,http://github.com/iwillspeak/rust-onig,MIT,"Will Speak <[email protected]>, Ivan Ivashchenko <[email protected]>"
 opaque-debug,https://github.com/RustCrypto/utils,MIT OR Apache-2.0,RustCrypto Developers
 ordered-float,https://github.com/reem/rust-ordered-float,MIT,"Jonathan Reem <[email protected]>, Matt Brubeck <[email protected]>"
+outref,https://github.com/Nugine/outref,MIT,The outref Authors
 overload,https://github.com/danaugrs/overload,MIT,Daniel Salvadori <[email protected]>
 owo-colors,https://github.com/owo-colors/owo-colors,MIT,jam1garner <[email protected]>
 pad,https://github.com/ogham/rust-pad,MIT,Ben S <[email protected]>
@@ -259,6 +260,7 @@ utf8-width,https://github.com/magiclen/utf8-width,MIT,Magic Len <[email protected]
 utf8_iter,https://github.com/hsivonen/utf8_iter,Apache-2.0 OR MIT,Henri Sivonen <[email protected]>
 uuid,https://github.com/uuid-rs/uuid,Apache-2.0 OR MIT,"Ashley Mannix<[email protected]>, Dylan DPC<[email protected]>, Hunar Roop Kahlon<[email protected]>"
 valuable,https://github.com/tokio-rs/valuable,MIT,The valuable Authors
+vsimd,https://github.com/Nugine/simd,MIT,The vsimd Authors
 vte,https://github.com/alacritty/vte,Apache-2.0 OR MIT,"Joe Wilm <[email protected]>, Christian Duerr <[email protected]>"
 wasi,https://github.com/bytecodealliance/wasi,Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT,The Cranelift Project Developers
 wasm-bindgen,https://github.com/rustwasm/wasm-bindgen,MIT OR Apache-2.0,The wasm-bindgen Developers

LICENSE-3rdparty.csv

@@ -1,29 +1,23 @@
 use crate::compiler::prelude::*;
-use base64::Engine as _;
-use std::str::FromStr;
-
-use super::util::Base64Charset;
 
 fn decode_base64(charset: Option<Value>, value: Value) -> Resolved {
+    let value = value.try_bytes()?;
     let charset = charset
         .map(Value::try_bytes)
         .transpose()?
-        .map(|c| Base64Charset::from_str(&String::from_utf8_lossy(&c)))
-        .transpose()?
-        .unwrap_or_default();
-    let alphabet = match charset {
-        Base64Charset::Standard => base64::alphabet::STANDARD,
-        Base64Charset::UrlSafe => base64::alphabet::URL_SAFE,
+        .unwrap_or(Bytes::from_static(b"standard"));
+
+    let decoder = match charset.as_ref() {
+        b"standard" => base64_simd::STANDARD_NO_PAD,
+        b"url_safe" => base64_simd::URL_SAFE_NO_PAD,
+        _ => Err("unknown charset")?,
     };
-    let value = value.try_bytes()?;
-    let config = base64::engine::general_purpose::GeneralPurposeConfig::new()
-        .with_decode_padding_mode(base64::engine::DecodePaddingMode::Indifferent);
-    let engine = base64::engine::GeneralPurpose::new(&alphabet, config);
 
-    match engine.decode(value) {
-        Ok(s) => Ok(Value::from(Bytes::from(s))),
-        Err(_) => Err("unable to decode value to base64".into()),
-    }
+    let decoded = decoder
+        .decode_to_vec(value)
+        .map_err(|_| "unable to decode value to base64")?;
+
+    Ok(Value::Bytes(Bytes::from(decoded)))
 }
 
 #[derive(Clone, Copy, Debug)]
@@ -117,6 +111,12 @@ mod test {
             tdef: TypeDef::bytes().fallible(),
         }
 
+        with_invalid_charset {
+            args: func_args![value: value!("c29tZSs9c3RyaW5nL3ZhbHVl"), charset: value!("invalid")],
+            want: Err("unknown charset"),
+            tdef: TypeDef::bytes().fallible(),
+        }
+
         empty_string_standard_charset {
             args: func_args![value: value!(""), charset: value!("standard")],
             want: Ok(value!("")),

src/stdlib/decode_base64.rs

@@ -71,16 +71,12 @@ mod tests {
     use super::*;
     use crate::value;
 
-    use base64::Engine;
     use nom::AsBytes;
 
     fn decode_base64(text: &str) -> Vec<u8> {
-        let engine = base64::engine::GeneralPurpose::new(
-            &base64::alphabet::STANDARD,
-            base64::engine::general_purpose::GeneralPurposeConfig::new(),
-        );
-
-        engine.decode(text).expect("Cannot decode from Base64")
+        base64_simd::STANDARD
+            .decode_to_vec(text)
+            .expect("Cannot decode from Base64")
     }
 
     test_function![

src/stdlib/decode_lz4.rs

@@ -1,5 +1,4 @@
 use charset::Charset;
-use data_encoding::BASE64_MIME;
 use nom::{
     branch::alt,
     bytes::complete::{tag, take_until, take_until1},
@@ -160,8 +159,8 @@ impl EncodedWord<'_> {
 
         // Decode
         let decoded = match self.encoding {
-            "B" | "b" => BASE64_MIME
-                .decode(self.input.as_bytes())
+            "B" | "b" => base64_simd::STANDARD_NO_PAD
+                .decode_to_vec(self.input.as_bytes())
                 .map_err(|_| "Unable to decode base64 value")?,
             "Q" | "q" => {
                 // The quoted_printable module does a trim_end on the input, so if

src/stdlib/decode_mime_q.rs

@@ -70,16 +70,12 @@ impl FunctionExpression for DecodeSnappyFn {
 mod tests {
     use super::*;
     use crate::value;
-    use base64::Engine;
     use nom::AsBytes;
 
     fn decode_base64(text: &str) -> Vec<u8> {
-        let engine = base64::engine::GeneralPurpose::new(
-            &base64::alphabet::STANDARD,
-            base64::engine::general_purpose::GeneralPurposeConfig::new(),
-        );
-
-        engine.decode(text).expect("Cannot decode from Base64")
+        base64_simd::STANDARD
+            .decode_to_vec(text)
+            .expect("Cannot decode from Base64")
     }
 
     test_function![

src/stdlib/decode_snappy.rs

@@ -1,29 +1,26 @@
 use crate::compiler::prelude::*;
-use base64::Engine as _;
-use std::str::FromStr;
-
-use super::util::Base64Charset;
 
 fn encode_base64(value: Value, padding: Option<Value>, charset: Option<Value>) -> Resolved {
     let value = value.try_bytes()?;
     let padding = padding
         .map(VrlValueConvert::try_boolean)
         .transpose()?
         .unwrap_or(true);
+
     let charset = charset
         .map(VrlValueConvert::try_bytes)
         .transpose()?
-        .map(|c| Base64Charset::from_str(&String::from_utf8_lossy(&c)))
-        .transpose()?
-        .unwrap_or_default();
+        .unwrap_or(Bytes::from_static(b"standard"));
 
-    let engine = base64::engine::GeneralPurpose::new(
-        &base64::alphabet::Alphabet::from(charset),
-        base64::engine::general_purpose::GeneralPurposeConfig::default()
-            .with_encode_padding(padding),
-    );
+    let encoder = match (padding, charset.as_ref()) {
+        (true, b"standard") => base64_simd::STANDARD,
+        (false, b"standard") => base64_simd::STANDARD_NO_PAD,
+        (true, b"url_safe") => base64_simd::URL_SAFE,
+        (false, b"url_safe") => base64_simd::URL_SAFE_NO_PAD,
+        _ => Err("unknown charset")?,
+    };
 
-    Ok(engine.encode(value).into())
+    Ok(encoder.encode_to_string(value).into())
 }
 
 #[derive(Clone, Copy, Debug)]

src/stdlib/encode_base64.rs

@@ -66,16 +66,12 @@ impl FunctionExpression for EncodeLz4Fn {
 mod test {
     use super::*;
     use crate::value;
-    use base64::Engine;
     use nom::AsBytes;
 
     fn decode_base64(text: &str) -> Vec<u8> {
-        let engine = base64::engine::GeneralPurpose::new(
-            &base64::alphabet::STANDARD,
-            base64::engine::general_purpose::GeneralPurposeConfig::new(),
-        );
-
-        engine.decode(text).expect("Cannot decode from Base64")
+        base64_simd::STANDARD
+            .decode_to_vec(text)
+            .expect("Cannot decode from Base64")
     }
 
     test_function![

src/stdlib/encode_lz4.rs

@@ -70,16 +70,12 @@ impl FunctionExpression for EncodeSnappyFn {
 mod test {
     use super::*;
     use crate::value;
-    use base64::Engine;
     use nom::AsBytes;
 
     fn decode_base64(text: &str) -> Vec<u8> {
-        let engine = base64::engine::GeneralPurpose::new(
-            &base64::alphabet::STANDARD,
-            base64::engine::general_purpose::GeneralPurposeConfig::new(),
-        );
-
-        engine.decode(text).expect("Cannot decode from Base64")
+        base64_simd::STANDARD
+            .decode_to_vec(text)
+            .expect("Cannot decode from Base64")
     }
 
     test_function![

src/stdlib/encode_snappy.rs

@@ -1,5 +1,4 @@
 use crate::compiler::prelude::*;
-use base64::engine::Engine;
 use std::{
     borrow::Cow,
     convert::{TryFrom, TryInto},
@@ -460,7 +459,7 @@ impl Encoder {
     fn encode(self, data: &[u8]) -> String {
         use Encoder::{Base16, Base64};
         match self {
-            Base64 => base64::engine::general_purpose::STANDARD.encode(data),
+            Base64 => base64_simd::STANDARD.encode_to_string(data),
             Base16 => base16::encode_lower(data),
         }
     }

src/stdlib/redact.rs

@@ -81,43 +81,6 @@ pub(crate) fn is_nullish(value: &Value) -> bool {
     }
 }
 
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub enum Base64Charset {
-    Standard,
-    UrlSafe,
-}
-
-impl Default for Base64Charset {
-    fn default() -> Self {
-        Self::Standard
-    }
-}
-
-impl From<Base64Charset> for base64::alphabet::Alphabet {
-    fn from(charset: Base64Charset) -> base64::alphabet::Alphabet {
-        use Base64Charset::{Standard, UrlSafe};
-
-        match charset {
-            Standard => base64::alphabet::STANDARD,
-            UrlSafe => base64::alphabet::URL_SAFE,
-        }
-    }
-}
-
-impl std::str::FromStr for Base64Charset {
-    type Err = &'static str;
-
-    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
-        use Base64Charset::{Standard, UrlSafe};
-
-        match s {
-            "standard" => Ok(Standard),
-            "url_safe" => Ok(UrlSafe),
-            _ => Err("unknown charset"),
-        }
-    }
-}
-
 #[derive(Clone, Debug)]
 pub(super) enum ConstOrExpr {
     Const(Value),