[Project Fluent] Basic internationalization support (would close PR #2553)

crates/atuin-client/Cargo.toml

@@ -20,6 +20,7 @@ check-update = []
 
 [dependencies]
 atuin-common = { path = "../atuin-common", version = "18.5.0" }
+atuin-macro = { path = "../atuin-macro", version = "18.5.0" }
 
 log = { workspace = true }
 base64 = { workspace = true }

crates/atuin-client/src/api_client.rs

@@ -325,12 +325,7 @@ impl<'a> Client<'a> {
         start: RecordIdx,
         count: u64,
     ) -> Result<Vec<Record<EncryptedData>>> {
-        debug!(
-            "fetching record/s from host {}/{}/{}",
-            host.0.to_string(),
-            tag,
-            start
-        );
+        debug!("fetching record/s from host {}/{}/{}", host.0, tag, start);
 
         let url = format!(
             "{}/api/v0/record/next?host={}&tag={}&count={}&start={}",

crates/atuin-common/Cargo.toml

@@ -13,6 +13,8 @@ repository = { workspace = true }
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
+atuin-macro = { path = "../atuin-macro" }
+
 time = { workspace = true }
 serde = { workspace = true }
 uuid = { workspace = true }
@@ -25,8 +27,15 @@ directories = { workspace = true }
 sysinfo = "0.30.7"
 base64 = { workspace = true }
 getrandom = "0.2"
+sys-locale = "0.3.2"
 
 lazy_static = "1.4.0"
+i18n-embed = { version = "0.15.3", features = ["fluent", "fluent-system", "tr", "locale_config", "desktop-requester", "walkdir", "filesystem-assets"] }
+rust-embed = "8"
+i18n-embed-fl = "0.9.3"
+slugify = "0.1.0"
+paste = "1.0.15"
+unic-langid = "0.9.5"
 
 [dev-dependencies]
 pretty_assertions = { workspace = true }

crates/atuin-common/i18n.toml

@@ -0,0 +1,13 @@
+# (Required) The language identifier of the language used in the
+# source code for gettext system, and the primary fallback language
+# (for which all strings must be present) when using the fluent
+# system.
+fallback_language = "en-GB"
+
+# Use the fluent localization system.
+[fluent]
+domain = "atuin"
+# (Required) The path to the assets directory.
+# The paths inside the assets directory should be structured like so:
+# `assets_dir/{language}/{domain}.ftl`
+assets_dir = "../../i18n"

crates/atuin-common/src/i18n.rs

@@ -0,0 +1,42 @@
+use i18n_embed::{
+    DesktopLanguageRequester,
+    fluent::{FluentLanguageLoader, fluent_language_loader},
+};
+pub use i18n_embed_fl::fl;
+use rust_embed::RustEmbed;
+
+#[derive(RustEmbed)]
+#[folder = "../../i18n"] // path to the compiled localization resources
+struct Localizations;
+
+pub use atuin_macro::tl;
+use lazy_static::lazy_static;
+
+lazy_static! {
+    // We assume that one LOADER is sufficient. Fluent provides more
+    // flexibility, but for now, this simplifies integration.
+    pub static ref LOADER: FluentLanguageLoader = {
+        // Load languages from central internationalization folder.
+        let language_loader: FluentLanguageLoader = fluent_language_loader!();
+        let requested_languages = DesktopLanguageRequester::requested_languages();
+
+        let _result = i18n_embed::select(
+            &language_loader, &Localizations, &requested_languages);
+        language_loader
+    };
+}
+
+#[macro_export]
+macro_rules! t {
+    // Case that t!("foo bar") is called with no runtime parameters to interpolate.
+    ($message_id:literal) => {
+        $crate::i18n::tl!($crate::i18n::fl, $crate::i18n::LOADER, $message_id)
+    };
+
+    // Case that t!("foo %{bar}", bar=baz.to_string()) is called with runtime parameters to interpolate.
+    ($message_id:literal, $($args:expr),*) => {{
+        $crate::i18n::tl!($crate::i18n::fl, $crate::i18n::LOADER, $message_id, $($args), *)
+    }};
+}
+
+pub use t;

crates/atuin-common/src/lib.rs

@@ -54,6 +54,7 @@ macro_rules! new_uuid {
 }
 
 pub mod api;
+pub mod i18n;
 pub mod record;
 pub mod shell;
 pub mod utils;

crates/atuin-macro/Cargo.toml

@@ -0,0 +1,41 @@
+[package]
+name = "atuin-macro"
+edition = "2021"
+description = "macro library for atuin"
+
+rust-version = { workspace = true }
+version = { workspace = true }
+authors = { workspace = true }
+license = { workspace = true }
+homepage = { workspace = true }
+repository = { workspace = true }
+
+[lib]
+proc-macro = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+time = { workspace = true }
+serde = { workspace = true }
+uuid = { workspace = true }
+typed-builder = { workspace = true }
+eyre = { workspace = true }
+sqlx = { workspace = true }
+semver = { workspace = true }
+thiserror = { workspace = true }
+directories = { workspace = true }
+sysinfo = "0.30.7"
+base64 = { workspace = true }
+getrandom = "0.2"
+sys-locale = "0.3.2"
+
+lazy_static = "1.4.0"
+i18n-embed = { version = "0.15.3", features = ["fluent", "fluent-system", "tr", "locale_config", "desktop-requester", "walkdir", "filesystem-assets"] }
+rust-embed = "8"
+i18n-embed-fl = "0.9.3"
+slugify = "0.1.0"
+paste = "1.0.15"
+syn = "2.0.96"
+quote = "1.0.38"
+proc-macro2 = "1.0.93"

crates/atuin-macro/i18n.toml

@@ -0,0 +1,13 @@
+# (Required) The language identifier of the language used in the
+# source code for gettext system, and the primary fallback language
+# (for which all strings must be present) when using the fluent
+# system.
+fallback_language = "en-GB"
+
+# Use the fluent localization system.
+[fluent]
+domain = "atuin"
+# (Required) The path to the assets directory.
+# The paths inside the assets directory should be structured like so:
+# `assets_dir/{language}/{domain}.ftl`
+assets_dir = "./tests/i18n"

crates/atuin-macro/src/lib.rs

@@ -0,0 +1,76 @@
+#![forbid(unsafe_code)]
+extern crate proc_macro;
+
+use proc_macro::TokenStream;
+use proc_macro2::TokenStream as TokenStream2;
+use quote::quote;
+use slugify::slugify;
+use syn::parse::Parser;
+
+fn literal_to_slug(literal: &syn::ExprLit) -> TokenStream2 {
+    // We pull out the actual text from the literal string expression.
+    let quoted: String = match &literal.lit {
+        syn::Lit::Str(message_id) => message_id.value(),
+        _ => panic!("Message ID must be a literal string"),
+    };
+    // ...and pass it to slugify,
+    let slug = slugify!(quoted.as_str());
+    // ...then turn it back into a literal string.
+    quote!(#slug)
+}
+
+#[proc_macro]
+pub fn tl(tokens: TokenStream) -> TokenStream {
+    // Begin by getting the individual arguments to tl!
+    let args = syn::punctuated::Punctuated::<syn::Expr, syn::Token![,]>::parse_terminated
+        .parse(tokens)
+        .unwrap();
+
+    let mut arg_iter = args.iter();
+
+    // The first should always be the fl! macro for fluent (cf. atuin-common/src/i18n.rs)
+    let fl = arg_iter.next().unwrap();
+    // atuin-common will send the universal loader as the second argument. This avoids
+    // every translation string having to explicitly pass it.
+    let loader = arg_iter.next().unwrap();
+
+    // The third argument should be the message ID. This logic takes the human-readable
+    // string and slugifies it. One of the main benefits of Fluent is that English-language
+    // ASCII is not the de facto reference (and things like gender and plurality can be
+    // encoded even where English makes no grammatical distinction). However, this approach
+    // still allows the `fl!` macro to be used directly, but saves having to switch all
+    // strings to slugs throughout the codebase just to make them translatable at all.
+
+    // It is possible that the string literal representing the message (e.g. "Danger, Bill Bobinson")
+    // appears wrapped in a group or not, so we handle both possibilities.
+    // We use literal_to_slug to turn it to a slug, e.g. "danger-bill-bobinson"
+    let message_id: proc_macro2::TokenStream = match arg_iter.next() {
+        Some(syn::Expr::Group(arg)) => match *arg.expr.clone() {
+            syn::Expr::Lit(arg) => literal_to_slug(&arg),
+            arg => panic!("Message ID {:?} must be a literal", arg),
+        },
+        Some(syn::Expr::Lit(arg)) => literal_to_slug(arg),
+        arg => panic!("Message ID {:?} must be a literal", arg),
+    };
+
+    // Reconstruct the arguments that we initially had, and pull in any extra ones
+    // that should go right through to Fluent. For example:
+    //     t!("Danger ${name}", name="Bill Bobinson")
+    //     -> tr!(fl, LOADER, "Danger ${name}", name="Bill Bobinson")
+    //     -> fl!(LOADER, "danger-name", name="Bill Bobinson")
+    // `danger-name` is then searched for in the i18n/ folder, and should map
+    // to a template like `Danger, { $name }` that Fluent can insert the parameter into.
+    let args: Vec<_> = arg_iter.collect();
+
+    // If there are no parameters, then Fluent can do this entirely statically.
+    // Otherwise, it will require runtime interpolation.
+    if args.is_empty() {
+        TokenStream::from(quote!(
+            #fl!(#loader, #message_id)
+        ))
+    } else {
+        TokenStream::from(quote!(
+            #fl!(#loader, #message_id, #(#args),*)
+        ))
+    }
+}

crates/atuin-macro/tests/basic.rs

@@ -0,0 +1,123 @@
+use i18n_embed::fluent::{fluent_language_loader, FluentLanguageLoader};
+pub use i18n_embed_fl::fl;
+use lazy_static::lazy_static;
+use rust_embed::RustEmbed;
+
+#[derive(RustEmbed)]
+#[folder = "tests/i18n"] // path to the compiled localization resources
+struct Localizations;
+
+pub use atuin_macro::tl;
+
+lazy_static! {
+    // We assume that one LOADER is sufficient. Fluent provides more
+    // flexibility, but for now, this simplifies integration.
+    pub static ref LOADER: FluentLanguageLoader = {
+        // Load languages from central internationalization folder.
+        let language_loader: FluentLanguageLoader = fluent_language_loader!();
+        let requested_languages = vec!["en-GB".parse().unwrap()];
+
+        let _result = i18n_embed::select(
+            &language_loader, &Localizations, &requested_languages);
+        language_loader
+    };
+}
+
+#[test]
+fn basic_tl_without_parameter() {
+    assert_eq!(
+        tl!(fl, LOADER, "Danger, Bill Bobinson"),
+        "Danger, William of Bobinson"
+    );
+}
+
+#[test]
+fn basic_tl_with_parameter() {
+    assert_eq!(
+        tl!(
+            fl,
+            LOADER,
+            "unrecognized subcommand '%{subcommand}'",
+            subcommand = "SUB"
+        ),
+        "unrecognised subcommand '\u{2068}SUB\u{2069}'"
+    );
+}
+
+#[test]
+fn tl_with_non_en_range_without_parameter() {
+    let language_loader: FluentLanguageLoader = fluent_language_loader!();
+    let requested_languages = vec!["ga-IE".parse().unwrap()];
+
+    let _result = i18n_embed::select(&language_loader, &Localizations, &requested_languages);
+
+    assert_eq!(
+        tl!(fl, language_loader, "Danger, Bill Bobinson"),
+        "Contúirt, a Uilliam Mac Bhoboin"
+    );
+}
+
+#[test]
+fn tl_with_non_en_range_with_parameter() {
+    let language_loader: FluentLanguageLoader = fluent_language_loader!();
+    let requested_languages = vec!["hi-IN".parse().unwrap()];
+
+    let _result = i18n_embed::select(&language_loader, &Localizations, &requested_languages);
+
+    assert_eq!(
+        tl!(
+            fl,
+            language_loader,
+            "Hello, my name is %{name}",
+            name = "रीमा"
+        ),
+        "नमस्ते, मेरा नाम \u{2068}रीमा\u{2069} है।"
+    );
+}
+
+#[test]
+fn tl_with_selector_parameter() {
+    let language_loader: FluentLanguageLoader = fluent_language_loader!();
+
+    let _result = i18n_embed::select(
+        &language_loader,
+        &Localizations,
+        &vec!["en-GB".parse().unwrap()],
+    );
+
+    assert_eq!(
+        tl!(fl, language_loader, "the user that has files", gender = "f"),
+        "the user that has files"
+    );
+
+    assert_eq!(
+        tl!(fl, language_loader, "the user that has files", gender = "m"),
+        "the user that has files"
+    );
+
+    assert_eq!(
+        tl!(fl, language_loader, "the user that has files", gender = "o"),
+        "the user that has files"
+    );
+
+    let _result = i18n_embed::select(
+        &language_loader,
+        &Localizations,
+        &vec!["ga-IE".parse().unwrap()],
+    );
+
+    assert_eq!(
+        tl!(fl, language_loader, "the user that has files", gender = "f"),
+        "an t-úsáideoir a bhfuil comhaid aici"
+    );
+
+    assert_eq!(
+        tl!(fl, language_loader, "the user that has files", gender = "m"),
+        "an t-úsáideoir a bhfuil comhaid aige"
+    );
+
+    assert_eq!(
+        tl!(fl, language_loader, "the user that has files", gender = "o"),
+        "an t-úsáideoir a bhfuil comhaid acu"
+    );
+}

crates/atuin-macro/tests/i18n/en-GB/atuin.ftl

@@ -0,0 +1,10 @@
+unrecognized-subcommand-subcommand =
+  unrecognised subcommand '{ $subcommand }'
+danger-bill-bobinson =
+  Danger, William of Bobinson
+hello-my-name-is-name =
+  Hello, my name is { $name }
+the-user-that-has-files =
+  { $gender ->
+    *[other] the user that has files
+  }

crates/atuin-macro/tests/i18n/ga-IE/atuin.ftl

@@ -0,0 +1,8 @@
+danger-bill-bobinson =
+  Contúirt, a Uilliam Mac Bhoboin
+the-user-that-has-files =
+  { $gender ->
+    [f] an t-úsáideoir a bhfuil comhaid aici
+    [m] an t-úsáideoir a bhfuil comhaid aige
+    *[other] an t-úsáideoir a bhfuil comhaid acu
+  }

crates/atuin-macro/tests/i18n/hi-IN/atuin.ftl

@@ -0,0 +1,2 @@
+hello-my-name-is-name =
+    नमस्ते, मेरा नाम { $name } है।