From cbffc51eac0bc04a8469f8b63eda35082c585356 Mon Sep 17 00:00:00 2001 From: void cat Date: Fri, 7 Feb 2025 00:54:46 +0800 Subject: [PATCH] ci: Add integrated test for L10n (#510) --- .github/workflows/cargo_test.yml | 1 + Cargo.lock | 18 +++ Cargo.toml | 1 + phira/Cargo.toml | 5 + phira/src/client.rs | 3 +- phira/src/lib.rs | 4 +- phira/src/login.rs | 2 +- phira/src/mp.rs | 2 +- phira/src/page/coll.rs | 2 +- phira/src/page/home.rs | 4 +- phira/src/page/library.rs | 2 +- phira/src/page/message.rs | 2 +- phira/src/page/offset.rs | 2 +- phira/src/page/respack.rs | 2 +- phira/src/page/settings.rs | 4 +- phira/src/rate.rs | 2 +- phira/src/resource.rs | 2 +- phira/src/scene.rs | 2 +- phira/src/scene/chapter.rs | 2 +- phira/src/scene/chart_order.rs | 2 +- phira/src/scene/event.rs | 2 +- phira/src/scene/profile.rs | 2 +- phira/src/scene/song.rs | 2 +- phira/src/tags.rs | 2 +- phira/tests/integrated_test.rs | 9 ++ prpr-l10n/Cargo.toml | 14 ++ prpr-l10n/src/global.rs | 42 ++++++ prpr-l10n/src/lib.rs | 79 ++++++++++ prpr-l10n/src/local.rs | 60 ++++++++ prpr-l10n/src/macros.rs | 99 +++++++++++++ prpr-l10n/src/tools.rs | 71 +++++++++ prpr-l10n/tests/langid.rs | 7 + prpr/Cargo.toml | 4 + prpr/src/core/chart.rs | 3 +- prpr/src/l10n.rs | 242 ------------------------------- prpr/src/lib.rs | 1 - prpr/src/parse/extra.rs | 3 +- prpr/src/parse/pec.rs | 2 +- prpr/src/parse/pgr.rs | 2 +- prpr/src/parse/rpe.rs | 2 +- prpr/src/scene.rs | 2 +- prpr/src/scene/ending.rs | 2 +- prpr/src/scene/game.rs | 2 +- prpr/src/ui/chart_info.rs | 2 +- prpr/src/ui/dialog.rs | 2 +- prpr/tests/integrated_test.rs | 9 ++ 46 files changed, 454 insertions(+), 277 deletions(-) create mode 100644 phira/tests/integrated_test.rs create mode 100644 prpr-l10n/Cargo.toml create mode 100644 prpr-l10n/src/global.rs create mode 100644 prpr-l10n/src/lib.rs create mode 100644 prpr-l10n/src/local.rs create mode 100644 prpr-l10n/src/macros.rs create mode 100644 prpr-l10n/src/tools.rs create mode 100644 prpr-l10n/tests/langid.rs delete mode 100644 prpr/src/l10n.rs create mode 100644 prpr/tests/integrated_test.rs diff --git a/.github/workflows/cargo_test.yml b/.github/workflows/cargo_test.yml index d374fff1..08b36499 100644 --- a/.github/workflows/cargo_test.yml +++ b/.github/workflows/cargo_test.yml @@ -34,3 +34,4 @@ jobs: - name: Run tests run: | cargo test -p phira --no-default-features -- --skip test_parse_chart + cargo test -p prpr --no-default-features diff --git a/Cargo.lock b/Cargo.lock index 8f1a9e4a..acfb9dae 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2438,6 +2438,8 @@ dependencies = [ "cacache", "chrono", "dotenv-build", + "fluent", + "fluent-syntax", "futures-util", "hex", "image", @@ -2456,6 +2458,7 @@ dependencies = [ "phira-mp-common", "pollster", "prpr", + "prpr-l10n", "rand", "regex", "reqwest", @@ -2697,6 +2700,7 @@ dependencies = [ "ordered-float 3.9.2", "phf", "prpr-avc", + "prpr-l10n", "rand", "regex", "rfd", @@ -2730,6 +2734,20 @@ dependencies = [ "tracing", ] +[[package]] +name = "prpr-l10n" +version = "0.1.0" +dependencies = [ + "fluent", + "fluent-syntax", + "lru 0.9.0", + "once_cell", + "sys-locale", + "tracing", + "unic-langid", + "walkdir", +] + [[package]] name = "prpr-pbc" version = "0.1.0" diff --git a/Cargo.toml b/Cargo.toml index e80999bf..e78813f3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "prpr", "prpr-avc", "prpr-pbc", + "prpr-l10n", "phira", "phira-main", "phira-monitor", diff --git a/phira/Cargo.toml b/phira/Cargo.toml index df1a5a6a..61f1c0ce 100644 --- a/phira/Cargo.toml +++ b/phira/Cargo.toml @@ -35,6 +35,7 @@ once_cell = "*" openssl = { version = "*", features = ["vendored"] } pollster = "0.3.0" prpr = { path = "../prpr", features = ["log"], default-features = false } +prpr-l10n ={ path = "../prpr-l10n" } rand = "0.8.5" regex = "1.7.0" reqwest = { version = "0.12.5", features = ["json", "stream", "gzip"] } @@ -75,3 +76,7 @@ objc-foundation = "*" [build-dependencies] dotenv-build = "0.1" + +[dev-dependencies] +fluent = "0.16.0" +fluent-syntax = "0.11.0" diff --git a/phira/src/client.rs b/phira/src/client.rs index d7c8bcb6..8522622a 100644 --- a/phira/src/client.rs +++ b/phira/src/client.rs @@ -6,7 +6,8 @@ use crate::{anti_addiction_action, get_data, get_data_mut, save_data}; use anyhow::{anyhow, bail, Context, Result}; use arc_swap::ArcSwap; use once_cell::sync::Lazy; -use prpr::{l10n::LANG_IDENTS, scene::SimpleRecord}; +use prpr::scene::SimpleRecord; +use prpr_l10n::LANG_IDENTS; use reqwest::{header, ClientBuilder, Method, RequestBuilder, Response, StatusCode}; use serde::{Deserialize, Serialize}; use serde_json::json; diff --git a/phira/src/lib.rs b/phira/src/lib.rs index 94f65d78..73375245 100644 --- a/phira/src/lib.rs +++ b/phira/src/lib.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("common" ttl crate::); +prpr_l10n::tl_file!("common" ttl crate::); #[cfg(feature = "closed")] mod inner; @@ -28,13 +28,13 @@ use prpr::{ build_conf, core::{init_assets, PGR_FONT}, ext::SafeTexture, - l10n::{set_prefered_locale, GLOBAL, LANGS}, log, scene::{show_error, show_message}, time::TimeManager, ui::{FontArc, TextPainter}, Main, }; +use prpr_l10n::{set_prefered_locale, GLOBAL, LANGS}; use scene::MainScene; use std::sync::{mpsc, Mutex}; use tracing::{error, info}; diff --git a/phira/src/login.rs b/phira/src/login.rs index 4c72bac2..d9aa5cc8 100644 --- a/phira/src/login.rs +++ b/phira/src/login.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("login"); +prpr_l10n::tl_file!("login"); use crate::{ client::{Client, LoginParams, User, UserManager}, diff --git a/phira/src/mp.rs b/phira/src/mp.rs index 9bf7c878..59f6914c 100644 --- a/phira/src/mp.rs +++ b/phira/src/mp.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("multiplayer" mtl); +prpr_l10n::tl_file!("multiplayer" mtl); mod panel; pub use panel::MPPanel; diff --git a/phira/src/page/coll.rs b/phira/src/page/coll.rs index a1018aba..d0ab2c74 100644 --- a/phira/src/page/coll.rs +++ b/phira/src/page/coll.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("collection"); +prpr_l10n::tl_file!("collection"); use super::{Illustration, NextPage, Page, SharedState}; use crate::{icons::Icons, load_res_tex, resource::rtl, scene::ChapterScene}; diff --git a/phira/src/page/home.rs b/phira/src/page/home.rs index fba02d03..53ae1ab5 100644 --- a/phira/src/page/home.rs +++ b/phira/src/page/home.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("home"); +prpr_l10n::tl_file!("home"); use super::{ load_font_with_cksum, set_bold_font, EventPage, LibraryPage, MessagePage, NextPage, Page, ResPackPage, SFader, SettingsPage, SharedState, @@ -15,6 +15,7 @@ use crate::{ sync_data, threed::ThreeD, }; +use prpr_l10n::LANG_IDENTS; use ::rand::{random, thread_rng, Rng}; use anyhow::{bail, Context, Result}; use chrono::NaiveDate; @@ -24,7 +25,6 @@ use prpr::{ core::BOLD_FONT, ext::{open_url, screen_aspect, semi_black, semi_white, RectExt, SafeTexture, ScaleType}, info::ChartInfo, - l10n::LANG_IDENTS, scene::{show_error, NextScene}, task::Task, ui::{button_hit_large, clip_rounded_rect, ClipType, DRectButton, Dialog, FontArc, RectButton, Scroll, Ui}, diff --git a/phira/src/page/library.rs b/phira/src/page/library.rs index 3ca4db47..2ec3c38f 100644 --- a/phira/src/page/library.rs +++ b/phira/src/page/library.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("library"); +prpr_l10n::tl_file!("library"); use super::{CollectionPage, NextPage, Page, SharedState}; use crate::{ diff --git a/phira/src/page/message.rs b/phira/src/page/message.rs index 4ab45f49..4ddd7a5a 100644 --- a/phira/src/page/message.rs +++ b/phira/src/page/message.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("message"); +prpr_l10n::tl_file!("message"); use std::borrow::Cow; diff --git a/phira/src/page/offset.rs b/phira/src/page/offset.rs index a346e554..3f1b0cf3 100644 --- a/phira/src/page/offset.rs +++ b/phira/src/page/offset.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("cali"); +prpr_l10n::tl_file!("cali"); use std::borrow::Cow; diff --git a/phira/src/page/respack.rs b/phira/src/page/respack.rs index 2cb6ad7f..7954a09c 100644 --- a/phira/src/page/respack.rs +++ b/phira/src/page/respack.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("respack"); +prpr_l10n::tl_file!("respack"); use super::{Page, SharedState}; use crate::{ diff --git a/phira/src/page/settings.rs b/phira/src/page/settings.rs index 0c93057c..4f3333ac 100644 --- a/phira/src/page/settings.rs +++ b/phira/src/page/settings.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("settings"); +prpr_l10n::tl_file!("settings"); use super::{NextPage, OffsetPage, Page, SharedState}; use crate::{ @@ -15,11 +15,11 @@ use macroquad::prelude::*; use prpr::{ core::BOLD_FONT, ext::{open_url, poll_future, semi_white, LocalTask, RectExt, SafeTexture}, - l10n::{LanguageIdentifier, LANG_IDENTS, LANG_NAMES}, scene::{request_input, return_input, show_error, show_message, take_input}, task::Task, ui::{DRectButton, Scroll, Slider, Ui}, }; +use prpr_l10n::{LanguageIdentifier, LANG_IDENTS, LANG_NAMES}; use reqwest::Url; use std::{borrow::Cow, fs, io, net::ToSocketAddrs, path::PathBuf, sync::atomic::Ordering}; diff --git a/phira/src/rate.rs b/phira/src/rate.rs index ee841cf4..d67f3723 100644 --- a/phira/src/rate.rs +++ b/phira/src/rate.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("rate"); +prpr_l10n::tl_file!("rate"); use crate::page::Fader; use macroquad::prelude::*; diff --git a/phira/src/resource.rs b/phira/src/resource.rs index e17743b7..bb649d5b 100644 --- a/phira/src/resource.rs +++ b/phira/src/resource.rs @@ -1 +1 @@ -prpr::tl_file!("resource" rtl); +prpr_l10n::tl_file!("resource" rtl); diff --git a/phira/src/scene.rs b/phira/src/scene.rs index e4dcbf01..1e4b26b1 100644 --- a/phira/src/scene.rs +++ b/phira/src/scene.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("import" itl); +prpr_l10n::tl_file!("import" itl); mod chart_order; pub use chart_order::{ChartOrder, ORDERS}; diff --git a/phira/src/scene/chapter.rs b/phira/src/scene/chapter.rs index 36745974..b0fff563 100644 --- a/phira/src/scene/chapter.rs +++ b/phira/src/scene/chapter.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("chapter"); +prpr_l10n::tl_file!("chapter"); use crate::{ anim::Anim, diff --git a/phira/src/scene/chart_order.rs b/phira/src/scene/chart_order.rs index 142a5729..329fa453 100644 --- a/phira/src/scene/chart_order.rs +++ b/phira/src/scene/chart_order.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("chart_order"); +prpr_l10n::tl_file!("chart_order"); use crate::page::ChartItem; diff --git a/phira/src/scene/event.rs b/phira/src/scene/event.rs index 6f38329d..dc968367 100644 --- a/phira/src/scene/event.rs +++ b/phira/src/scene/event.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("event"); +prpr_l10n::tl_file!("event"); use super::{render_ldb, LdbDisplayItem, ProfileScene}; use crate::{ diff --git a/phira/src/scene/profile.rs b/phira/src/scene/profile.rs index 0fb7adca..9819090d 100644 --- a/phira/src/scene/profile.rs +++ b/phira/src/scene/profile.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("profile"); +prpr_l10n::tl_file!("profile"); use super::{confirm_delete, TEX_BACKGROUND, TEX_ICON_BACK}; use crate::{ diff --git a/phira/src/scene/song.rs b/phira/src/scene/song.rs index 2a6a22df..bd358c84 100644 --- a/phira/src/scene/song.rs +++ b/phira/src/scene/song.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("song"); +prpr_l10n::tl_file!("song"); #[cfg(feature = "video")] use super::UnlockScene; diff --git a/phira/src/tags.rs b/phira/src/tags.rs index 0c0032bf..03589d56 100644 --- a/phira/src/tags.rs +++ b/phira/src/tags.rs @@ -1,4 +1,4 @@ -prpr::tl_file!("tags"); +prpr_l10n::tl_file!("tags"); use crate::{client::Permissions, page::Fader}; use macroquad::prelude::*; diff --git a/phira/tests/integrated_test.rs b/phira/tests/integrated_test.rs new file mode 100644 index 00000000..bd45e911 --- /dev/null +++ b/phira/tests/integrated_test.rs @@ -0,0 +1,9 @@ +use prpr_l10n::tools::check_langfile; + +#[test] +fn check_all() { + match check_langfile(concat!(env!("CARGO_MANIFEST_DIR"), "/locales/")) { + Ok(_) => {} + Err(e) => panic!("Error: {}", e), + } +} diff --git a/prpr-l10n/Cargo.toml b/prpr-l10n/Cargo.toml new file mode 100644 index 00000000..0445cb48 --- /dev/null +++ b/prpr-l10n/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "prpr-l10n" +version = "0.1.0" +edition = "2021" + +[dependencies] +fluent = "0.16.0" +fluent-syntax = "0.11.0" +lru = "0.9.0" +once_cell = "1.16.0" +sys-locale = "0.3.1" +tracing = "0.1.37" +unic-langid = { version = "0.9.1", features = ["macros"] } +walkdir = "2.3.3" diff --git a/prpr-l10n/src/global.rs b/prpr-l10n/src/global.rs new file mode 100644 index 00000000..1a0c3036 --- /dev/null +++ b/prpr-l10n/src/global.rs @@ -0,0 +1,42 @@ +use std::{collections::HashMap, sync::Mutex}; + +use tracing::warn; +use unic_langid::LanguageIdentifier; + +use crate::{fallback_langid, FALLBACK_LANG, LANG_IDENTS}; + +pub struct L10nGlobal { + pub lang_map: HashMap, + pub order: Mutex>, +} + +impl Default for L10nGlobal { + fn default() -> Self { + Self::new() + } +} + +impl L10nGlobal { + pub fn new() -> Self { + let mut lang_map = HashMap::new(); + let mut order = Vec::new(); + let locale_lang = sys_locale::get_locale().unwrap_or_else(|| String::from(FALLBACK_LANG)); + let locale_lang: LanguageIdentifier = locale_lang.parse().unwrap_or_else(|_| { + warn!("Invalid locale detected, defaulting to `{}`", FALLBACK_LANG); + // Debug log: send lang tag to log + warn!("Locale detected: {:?}", locale_lang); + fallback_langid!() + }); + for (id, lang) in LANG_IDENTS.iter().enumerate() { + lang_map.insert(lang.clone(), id); + if *lang == locale_lang { + order.push(id); + } + } + order.push(*lang_map.get(&fallback_langid!()).unwrap()); + Self { + lang_map, + order: order.into(), + } + } +} diff --git a/prpr-l10n/src/lib.rs b/prpr-l10n/src/lib.rs new file mode 100644 index 00000000..3511e96a --- /dev/null +++ b/prpr-l10n/src/lib.rs @@ -0,0 +1,79 @@ +//! Localization utilities. + +pub use fluent::{fluent_args, FluentBundle, FluentResource}; +pub use once_cell::sync::Lazy; +pub use unic_langid::LanguageIdentifier; + +use std::sync::atomic::{AtomicU8, Ordering}; + +mod global; +pub use global::*; + +mod local; +pub use local::*; + +mod macros; + +pub mod tools; + +langs! { + "en-US": "English", + "fr-FR": "Français", + "id-ID": "Bahasa Indonesia", + "ja-JP": "日本語", + "ko-KR": "한국어", + "pl-PL": "Polski", + "pt-BR": "Português", + "ru-RU": "Русский", + "th-TH": "แบบไทย", + "vi-VN": "Tiếng Việt", + "zh-CN": "简体中文", + "zh-TW": "繁體中文" +} + +#[macro_export] +macro_rules! fallback_langid { + () => { + unic_langid::langid!("en-US") + }; +} + +pub const FALLBACK_LANG: &str = "en-US"; + +pub static GLOBAL: Lazy = Lazy::new(L10nGlobal::new); + +pub fn set_prefered_locale(locale: Option) { + let mut ids = Vec::new(); + let map = &GLOBAL.lang_map; + if let Some(lang) = locale.and_then(|it| map.get(&it)) { + ids.push(*lang); + } + if let Some(lang) = sys_locale::get_locale() + .and_then(|it| it.parse::().ok()) + .and_then(|it| map.get(&it)) + { + ids.push(*lang); + } + ids.push(*map.get(&fallback_langid!()).unwrap()); + *GLOBAL.order.lock().unwrap() = ids; + GENERATION.fetch_add(1, Ordering::Relaxed); +} + +pub fn locale_order() -> Vec { + GLOBAL.order.lock().unwrap().clone() +} + +pub struct L10nBundles { + inner: Vec>, +} + +impl From>> for L10nBundles { + fn from(inner: Vec>) -> Self { + Self { inner } + } +} + +unsafe impl Send for L10nBundles {} +unsafe impl Sync for L10nBundles {} + +pub static GENERATION: AtomicU8 = AtomicU8::new(0); diff --git a/prpr-l10n/src/local.rs b/prpr-l10n/src/local.rs new file mode 100644 index 00000000..41a693b2 --- /dev/null +++ b/prpr-l10n/src/local.rs @@ -0,0 +1,60 @@ +use fluent::{FluentArgs, FluentError}; +use fluent_syntax::ast::Pattern; +use lru::LruCache; +use std::{borrow::Cow, sync::atomic::Ordering}; +use tracing::warn; + +use crate::{L10nBundles, GENERATION, GLOBAL}; + +pub struct L10nLocal { + bundles: &'static L10nBundles, + cache: LruCache, (usize, &'static Pattern<&'static str>)>, + generation: u8, +} + +impl L10nLocal { + pub fn new(bundles: &'static L10nBundles) -> Self { + Self { + bundles, + cache: LruCache::new(16.try_into().unwrap()), + generation: 0, + } + } + + fn format_with_errors<'s>(&mut self, key: Cow<'static, str>, args: Option<&'s FluentArgs<'s>>, errors: &mut Vec) -> Cow<'s, str> { + let gen = GENERATION.load(Ordering::Relaxed); + if gen > self.generation { + self.generation = gen; + self.cache.clear(); + } + if let Some((id, pattern)) = { + let get_result = self.cache.get(&key); + if get_result.is_none() { + let guard = GLOBAL.order.lock().unwrap(); + guard + .iter() + .filter_map(|id| self.bundles.inner[*id].get_message(&key).map(|msg| (*id, msg))) + .next() + .map(|(id, message)| (id, message.value().unwrap())) + .map(|val| self.cache.get_or_insert(key.clone(), || val)) + } else { + get_result + } + } { + unsafe { std::mem::transmute(self.bundles.inner[*id].format_pattern(pattern, args, errors)) } + } else { + warn!("no translation found for {key}, returning key"); + key + } + } + + pub fn format<'s>(&mut self, key: impl Into>, args: Option<&'s FluentArgs<'s>>) -> Cow<'s, str> { + let mut errors = Vec::new(); + let key: Cow<'static, str> = key.into(); + let res = self.format_with_errors(key.clone(), args, &mut errors); + for error in errors { + warn!("l10n error {key}: {error:?}"); + } + res + } +} diff --git a/prpr-l10n/src/macros.rs b/prpr-l10n/src/macros.rs new file mode 100644 index 00000000..3f67dc8c --- /dev/null +++ b/prpr-l10n/src/macros.rs @@ -0,0 +1,99 @@ +#[macro_export] +macro_rules! langs { + ($($lang_id:literal: $lang_name:expr),*) => { + macro_rules! __create_bundles_builder { + ($d:tt) => { + #[macro_export] + macro_rules! create_bundles { + ($d file:literal) => {{ + let mut bundles = Vec::new(); + $( + bundles.push($crate::create_bundle!($lang_id, $file)); + )* + bundles + }}; + } + } + } + + macro_rules! __count_builder { + ($d:tt) => { + macro_rules! count { + () => (0usize); + ($d _a:tt $d _b:tt $d _c:tt $d _d:tt $d _e:tt $d ($d tail:tt)*) => ( + 5usize + count!($d ($d tail)*) + ); + ($d _a:tt $d _b:tt $d ($d tail:tt)*) => (2usize + count!($d ($d tail)*)); + ($d _a:tt $d ($d tail:tt)*) => (1usize + count!($d ($d tail)*)); + } + } + } + + __create_bundles_builder!($); + + __count_builder!($); + + pub const LANG_COUNT: usize = count!($($lang_id)*); + + pub static LANGS: [&str; LANG_COUNT] = [$($lang_id,)*]; + pub static LANG_NAMES: [&str; LANG_COUNT] = [$($lang_name,)*]; + pub static LANG_IDENTS: Lazy<[LanguageIdentifier; LANG_COUNT]> = Lazy::new(|| LANGS.map(|lang| lang.parse().unwrap())); + }; +} + +#[macro_export] +macro_rules! create_bundle { + ($locale:literal, $file:literal) => {{ + let mut bundle = $crate::FluentBundle::new($crate::LANG_IDENTS.iter().cloned().collect()); + bundle + .add_resource( + $crate::FluentResource::try_new( + include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/locales/", $locale, "/", $file, ".ftl")).to_owned(), + ) + .unwrap(), + ) + .unwrap(); + bundle.set_use_isolating(false); + bundle + }}; +} + +#[macro_export] +macro_rules! tl_file { + ($file:literal) => { + $crate::tl_file!($file tl); + }; + ($file:literal $macro_name:ident $($p:tt)*) => { + static L10N_BUNDLES: $crate::Lazy<$crate::L10nBundles> = $crate::Lazy::new(|| $crate::create_bundles!($file).into()); + + thread_local! { + pub static L10N_LOCAL: std::cell::RefCell<$crate::L10nLocal> = $crate::L10nLocal::new(&*L10N_BUNDLES).into(); + } + + macro_rules! __tl_builder { + ($d:tt) => { + macro_rules! $macro_name { + ($d key:expr) => { + $($p)* L10N_LOCAL.with(|it| it.borrow_mut().format($key, None)) + }; + ($d key:expr, $d args:expr) => { + $($p)* L10N_LOCAL.with(|it| it.borrow_mut().format($key, Some($args))) + }; + ($d key:expr, $d ($d name:expr => $d value:expr),+) => { + $($p)* L10N_LOCAL.with(|it| it.borrow_mut().format($key, Some(&$crate::fluent_args![$d($d name => $d value), *])).to_string()) + }; + (err $d ($d body:tt)*) => { + anyhow::Error::msg($macro_name!($d($d body)*)) + }; + (bail $d ($d body:tt)*) => { + return anyhow::Result::Err(anyhow::Error::msg($macro_name!($d($d body)*))) + }; + } + + pub(crate) use $macro_name; + } + } + + __tl_builder!($); + }; +} diff --git a/prpr-l10n/src/tools.rs b/prpr-l10n/src/tools.rs new file mode 100644 index 00000000..60cc8bfa --- /dev/null +++ b/prpr-l10n/src/tools.rs @@ -0,0 +1,71 @@ +use std::{collections::HashSet, error::Error, fmt::Display, path::Path}; + +use walkdir::WalkDir; + +use crate::LANGS; + +#[derive(Debug)] +struct IllegalLanguages { + pub languages: Vec, +} + +impl Display for IllegalLanguages { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:?}", self.languages)?; + Ok(()) + } +} + +impl Error for IllegalLanguages {} + +fn get_ftl_files(path: &Path) -> Result, Box> { + let mut files = HashSet::new(); + for entry in WalkDir::new(path) { + let entry = entry?; + if entry.file_type().is_file() { + let path = entry.path(); + if path.extension().map_or(false, |ext| ext == "ftl") { + let relative_path = path.strip_prefix(path.parent().unwrap())?; + let normalized = relative_path.to_string_lossy().replace('\\', "/"); + files.insert(normalized); + } + } + } + Ok(files) +} + +pub fn check_langfile(path: &str) -> Result<(), Box> { + let locales_dir = Path::new(path); + let zh_cn_dir = locales_dir.join("zh-CN"); + let all_locales: [std::path::PathBuf; 12] = LANGS.map(|x| locales_dir.join(x)); + let zh_cn_files = get_ftl_files(&zh_cn_dir)?; + let mut inconsistent_languages = Vec::new(); + let mut i = 0; + while i < LANGS.len() { + let path = all_locales[i].to_owned(); + if path.is_dir() { + let lang_code = LANGS[i]; + i += 1; + if lang_code == "zh-CN" { + continue; + } + + match get_ftl_files(&path) { + Ok(files) => { + if files != zh_cn_files { + inconsistent_languages.push(lang_code); + } + } + Err(_) => inconsistent_languages.push(lang_code), + } + } + } + + if !inconsistent_languages.is_empty() { + return Err(Box::new(IllegalLanguages { + languages: inconsistent_languages.iter().map(|x| x.to_string()).collect(), + })); + } + + Ok(()) +} diff --git a/prpr-l10n/tests/langid.rs b/prpr-l10n/tests/langid.rs new file mode 100644 index 00000000..2ae393e9 --- /dev/null +++ b/prpr-l10n/tests/langid.rs @@ -0,0 +1,7 @@ +use prpr_l10n::Lazy; + +#[test] +fn check_langid() { + // Lang ID is illegal if panicked + Lazy::force(&prpr_l10n::LANG_IDENTS); +} diff --git a/prpr/Cargo.toml b/prpr/Cargo.toml index dcaf0c6f..f9b4e1bd 100644 --- a/prpr/Cargo.toml +++ b/prpr/Cargo.toml @@ -69,6 +69,7 @@ tracing-subscriber = { version = "0.3.17", optional = true } colored = { version = "=2.0.0", optional = true } prpr-avc = { path = "../prpr-avc", optional = true } +prpr-l10n ={ path = "../prpr-l10n" } ahash = "=0.8.6" cfg-expr = "=0.15.4" @@ -111,3 +112,6 @@ wasm-bindgen-futures = "0.4" [build-dependencies] walkdir = "2.3.2" + +[dev-dependencies] +walkdir = "2.3.2" diff --git a/prpr/src/core/chart.rs b/prpr/src/core/chart.rs index 35fe2bf7..bb9032d1 100644 --- a/prpr/src/core/chart.rs +++ b/prpr/src/core/chart.rs @@ -4,7 +4,6 @@ use anyhow::{Context, Result}; use macroquad::prelude::*; use sasa::AudioClip; use std::{cell::RefCell, collections::HashMap}; -use tracing::warn; #[derive(Default)] pub struct ChartExtra { @@ -116,7 +115,7 @@ impl Chart { #[cfg(feature = "video")] for video in &mut self.extra.videos { if let Err(err) = video.update(res.time) { - warn!("video error: {err:?}"); + tracing::warn!("video error: {err:?}"); } } } diff --git a/prpr/src/l10n.rs b/prpr/src/l10n.rs deleted file mode 100644 index af09c1a9..00000000 --- a/prpr/src/l10n.rs +++ /dev/null @@ -1,242 +0,0 @@ -//! Localization utilities. - -pub use fluent::{fluent_args, FluentBundle, FluentResource}; -pub use once_cell::sync::Lazy; -pub use unic_langid::{langid, LanguageIdentifier}; - -use fluent::{FluentArgs, FluentError}; -use fluent_syntax::ast::Pattern; -use lru::LruCache; -use std::{ - borrow::Cow, - collections::HashMap, - sync::{ - atomic::{AtomicU8, Ordering}, - Mutex, - }, -}; -use sys_locale::get_locale; -use tracing::warn; - -pub static LANGS: [&str; 12] = [ - "en-US", "fr-FR", "id-ID", "ja-JP", "ko-KR", "pl-PL", "pt-BR", "ru-RU", "th-TH", "vi-VN", "zh-CN", "zh-TW", -]; // this should be consistent with the macro below (create_bundles) -pub static LANG_NAMES: [&str; 12] = [ - "English", - "Français", - "Bahasa Indonesia", - "日本語", - "한국어", - "Polski", - "Português", - "Русский", - "แบบไทย", - "Tiếng Việt", - "简体中文", - "繁體中文", -]; // this should be consistent with the macro below (create_bundles) -pub static LANG_IDENTS: Lazy<[LanguageIdentifier; 12]> = Lazy::new(|| LANGS.map(|lang| lang.parse().unwrap())); - -#[macro_export] -macro_rules! create_bundle { - ($locale:literal, $file:literal) => {{ - let mut bundle = $crate::l10n::FluentBundle::new($crate::l10n::LANG_IDENTS.iter().cloned().collect()); - bundle - .add_resource( - $crate::l10n::FluentResource::try_new( - include_str!(concat!(env!("CARGO_MANIFEST_DIR"), "/locales/", $locale, "/", $file, ".ftl")).to_owned(), - ) - .unwrap(), - ) - .unwrap(); - bundle.set_use_isolating(false); - bundle - }}; -} - -#[macro_export] -macro_rules! create_bundles { - ($file:literal) => {{ - let mut bundles = Vec::new(); - bundles.push($crate::create_bundle!("en-US", $file)); - bundles.push($crate::create_bundle!("fr-FR", $file)); - bundles.push($crate::create_bundle!("id-ID", $file)); - bundles.push($crate::create_bundle!("ja-JP", $file)); - bundles.push($crate::create_bundle!("ko-KR", $file)); - bundles.push($crate::create_bundle!("pl-PL", $file)); - bundles.push($crate::create_bundle!("pt-BR", $file)); - bundles.push($crate::create_bundle!("ru-RU", $file)); - bundles.push($crate::create_bundle!("th-TH", $file)); - bundles.push($crate::create_bundle!("vi-VN", $file)); - bundles.push($crate::create_bundle!("zh-CN", $file)); - bundles.push($crate::create_bundle!("zh-TW", $file)); - bundles - }}; -} - -pub struct L10nGlobal { - pub lang_map: HashMap, - pub order: Mutex>, -} - -impl Default for L10nGlobal { - fn default() -> Self { - Self::new() - } -} - -impl L10nGlobal { - pub fn new() -> Self { - let mut lang_map = HashMap::new(); - let mut order = Vec::new(); - let locale_lang = get_locale().unwrap_or_else(|| String::from("en-US")); - let locale_lang: LanguageIdentifier = locale_lang.parse().unwrap_or_else(|_| { - warn!("Invalid locale detected, defaulting to en-US"); - // Debug log: send lang tag to log - warn!("Locale detected: {:?}", locale_lang); - langid!("en-US") - }); - for (id, lang) in LANG_IDENTS.iter().enumerate() { - lang_map.insert(lang.clone(), id); - if *lang == locale_lang { - order.push(id); - } - } - order.push(*lang_map.get(&langid!("en-US")).unwrap()); - Self { - lang_map, - order: order.into(), - } - } -} - -pub static GLOBAL: Lazy = Lazy::new(L10nGlobal::new); - -pub fn set_prefered_locale(locale: Option) { - let mut ids = Vec::new(); - let map = &GLOBAL.lang_map; - if let Some(lang) = locale.and_then(|it| map.get(&it)) { - ids.push(*lang); - } - if let Some(lang) = get_locale() - .and_then(|it| it.parse::().ok()) - .and_then(|it| map.get(&it)) - { - ids.push(*lang); - } - ids.push(*map.get(&langid!("en-US")).unwrap()); - *GLOBAL.order.lock().unwrap() = ids; - GENERATION.fetch_add(1, Ordering::Relaxed); -} - -pub fn locale_order() -> Vec { - GLOBAL.order.lock().unwrap().clone() -} - -pub struct L10nBundles { - inner: Vec>, -} - -impl From>> for L10nBundles { - fn from(inner: Vec>) -> Self { - Self { inner } - } -} - -unsafe impl Send for L10nBundles {} -unsafe impl Sync for L10nBundles {} - -pub static GENERATION: AtomicU8 = AtomicU8::new(0); - -pub struct L10nLocal { - bundles: &'static L10nBundles, - cache: LruCache, (usize, &'static Pattern<&'static str>)>, - generation: u8, -} - -impl L10nLocal { - pub fn new(bundles: &'static L10nBundles) -> Self { - Self { - bundles, - cache: LruCache::new(16.try_into().unwrap()), - generation: 0, - } - } - - fn format_with_errors<'s>(&mut self, key: Cow<'static, str>, args: Option<&'s FluentArgs<'s>>, errors: &mut Vec) -> Cow<'s, str> { - let gen = GENERATION.load(Ordering::Relaxed); - if gen > self.generation { - self.generation = gen; - self.cache.clear(); - } - if let Some((id, pattern)) = { - let get_result = self.cache.get(&key); - if get_result.is_none() { - let guard = GLOBAL.order.lock().unwrap(); - guard - .iter() - .filter_map(|id| self.bundles.inner[*id].get_message(&key).map(|msg| (*id, msg))) - .next() - .map(|(id, message)| (id, message.value().unwrap())) - .map(|val| self.cache.get_or_insert(key.clone(), || val)) - } else { - get_result - } - } { - unsafe { std::mem::transmute(self.bundles.inner[*id].format_pattern(pattern, args, errors)) } - } else { - warn!("no translation found for {key}, returning key"); - key - } - } - - pub fn format<'s>(&mut self, key: impl Into>, args: Option<&'s FluentArgs<'s>>) -> Cow<'s, str> { - let mut errors = Vec::new(); - let key: Cow<'static, str> = key.into(); - let res = self.format_with_errors(key.clone(), args, &mut errors); - for error in errors { - warn!("l10n error {key}: {error:?}"); - } - res - } -} - -#[macro_export] -macro_rules! tl_file { - ($file:literal) => { - $crate::tl_file!($file tl); - }; - ($file:literal $macro_name:ident $($p:tt)*) => { - static L10N_BUNDLES: $crate::l10n::Lazy<$crate::l10n::L10nBundles> = $crate::l10n::Lazy::new(|| $crate::create_bundles!($file).into()); - - thread_local! { - pub static L10N_LOCAL: std::cell::RefCell<$crate::l10n::L10nLocal> = $crate::l10n::L10nLocal::new(&*L10N_BUNDLES).into(); - } - - macro_rules! __tl_builder { - ($d:tt) => { - macro_rules! $macro_name { - ($d key:expr) => { - $($p)* L10N_LOCAL.with(|it| it.borrow_mut().format($key, None)) - }; - ($d key:expr, $d args:expr) => { - $($p)* L10N_LOCAL.with(|it| it.borrow_mut().format($key, Some($args))) - }; - ($d key:expr, $d ($d name:expr => $d value:expr),+) => { - $($p)* L10N_LOCAL.with(|it| it.borrow_mut().format($key, Some(&$crate::l10n::fluent_args![$d($d name => $d value), *])).to_string()) - }; - (err $d ($d body:tt)*) => { - anyhow::Error::msg($macro_name!($d($d body)*)) - }; - (bail $d ($d body:tt)*) => { - return anyhow::Result::Err(anyhow::Error::msg($macro_name!($d($d body)*))) - }; - } - - pub(crate) use $macro_name; - } - } - - __tl_builder!($); - }; -} diff --git a/prpr/src/lib.rs b/prpr/src/lib.rs index 15a139f4..3c806a3f 100644 --- a/prpr/src/lib.rs +++ b/prpr/src/lib.rs @@ -8,7 +8,6 @@ pub mod ext; pub mod fs; pub mod info; pub mod judge; -pub mod l10n; pub mod parse; pub mod particle; pub mod scene; diff --git a/prpr/src/parse/extra.rs b/prpr/src/parse/extra.rs index 24b17f69..dca15419 100644 --- a/prpr/src/parse/extra.rs +++ b/prpr/src/parse/extra.rs @@ -1,4 +1,4 @@ -crate::tl_file!("parser" ptl); +prpr_l10n::tl_file!("parser" ptl); use super::RPE_TWEEN_MAP; use crate::{ @@ -124,6 +124,7 @@ struct ExtEffect { global: bool, } +#[allow(dead_code)] #[derive(Deserialize)] struct ExtVideo { path: String, diff --git a/prpr/src/parse/pec.rs b/prpr/src/parse/pec.rs index 7161f99c..697e1717 100644 --- a/prpr/src/parse/pec.rs +++ b/prpr/src/parse/pec.rs @@ -1,4 +1,4 @@ -crate::tl_file!("parser" ptl); +prpr_l10n::tl_file!("parser" ptl); use super::{process_lines, RPE_TWEEN_MAP}; use crate::{ diff --git a/prpr/src/parse/pgr.rs b/prpr/src/parse/pgr.rs index 0646334f..2b7a7118 100644 --- a/prpr/src/parse/pgr.rs +++ b/prpr/src/parse/pgr.rs @@ -1,4 +1,4 @@ -crate::tl_file!("parser" ptl); +prpr_l10n::tl_file!("parser" ptl); use super::process_lines; use crate::{ diff --git a/prpr/src/parse/rpe.rs b/prpr/src/parse/rpe.rs index 4401186d..41e8b047 100644 --- a/prpr/src/parse/rpe.rs +++ b/prpr/src/parse/rpe.rs @@ -1,4 +1,4 @@ -crate::tl_file!("parser" ptl); +prpr_l10n::tl_file!("parser" ptl); use super::{process_lines, RPE_TWEEN_MAP}; use crate::{ diff --git a/prpr/src/scene.rs b/prpr/src/scene.rs index 5a4c8fd5..dcb61537 100644 --- a/prpr/src/scene.rs +++ b/prpr/src/scene.rs @@ -1,6 +1,6 @@ //! Scene management module. -crate::tl_file!("scene" ttl); +prpr_l10n::tl_file!("scene" ttl); mod ending; pub use ending::{EndingScene, RecordUpdateState}; diff --git a/prpr/src/scene/ending.rs b/prpr/src/scene/ending.rs index fb4ce5ef..4ed03856 100644 --- a/prpr/src/scene/ending.rs +++ b/prpr/src/scene/ending.rs @@ -1,4 +1,4 @@ -crate::tl_file!("ending"); +prpr_l10n::tl_file!("ending"); use super::{draw_background, game::SimpleRecord, loading::UploadFn, NextScene, Scene}; use crate::{ diff --git a/prpr/src/scene/game.rs b/prpr/src/scene/game.rs index e661f161..6044cb08 100644 --- a/prpr/src/scene/game.rs +++ b/prpr/src/scene/game.rs @@ -1,6 +1,6 @@ #![allow(unused)] -crate::tl_file!("game"); +prpr_l10n::tl_file!("game"); use super::{ draw_background, diff --git a/prpr/src/ui/chart_info.rs b/prpr/src/ui/chart_info.rs index 701d117a..1d7aee47 100644 --- a/prpr/src/ui/chart_info.rs +++ b/prpr/src/ui/chart_info.rs @@ -1,4 +1,4 @@ -crate::tl_file!("chart_info"); +prpr_l10n::tl_file!("chart_info"); use super::Ui; use crate::{core::BOLD_FONT, ext::parse_time, info::ChartInfo, scene::show_message}; diff --git a/prpr/src/ui/dialog.rs b/prpr/src/ui/dialog.rs index 8d062226..3a1208fa 100644 --- a/prpr/src/ui/dialog.rs +++ b/prpr/src/ui/dialog.rs @@ -1,4 +1,4 @@ -crate::tl_file!("dialog"); +prpr_l10n::tl_file!("dialog"); use super::{DRectButton, RectButton, Scroll, Ui}; use crate::{core::BOLD_FONT, ext::RectExt, scene::show_message}; diff --git a/prpr/tests/integrated_test.rs b/prpr/tests/integrated_test.rs new file mode 100644 index 00000000..bd45e911 --- /dev/null +++ b/prpr/tests/integrated_test.rs @@ -0,0 +1,9 @@ +use prpr_l10n::tools::check_langfile; + +#[test] +fn check_all() { + match check_langfile(concat!(env!("CARGO_MANIFEST_DIR"), "/locales/")) { + Ok(_) => {} + Err(e) => panic!("Error: {}", e), + } +}