ci: Add integrated test for L10n (#510)

.github/workflows/cargo_test.yml

@@ -34,3 +34,4 @@ jobs:
       - name: Run tests
         run: |
           cargo test -p phira --no-default-features -- --skip test_parse_chart
+          cargo test -p prpr --no-default-features

Cargo.toml

@@ -3,6 +3,7 @@ members = [
 	"prpr",
 	"prpr-avc",
 	"prpr-pbc",
+	"prpr-l10n",
 	"phira",
 	"phira-main",
 	"phira-monitor",

phira/Cargo.toml

@@ -35,6 +35,7 @@ once_cell = "*"
 openssl = { version = "*", features = ["vendored"] }
 pollster = "0.3.0"
 prpr = { path = "../prpr", features = ["log"], default-features = false }
+prpr-l10n ={ path = "../prpr-l10n" }
 rand = "0.8.5"
 regex = "1.7.0"
 reqwest = { version = "0.12.5", features = ["json", "stream", "gzip"] }
@@ -75,3 +76,7 @@ objc-foundation = "*"
 
 [build-dependencies]
 dotenv-build = "0.1"
+
+[dev-dependencies]
+fluent = "0.16.0"
+fluent-syntax = "0.11.0"

phira/src/client.rs

@@ -6,7 +6,8 @@ use crate::{anti_addiction_action, get_data, get_data_mut, save_data};
 use anyhow::{anyhow, bail, Context, Result};
 use arc_swap::ArcSwap;
 use once_cell::sync::Lazy;
-use prpr::{l10n::LANG_IDENTS, scene::SimpleRecord};
+use prpr::scene::SimpleRecord;
+use prpr_l10n::LANG_IDENTS;
 use reqwest::{header, ClientBuilder, Method, RequestBuilder, Response, StatusCode};
 use serde::{Deserialize, Serialize};
 use serde_json::json;

phira/src/lib.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("common" ttl crate::);
+prpr_l10n::tl_file!("common" ttl crate::);
 
 #[cfg(feature = "closed")]
 mod inner;
@@ -28,13 +28,13 @@ use prpr::{
     build_conf,
     core::{init_assets, PGR_FONT},
     ext::SafeTexture,
-    l10n::{set_prefered_locale, GLOBAL, LANGS},
     log,
     scene::{show_error, show_message},
     time::TimeManager,
     ui::{FontArc, TextPainter},
     Main,
 };
+use prpr_l10n::{set_prefered_locale, GLOBAL, LANGS};
 use scene::MainScene;
 use std::sync::{mpsc, Mutex};
 use tracing::{error, info};

phira/src/login.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("login");
+prpr_l10n::tl_file!("login");
 
 use crate::{
     client::{Client, LoginParams, User, UserManager},

phira/src/mp.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("multiplayer" mtl);
+prpr_l10n::tl_file!("multiplayer" mtl);
 
 mod panel;
 pub use panel::MPPanel;

phira/src/page/coll.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("collection");
+prpr_l10n::tl_file!("collection");
 
 use super::{Illustration, NextPage, Page, SharedState};
 use crate::{icons::Icons, load_res_tex, resource::rtl, scene::ChapterScene};

phira/src/page/home.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("home");
+prpr_l10n::tl_file!("home");
 
 use super::{
     load_font_with_cksum, set_bold_font, EventPage, LibraryPage, MessagePage, NextPage, Page, ResPackPage, SFader, SettingsPage, SharedState,
@@ -15,6 +15,7 @@ use crate::{
     sync_data,
     threed::ThreeD,
 };
+use prpr_l10n::LANG_IDENTS;
 use ::rand::{random, thread_rng, Rng};
 use anyhow::{bail, Context, Result};
 use chrono::NaiveDate;
@@ -24,7 +25,6 @@ use prpr::{
     core::BOLD_FONT,
     ext::{open_url, screen_aspect, semi_black, semi_white, RectExt, SafeTexture, ScaleType},
     info::ChartInfo,
-    l10n::LANG_IDENTS,
     scene::{show_error, NextScene},
     task::Task,
     ui::{button_hit_large, clip_rounded_rect, ClipType, DRectButton, Dialog, FontArc, RectButton, Scroll, Ui},

phira/src/page/library.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("library");
+prpr_l10n::tl_file!("library");
 
 use super::{CollectionPage, NextPage, Page, SharedState};
 use crate::{

phira/src/page/message.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("message");
+prpr_l10n::tl_file!("message");
 
 use std::borrow::Cow;
 

phira/src/page/offset.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("cali");
+prpr_l10n::tl_file!("cali");
 
 use std::borrow::Cow;
 

phira/src/page/respack.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("respack");
+prpr_l10n::tl_file!("respack");
 
 use super::{Page, SharedState};
 use crate::{

phira/src/page/settings.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("settings");
+prpr_l10n::tl_file!("settings");
 
 use super::{NextPage, OffsetPage, Page, SharedState};
 use crate::{
@@ -15,11 +15,11 @@ use macroquad::prelude::*;
 use prpr::{
     core::BOLD_FONT,
     ext::{open_url, poll_future, semi_white, LocalTask, RectExt, SafeTexture},
-    l10n::{LanguageIdentifier, LANG_IDENTS, LANG_NAMES},
     scene::{request_input, return_input, show_error, show_message, take_input},
     task::Task,
     ui::{DRectButton, Scroll, Slider, Ui},
 };
+use prpr_l10n::{LanguageIdentifier, LANG_IDENTS, LANG_NAMES};
 use reqwest::Url;
 use std::{borrow::Cow, fs, io, net::ToSocketAddrs, path::PathBuf, sync::atomic::Ordering};
 

phira/src/rate.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("rate");
+prpr_l10n::tl_file!("rate");
 
 use crate::page::Fader;
 use macroquad::prelude::*;

phira/src/resource.rs

@@ -1 +1 @@
-prpr::tl_file!("resource" rtl);
+prpr_l10n::tl_file!("resource" rtl);

phira/src/scene.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("import" itl);
+prpr_l10n::tl_file!("import" itl);
 
 mod chart_order;
 pub use chart_order::{ChartOrder, ORDERS};

phira/src/scene/chapter.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("chapter");
+prpr_l10n::tl_file!("chapter");
 
 use crate::{
     anim::Anim,

phira/src/scene/chart_order.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("chart_order");
+prpr_l10n::tl_file!("chart_order");
 
 use crate::page::ChartItem;
 

phira/src/scene/event.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("event");
+prpr_l10n::tl_file!("event");
 
 use super::{render_ldb, LdbDisplayItem, ProfileScene};
 use crate::{

phira/src/scene/profile.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("profile");
+prpr_l10n::tl_file!("profile");
 
 use super::{confirm_delete, TEX_BACKGROUND, TEX_ICON_BACK};
 use crate::{

phira/src/scene/song.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("song");
+prpr_l10n::tl_file!("song");
 
 #[cfg(feature = "video")]
 use super::UnlockScene;

phira/src/tags.rs

@@ -1,4 +1,4 @@
-prpr::tl_file!("tags");
+prpr_l10n::tl_file!("tags");
 
 use crate::{client::Permissions, page::Fader};
 use macroquad::prelude::*;

phira/tests/integrated_test.rs

@@ -0,0 +1,9 @@
+use prpr_l10n::tools::check_langfile;
+
+#[test]
+fn check_all() {
+    match check_langfile(concat!(env!("CARGO_MANIFEST_DIR"), "/locales/")) {
+        Ok(_) => {}
+        Err(e) => panic!("Error: {}", e),
+    }
+}

prpr-l10n/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "prpr-l10n"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+fluent = "0.16.0"
+fluent-syntax = "0.11.0"
+lru = "0.9.0"
+once_cell = "1.16.0"
+sys-locale = "0.3.1"
+tracing = "0.1.37"
+unic-langid = { version = "0.9.1", features = ["macros"] }
+walkdir = "2.3.3"

prpr-l10n/src/global.rs

@@ -0,0 +1,42 @@
+use std::{collections::HashMap, sync::Mutex};
+
+use tracing::warn;
+use unic_langid::LanguageIdentifier;
+
+use crate::{fallback_langid, FALLBACK_LANG, LANG_IDENTS};
+
+pub struct L10nGlobal {
+    pub lang_map: HashMap<LanguageIdentifier, usize>,
+    pub order: Mutex<Vec<usize>>,
+}
+
+impl Default for L10nGlobal {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl L10nGlobal {
+    pub fn new() -> Self {
+        let mut lang_map = HashMap::new();
+        let mut order = Vec::new();
+        let locale_lang = sys_locale::get_locale().unwrap_or_else(|| String::from(FALLBACK_LANG));
+        let locale_lang: LanguageIdentifier = locale_lang.parse().unwrap_or_else(|_| {
+            warn!("Invalid locale detected, defaulting to `{}`", FALLBACK_LANG);
+            // Debug log: send lang tag to log
+            warn!("Locale detected: {:?}", locale_lang);
+            fallback_langid!()
+        });
+        for (id, lang) in LANG_IDENTS.iter().enumerate() {
+            lang_map.insert(lang.clone(), id);
+            if *lang == locale_lang {
+                order.push(id);
+            }
+        }
+        order.push(*lang_map.get(&fallback_langid!()).unwrap());
+        Self {
+            lang_map,
+            order: order.into(),
+        }
+    }
+}

prpr-l10n/src/lib.rs

@@ -0,0 +1,79 @@
+//! Localization utilities.
+
+pub use fluent::{fluent_args, FluentBundle, FluentResource};
+pub use once_cell::sync::Lazy;
+pub use unic_langid::LanguageIdentifier;
+
+use std::sync::atomic::{AtomicU8, Ordering};
+
+mod global;
+pub use global::*;
+
+mod local;
+pub use local::*;
+
+mod macros;
+
+pub mod tools;
+
+langs! {
+    "en-US": "English",
+    "fr-FR": "Français",
+    "id-ID": "Bahasa Indonesia",
+    "ja-JP": "日本語",
+    "ko-KR": "한국어",
+    "pl-PL": "Polski",
+    "pt-BR": "Português",
+    "ru-RU": "Русский",
+    "th-TH": "แบบไทย",
+    "vi-VN": "Tiếng Việt",
+    "zh-CN": "简体中文",
+    "zh-TW": "繁體中文"
+}
+
+#[macro_export]
+macro_rules! fallback_langid {
+    () => {
+        unic_langid::langid!("en-US")
+    };
+}
+
+pub const FALLBACK_LANG: &str = "en-US";
+
+pub static GLOBAL: Lazy<L10nGlobal> = Lazy::new(L10nGlobal::new);
+
+pub fn set_prefered_locale(locale: Option<LanguageIdentifier>) {
+    let mut ids = Vec::new();
+    let map = &GLOBAL.lang_map;
+    if let Some(lang) = locale.and_then(|it| map.get(&it)) {
+        ids.push(*lang);
+    }
+    if let Some(lang) = sys_locale::get_locale()
+        .and_then(|it| it.parse::<LanguageIdentifier>().ok())
+        .and_then(|it| map.get(&it))
+    {
+        ids.push(*lang);
+    }
+    ids.push(*map.get(&fallback_langid!()).unwrap());
+    *GLOBAL.order.lock().unwrap() = ids;
+    GENERATION.fetch_add(1, Ordering::Relaxed);
+}
+
+pub fn locale_order() -> Vec<usize> {
+    GLOBAL.order.lock().unwrap().clone()
+}
+
+pub struct L10nBundles {
+    inner: Vec<FluentBundle<FluentResource>>,
+}
+
+impl From<Vec<FluentBundle<FluentResource>>> for L10nBundles {
+    fn from(inner: Vec<FluentBundle<FluentResource>>) -> Self {
+        Self { inner }
+    }
+}
+
+unsafe impl Send for L10nBundles {}
+unsafe impl Sync for L10nBundles {}
+
+pub static GENERATION: AtomicU8 = AtomicU8::new(0);