Make sure the locale fallback works as expected

- Also makes sure that the fallback runs in the backend and is then
   picked up by the frontend
 - and explicitely fallback zh-CN to zh-Hans

crates/handlers/src/preferred_language.rs

@@ -21,7 +21,7 @@ use axum::{
     TypedHeader,
 };
 use mas_axum_utils::language_detection::AcceptLanguage;
-use mas_i18n::{DataLocale, Translator};
+use mas_i18n::{locale, DataLocale, Translator};
 
 pub struct PreferredLanguage(pub DataLocale);
 
@@ -37,16 +37,24 @@ where
         let translator: Arc<Translator> = FromRef::from_ref(state);
         let accept_language: Option<TypedHeader<AcceptLanguage>> =
             FromRequestParts::from_request_parts(parts, state).await?;
-        let supported_language = translator.available_locales();
 
-        let locale = accept_language
-            .and_then(|TypedHeader(accept_language)| {
-                accept_language.iter().find_map(|lang| {
-                    let locale: DataLocale = lang.into();
-                    supported_language.contains(&&locale).then_some(locale)
-                })
-            })
-            .unwrap_or("en".parse().unwrap());
+        let iter = accept_language
+            .iter()
+            .flat_map(|TypedHeader(accept_language)| accept_language.iter())
+            .flat_map(|lang| {
+                let lang = DataLocale::from(lang);
+                // XXX: this is hacky as we may want to actually maintain proper language
+                // aliases at some point, but `zh-CN` doesn't fallback
+                // automatically to `zh-Hans`, so we insert it manually here.
+                // For some reason, `zh-TW` does fallback to `zh-Hant` correctly.
+                if lang == locale!("zh-CN").into() {
+                    vec![lang, locale!("zh-Hans").into()]
+                } else {
+                    vec![lang]
+                }
+            });
+
+        let locale = translator.choose_locale(iter);
 
         Ok(PreferredLanguage(locale))
     }

crates/i18n/src/translator.rs

@@ -17,7 +17,9 @@ use std::{collections::HashMap, fs::File, str::FromStr};
 use camino::{Utf8Path, Utf8PathBuf};
 use icu_list::{ListError, ListFormatter, ListLength};
 use icu_locid::{Locale, ParserError};
-use icu_locid_transform::fallback::LocaleFallbacker;
+use icu_locid_transform::fallback::{
+    LocaleFallbackPriority, LocaleFallbackSupplement, LocaleFallbacker, LocaleFallbackerWithConfig,
+};
 use icu_plurals::{PluralRules, PluralsError};
 use icu_provider::{
     data_key, fallback::LocaleFallbackConfig, DataError, DataErrorKind, DataKey, DataLocale,
@@ -33,6 +35,13 @@ use crate::{sprintf::Message, translations::TranslationTree};
 /// Fake data key for errors
 const DATA_KEY: DataKey = data_key!("mas/translations@1");
 
+const FALLBACKER: LocaleFallbackerWithConfig<'static> = LocaleFallbacker::new().for_config({
+    let mut config = LocaleFallbackConfig::const_default();
+    config.priority = LocaleFallbackPriority::Collation;
+    config.fallback_supplement = Some(LocaleFallbackSupplement::Collation);
+    config
+});
+
 /// Error type for loading translations
 #[derive(Debug, Error)]
 #[error("Failed to load translations")]
@@ -49,7 +58,6 @@ pub struct Translator {
     translations: HashMap<DataLocale, TranslationTree>,
     plural_provider: LocaleFallbackProvider<icu_plurals::provider::Baked>,
     list_provider: LocaleFallbackProvider<icu_list::provider::Baked>,
-    fallbacker: LocaleFallbacker,
     default_locale: DataLocale,
 }
 
@@ -62,16 +70,13 @@ impl Translator {
             icu_plurals::provider::Baked,
             fallbacker.clone(),
         );
-        let list_provider = LocaleFallbackProvider::new_with_fallbacker(
-            icu_list::provider::Baked,
-            fallbacker.clone(),
-        );
+        let list_provider =
+            LocaleFallbackProvider::new_with_fallbacker(icu_list::provider::Baked, fallbacker);
 
         Self {
             translations,
             plural_provider,
             list_provider,
-            fallbacker,
             // TODO: make this configurable
             default_locale: icu_locid::locale!("en").into(),
         }
@@ -126,10 +131,11 @@ impl Translator {
         locale: DataLocale,
         key: &str,
     ) -> Option<(&Message, DataLocale)> {
-        let mut iter = self
-            .fallbacker
-            .for_config(LocaleFallbackConfig::default())
-            .fallback_for(locale);
+        if let Ok(message) = self.message(&locale, key) {
+            return Some((message, locale));
+        }
+
+        let mut iter = FALLBACKER.fallback_for(locale);
 
         loop {
             let locale = iter.get();
@@ -194,10 +200,7 @@ impl Translator {
         key: &str,
         count: usize,
     ) -> Option<(&Message, DataLocale)> {
-        let mut iter = self
-            .fallbacker
-            .for_config(LocaleFallbackConfig::default())
-            .fallback_for(locale);
+        let mut iter = FALLBACKER.fallback_for(locale);
 
         loop {
             let locale = iter.get();
@@ -369,29 +372,29 @@ impl Translator {
 
     /// Choose the best available locale from a list of candidates.
     #[must_use]
-    pub fn choose_locale<'a>(
-        &self,
-        iter: impl Iterator<Item = &'a DataLocale>,
-    ) -> Option<DataLocale> {
+    pub fn choose_locale(&self, iter: impl Iterator<Item = DataLocale>) -> DataLocale {
         for locale in iter {
-            let mut fallbacker = self
-                .fallbacker
-                .for_config(LocaleFallbackConfig::default())
-                .fallback_for(locale.clone());
+            println!("Trying for locale {locale:?}");
+            if self.has_locale(&locale) {
+                return locale;
+            }
+
+            let mut fallbacker = FALLBACKER.fallback_for(locale);
 
             loop {
+                println!("  Fallback: {:?}", fallbacker.get());
                 if fallbacker.get().is_und() {
                     break;
                 }
 
                 if self.has_locale(fallbacker.get()) {
-                    return Some(fallbacker.take());
+                    return fallbacker.take();
                 }
                 fallbacker.step();
             }
         }
 
-        None
+        self.default_locale.clone()
     }
 }
 

crates/templates/src/lib.rs

@@ -194,6 +194,8 @@ impl Templates {
                 .await??;
         let translator = Arc::new(translator);
 
+        debug!(locales = ?translator.available_locales(), "Loaded translations");
+
         let (loaded, mut env) = tokio::task::spawn_blocking(move || {
             span.in_scope(move || {
                 let mut loaded: HashSet<_> = HashSet::new();

frontend/src/i18n.ts

@@ -47,7 +47,8 @@ i18n
     pluralSeparator: ":",
     supportedLngs,
     detection: {
-      order: ["navigator", "htmlTag"],
+      // This lets the backend fully decide the language to use
+      order: ["htmlTag"],
     } satisfies DetectorOptions,
     interpolation: {
       escapeValue: false, // React has built-in XSS protections