Bug 1346819 - Port SanitizeAsBCP47 to LocaleService. r=baku,jfkthame
authorZibi Braniecki <gandalf@mozilla.com>
Mon, 13 Mar 2017 08:31:43 -0700
changeset 397948 b45d664c0a7b10d6a54ceae884f2f8956f10bbec
parent 397947 1a9984c2800dc3e1d75670ec46f3c46c3b18b62e
child 397949 dd6b11222fbc383a27bfbe2a30bd26d7847b7132
push id1490
push usermtabara@mozilla.com
push dateMon, 31 Jul 2017 14:08:16 +0000
treeherdermozilla-release@70e32e6bf15e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbaku, jfkthame
bugs1346819
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1346819 - Port SanitizeAsBCP47 to LocaleService. r=baku,jfkthame MozReview-Commit-ID: 2SXD5HaJPXr
addon-sdk/source/test/test-l10n-locale.js
dom/base/nsGlobalWindow.cpp
dom/base/nsGlobalWindow.h
dom/tests/mochitest/chrome/test_window_getAppLocales.html
dom/webidl/Window.webidl
intl/locale/DateTimeFormat.cpp
intl/locale/LocaleService.cpp
intl/locale/LocaleService.h
intl/locale/mozILocaleService.idl
intl/locale/nsCollation.cpp
intl/locale/tests/gtest/TestLocaleService.cpp
intl/locale/tests/unit/test_localeService.js
toolkit/components/mozintl/mozIntl.js
toolkit/components/telemetry/TelemetryEnvironment.jsm
--- a/addon-sdk/source/test/test-l10n-locale.js
+++ b/addon-sdk/source/test/test-l10n-locale.js
@@ -105,17 +105,17 @@ exports.testPreferedContentLocale = func
   prefs.reset(PREF_ACCEPT_LANGUAGES);
 }
 
 exports.testPreferedOsLocale = function(assert) {
   prefs.set(PREF_MATCH_OS_LOCALE, true);
   prefs.set(PREF_SELECTED_LOCALE, "");
   prefs.set(PREF_ACCEPT_LANGUAGES, "");
 
-  let expectedLocale = Services.locale.getAppLocale().toLowerCase();
+  let expectedLocale = Services.locale.getAppLocaleAsLangTag().toLowerCase();
   let expectedLocaleList = [expectedLocale];
 
   // Add default "en-us" fallback if the main language is not already en-us
   if (expectedLocale != "en-us")
     expectedLocaleList.push("en-us");
 
   assertPrefered(assert, expectedLocaleList, "Ensure that we select OS locale when related preference is set");
 
--- a/dom/base/nsGlobalWindow.cpp
+++ b/dom/base/nsGlobalWindow.cpp
@@ -14924,20 +14924,20 @@ nsGlobalWindow::GetPaintWorklet(ErrorRes
 
     mPaintWorklet = new Worklet(AsInner(), principal, Worklet::ePaintWorklet);
   }
 
   return mPaintWorklet;
 }
 
 void
-nsGlobalWindow::GetAppLocales(nsTArray<nsString>& aLocales)
+nsGlobalWindow::GetAppLocalesAsBCP47(nsTArray<nsString>& aLocales)
 {
   nsTArray<nsCString> appLocales;
-  mozilla::intl::LocaleService::GetInstance()->GetAppLocales(appLocales);
+  mozilla::intl::LocaleService::GetInstance()->GetAppLocalesAsBCP47(appLocales);
 
   for (uint32_t i = 0; i < appLocales.Length(); i++) {
     aLocales.AppendElement(NS_ConvertUTF8toUTF16(appLocales[i]));
   }
 }
 
 #ifdef ENABLE_INTL_API
 IntlUtils*
--- a/dom/base/nsGlobalWindow.h
+++ b/dom/base/nsGlobalWindow.h
@@ -946,17 +946,17 @@ public:
 
   mozilla::dom::Worklet*
   GetAudioWorklet(mozilla::ErrorResult& aRv);
 
   mozilla::dom::Worklet*
   GetPaintWorklet(mozilla::ErrorResult& aRv);
 
   void
-  GetAppLocales(nsTArray<nsString>& aLocales);
+  GetAppLocalesAsBCP47(nsTArray<nsString>& aLocales);
 
 #ifdef ENABLE_INTL_API
   mozilla::dom::IntlUtils*
   GetIntlUtils(mozilla::ErrorResult& aRv);
 #endif
 
 protected:
   bool AlertOrConfirm(bool aAlert, const nsAString& aMessage,
--- a/dom/tests/mochitest/chrome/test_window_getAppLocales.html
+++ b/dom/tests/mochitest/chrome/test_window_getAppLocales.html
@@ -9,15 +9,15 @@ https://bugzilla.mozilla.org/show_bug.cg
   <link rel="stylesheet" type="text/css" href="chrome://mochikit/content/tests/SimpleTest/test.css">
 </head>
 <body>
 <a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1337234">Mozilla Bug 1337234</a>
 <p id="display"></p>
 <div id="content" style="display: none">
 <script>
 
-let appLocales = window.getAppLocales();
+let appLocales = window.getAppLocalesAsBCP47();
 ok(appLocales.length > 0, "getAppLocales returns at least one locale.");
 is(appLocales[0], "en-US", "The first app locale should be en-US.");
 
 </script>
 </body>
 </html>
--- a/dom/webidl/Window.webidl
+++ b/dom/webidl/Window.webidl
@@ -527,17 +527,17 @@ partial interface Window {
    * The result is a sorted list of valid locale IDs and it should be
    * used for all APIs that accept list of locales, like ECMA402 and L10n APIs.
    *
    * This API always returns at least one locale.
    *
    * Example: ["en-US", "de", "pl", "sr-Cyrl", "zh-Hans-HK"]
    */
   [Func="IsChromeOrXBL"]
-  sequence<DOMString> getAppLocales();
+  sequence<DOMString> getAppLocalesAsBCP47();
 
 #ifdef ENABLE_INTL_API
   /**
    * Getter funcion for IntlUtils, which provides helper functions for
    * localization.
    */
   [Throws, Func="IsChromeOrXBL"]
   readonly attribute IntlUtils intlUtils;
--- a/intl/locale/DateTimeFormat.cpp
+++ b/intl/locale/DateTimeFormat.cpp
@@ -18,17 +18,17 @@ nsCString* DateTimeFormat::mLocale = nul
 DateTimeFormat::Initialize()
 {
   if (mLocale) {
     return NS_OK;
   }
 
   mLocale = new nsCString();
   nsAutoCString locale;
-  intl::LocaleService::GetInstance()->GetAppLocale(locale);
+  intl::LocaleService::GetInstance()->GetAppLocaleAsBCP47(locale);
   mLocale->Assign(locale);
 
   return NS_OK;
 }
 
 // performs a locale sensitive date formatting operation on the time_t parameter
 /*static*/ nsresult
 DateTimeFormat::FormatTime(const nsDateFormatSelector aDateFormatSelector,
--- a/intl/locale/LocaleService.cpp
+++ b/intl/locale/LocaleService.cpp
@@ -28,16 +28,53 @@ static const char* kObservedPrefs[] = {
 
 using namespace mozilla::intl;
 
 NS_IMPL_ISUPPORTS(LocaleService, mozILocaleService, nsIObserver)
 
 mozilla::StaticRefPtr<LocaleService> LocaleService::sInstance;
 
 /**
+ * This function transforms a canonical Mozilla Language Tag, into it's
+ * BCP47 compilant form.
+ *
+ * Example: "ja-JP-mac" -> "ja-JP-x-lvariant-mac"
+ *
+ * The BCP47 form should be used for all calls to ICU/Intl APIs.
+ * The canonical form is used for all internal operations.
+ */
+static void SanitizeForBCP47(nsACString& aLocale)
+{
+#ifdef ENABLE_INTL_API
+  // Currently, the only locale code we use that's not BCP47-conformant is
+  // "ja-JP-mac" on OS X, but let's try to be more general than just
+  // hard-coding that here.
+  const int32_t LANG_TAG_CAPACITY = 128;
+  char langTag[LANG_TAG_CAPACITY];
+  nsAutoCString locale(aLocale);
+  UErrorCode err = U_ZERO_ERROR;
+  // This is a fail-safe method that will set langTag to "und" if it cannot
+  // match any part of the input locale code.
+  int32_t len = uloc_toLanguageTag(locale.get(), langTag, LANG_TAG_CAPACITY,
+                                   false, &err);
+  if (U_SUCCESS(err) && len > 0) {
+    aLocale.Assign(langTag, len);
+  }
+#else
+  // This is only really needed for Intl API purposes, AFAIK,
+  // so probably won't be used in a non-ENABLE_INTL_API build.
+  // But let's fix up the single anomalous code we actually ship,
+  // just in case:
+  if (aLocale.EqualsLiteral("ja-JP-mac")) {
+    aLocale.AssignLiteral("ja-JP");
+  }
+#endif
+}
+
+/**
  * This function performs the actual language negotiation for the API.
  *
  * Currently it collects the locale ID used by nsChromeRegistry and
  * adds hardcoded "en-US" locale as a fallback.
  */
 static void
 ReadAppLocales(nsTArray<nsCString>& aRetVal)
 {
@@ -75,24 +112,37 @@ LocaleService::GetInstance()
 }
 
 LocaleService::~LocaleService()
 {
   Preferences::RemoveObservers(sInstance, kObservedPrefs);
 }
 
 void
-LocaleService::GetAppLocales(nsTArray<nsCString>& aRetVal)
+LocaleService::GetAppLocalesAsLangTags(nsTArray<nsCString>& aRetVal)
 {
   if (mAppLocales.IsEmpty()) {
     ReadAppLocales(mAppLocales);
   }
   aRetVal = mAppLocales;
 }
 
+void
+LocaleService::GetAppLocalesAsBCP47(nsTArray<nsCString>& aRetVal)
+{
+  if (mAppLocales.IsEmpty()) {
+    ReadAppLocales(mAppLocales);
+  }
+  for (uint32_t i = 0; i < mAppLocales.Length(); i++) {
+    nsAutoCString locale(mAppLocales[i]);
+    SanitizeForBCP47(locale);
+    aRetVal.AppendElement(locale);
+  }
+}
+
 bool
 LocaleService::GetRequestedLocales(nsTArray<nsCString>& aRetVal)
 {
   nsAutoCString locale;
 
   nsCOMPtr<nsIPrefBranch> prefs(do_GetService("@mozilla.org/preferences-service;1"));
 
   // First, we'll try to check if the user has `matchOS` pref selected
@@ -330,38 +380,61 @@ CreateOutArray(const nsTArray<nsCString>
   char** result = static_cast<char**>(moz_xmalloc(n * sizeof(char*)));
   for (uint32_t i = 0; i < n; i++) {
     result[i] = moz_xstrdup(aArray[i].get());
   }
   return result;
 }
 
 NS_IMETHODIMP
-LocaleService::GetAppLocales(uint32_t* aCount, char*** aOutArray)
+LocaleService::GetAppLocalesAsLangTags(uint32_t* aCount, char*** aOutArray)
 {
   if (mAppLocales.IsEmpty()) {
     ReadAppLocales(mAppLocales);
   }
 
   *aCount = mAppLocales.Length();
   *aOutArray = CreateOutArray(mAppLocales);
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
-LocaleService::GetAppLocale(nsACString& aRetVal)
+LocaleService::GetAppLocalesAsBCP47(uint32_t* aCount, char*** aOutArray)
+{
+  AutoTArray<nsCString, 32> locales;
+  GetAppLocalesAsBCP47(locales);
+
+  *aCount = locales.Length();
+  *aOutArray = CreateOutArray(locales);
+
+  return NS_OK;
+}
+
+NS_IMETHODIMP
+LocaleService::GetAppLocaleAsLangTag(nsACString& aRetVal)
 {
   if (mAppLocales.IsEmpty()) {
     ReadAppLocales(mAppLocales);
   }
   aRetVal = mAppLocales[0];
   return NS_OK;
 }
 
+NS_IMETHODIMP
+LocaleService::GetAppLocaleAsBCP47(nsACString& aRetVal)
+{
+  if (mAppLocales.IsEmpty()) {
+    ReadAppLocales(mAppLocales);
+  }
+  aRetVal = mAppLocales[0];
+  SanitizeForBCP47(aRetVal);
+  return NS_OK;
+}
+
 static LocaleService::LangNegStrategy
 ToLangNegStrategy(int32_t aStrategy)
 {
   switch (aStrategy) {
     case 1:
       return LocaleService::LangNegStrategy::Matching;
     case 2:
       return LocaleService::LangNegStrategy::Lookup;
--- a/intl/locale/LocaleService.h
+++ b/intl/locale/LocaleService.h
@@ -58,17 +58,17 @@ public:
     Lookup
   };
 
   /**
    * Create (if necessary) and return a raw pointer to the singleton instance.
    * Use this accessor in C++ code that just wants to call a method on the
    * instance, but does not need to hold a reference, as in
    *    nsAutoCString str;
-   *    LocaleService::GetInstance()->GetAppLocale(str);
+   *    LocaleService::GetInstance()->GetAppLocaleAsLangTag(str);
    */
   static LocaleService* GetInstance();
 
   /**
    * Return an addRef'd pointer to the singleton instance. This is used by the
    * XPCOM constructor that exists to support usage from JS.
    */
   static already_AddRefed<LocaleService> GetInstanceAddRefed()
@@ -83,21 +83,22 @@ public:
    * used for all APIs that accept list of locales, like ECMA402 and L10n APIs.
    *
    * This API always returns at least one locale.
    *
    * Example: ["en-US", "de", "pl", "sr-Cyrl", "zh-Hans-HK"]
    *
    * Usage:
    *   nsTArray<nsCString> appLocales;
-   *   LocaleService::GetInstance()->GetAppLocales(appLocales);
+   *   LocaleService::GetInstance()->GetAppLocalesAsLangTags(appLocales);
    *
    * (See mozILocaleService.idl for a JS-callable version of this.)
    */
-  void GetAppLocales(nsTArray<nsCString>& aRetVal);
+  void GetAppLocalesAsLangTags(nsTArray<nsCString>& aRetVal);
+  void GetAppLocalesAsBCP47(nsTArray<nsCString>& aRetVal);
 
   /**
    * Returns a list of locales that the user requested the app to be
    * localized to.
    *
    * The result is a sorted list of valid locale IDs and it should be
    * used as a requestedLocales input list for languages negotiation.
    *
--- a/intl/locale/mozILocaleService.idl
+++ b/intl/locale/mozILocaleService.idl
@@ -54,22 +54,29 @@ interface mozILocaleService : nsISupport
   /**
    * Returns a list of locales that the application should be localized to.
    *
    * The result is a ordered list of valid locale IDs and it should be
    * used for all APIs that accept list of locales, like ECMA402 and L10n APIs.
    *
    * This API always returns at least one locale.
    *
+   * When retrieving the locales for language negotiation and matching
+   * to language resources, use the language tag form.
+   * When retrieving the locales for Intl API or ICU locale settings,
+   * use the BCP47 form.
+   *
    * Example: ["en-US", "de", "pl", "sr-Cyrl", "zh-Hans-HK"]
    *
    * (See LocaleService.h for a more C++-friendly version of this.)
    */
-  void getAppLocales([optional] out unsigned long aCount,
-                     [retval, array, size_is(aCount)] out string aLocales);
+  void getAppLocalesAsLangTags([optional] out unsigned long aCount,
+                               [retval, array, size_is(aCount)] out string aLocales);
+  void getAppLocalesAsBCP47([optional] out unsigned long aCount,
+                            [retval, array, size_is(aCount)] out string aLocales);
 
   /**
    * Negotiates the best locales out of a ordered list of requested locales and
    * a list of available locales.
    *
    * Internally it uses the following naming scheme:
    *
    *  Requested - locales requested by the user
@@ -95,24 +102,30 @@ interface mozILocaleService : nsISupport
                           [retval, array, size_is(aCount)] out string aLocales);
 
   /**
    * Returns the best locale that the application should be localized to.
    *
    * The result is a valid locale ID and it should be
    * used for all APIs that do not handle language negotiation.
    *
-   * Where possible, getAppLocales() should be preferred over this API and
+   * When retrieving the locales for language negotiation and matching
+   * to language resources, use the language tag form.
+   * When retrieving the locales for Intl API or ICU locale settings,
+   * use the BCP47 form.
+   *
+   * Where possible, getAppLocales*() should be preferred over this API and
    * all callsites should handle some form of "best effort" language
    * negotiation to respect user preferences in case the use case does
    * not have data for the first locale in the list.
    *
    * Example: "zh-Hans-HK"
    */
-  ACString getAppLocale();
+  ACString getAppLocaleAsLangTag();
+  ACString getAppLocaleAsBCP47();
 
   /**
    * Returns a list of locales that the user requested the app to be
    * localized to.
    *
    * The result is an ordered list of locale IDs which should be
    * used as a requestedLocales input list for language negotiation.
    *
--- a/intl/locale/nsCollation.cpp
+++ b/intl/locale/nsCollation.cpp
@@ -18,17 +18,17 @@ using mozilla::dom::EncodingUtils;
 
 NS_DEFINE_CID(kCollationCID, NS_COLLATION_CID);
 
 NS_IMPL_ISUPPORTS(nsCollationFactory, nsICollationFactory)
 
 nsresult nsCollationFactory::CreateCollation(nsICollation** instancePtr)
 {
   nsAutoCString appLocale;
-  mozilla::intl::LocaleService::GetInstance()->GetAppLocale(appLocale);
+  mozilla::intl::LocaleService::GetInstance()->GetAppLocaleAsLangTag(appLocale);
 
   return CreateCollationForLocale(appLocale, instancePtr);
 }
 
 nsresult
 nsCollationFactory::CreateCollationForLocale(const nsACString& locale, nsICollation** instancePtr)
 {
   // Create a collation interface instance.
--- a/intl/locale/tests/gtest/TestLocaleService.cpp
+++ b/intl/locale/tests/gtest/TestLocaleService.cpp
@@ -6,54 +6,54 @@
 #include "gtest/gtest.h"
 #include "mozilla/intl/LocaleService.h"
 #include "mozilla/Services.h"
 #include "nsIToolkitChromeRegistry.h"
 
 using namespace mozilla::intl;
 
 
-TEST(Intl_Locale_LocaleService, GetAppLocales) {
+TEST(Intl_Locale_LocaleService, GetAppLocalesAsLangTags) {
   nsTArray<nsCString> appLocales;
-  LocaleService::GetInstance()->GetAppLocales(appLocales);
+  LocaleService::GetInstance()->GetAppLocalesAsLangTags(appLocales);
 
   ASSERT_FALSE(appLocales.IsEmpty());
 }
 
-TEST(Intl_Locale_LocaleService, GetAppLocales_firstMatchesChromeReg) {
+TEST(Intl_Locale_LocaleService, GetAppLocalesAsLangTags_firstMatchesChromeReg) {
   nsTArray<nsCString> appLocales;
-  LocaleService::GetInstance()->GetAppLocales(appLocales);
+  LocaleService::GetInstance()->GetAppLocalesAsLangTags(appLocales);
 
   nsAutoCString uaLangTag;
   nsCOMPtr<nsIToolkitChromeRegistry> cr =
     mozilla::services::GetToolkitChromeRegistryService();
   if (cr) {
     cr->GetSelectedLocale(NS_LITERAL_CSTRING("global"), true, uaLangTag);
   }
 
   ASSERT_TRUE(appLocales[0].Equals(uaLangTag));
 }
 
-TEST(Intl_Locale_LocaleService, GetAppLocales_lastIsEnUS) {
+TEST(Intl_Locale_LocaleService, GetAppLocalesAsLangTags_lastIsEnUS) {
   nsTArray<nsCString> appLocales;
-  LocaleService::GetInstance()->GetAppLocales(appLocales);
+  LocaleService::GetInstance()->GetAppLocalesAsLangTags(appLocales);
 
   int32_t len = appLocales.Length();
   ASSERT_TRUE(appLocales[len - 1].EqualsLiteral("en-US"));
 }
 
 TEST(Intl_Locale_LocaleService, GetRequestedLocales) {
   nsTArray<nsCString> reqLocales;
   LocaleService::GetInstance()->GetRequestedLocales(reqLocales);
 
   int32_t len = reqLocales.Length();
   ASSERT_TRUE(len > 0);
 }
 
-TEST(Intl_Locale_LocaleService, GetAppLocale) {
+TEST(Intl_Locale_LocaleService, GetAppLocaleAsLangTag) {
   nsTArray<nsCString> appLocales;
-  LocaleService::GetInstance()->GetAppLocales(appLocales);
+  LocaleService::GetInstance()->GetAppLocalesAsLangTags(appLocales);
 
   nsAutoCString locale;
-  LocaleService::GetInstance()->GetAppLocale(locale);
+  LocaleService::GetInstance()->GetAppLocaleAsLangTag(locale);
 
   ASSERT_TRUE(appLocales[0] == locale);
 }
--- a/intl/locale/tests/unit/test_localeService.js
+++ b/intl/locale/tests/unit/test_localeService.js
@@ -17,25 +17,25 @@ function run_test()
 {
   const localeService =
     Components.classes["@mozilla.org/intl/localeservice;1"]
     .getService(Components.interfaces.mozILocaleService);
 
   run_next_test();
 }
 
-add_test(function test_getAppLocales() {
+add_test(function test_getAppLocalesAsLangTags() {
   const localeService =
     Components.classes["@mozilla.org/intl/localeservice;1"]
     .getService(Components.interfaces.mozILocaleService);
 
-  const appLocale = localeService.getAppLocale();
+  const appLocale = localeService.getAppLocaleAsLangTag();
   do_check_true(appLocale != "", "appLocale is non-empty");
 
-  const appLocales = localeService.getAppLocales();
+  const appLocales = localeService.getAppLocalesAsLangTags();
   do_check_true(Array.isArray(appLocales), "appLocales returns an array");
 
   do_check_true(appLocale == appLocales[0], "appLocale matches first entry in appLocales");
 
   const enUSLocales = appLocales.filter(loc => loc === "en-US");
   do_check_true(enUSLocales.length == 1, "en-US is present exactly one time");
 
   run_next_test();
--- a/toolkit/components/mozintl/mozIntl.js
+++ b/toolkit/components/mozintl/mozIntl.js
@@ -14,17 +14,17 @@ const localeSvc =
 
 /**
  * This helper function retrives currently used app locales, allowing
  * all mozIntl APIs to use the current app locales unless called with
  * explicitly listed locales.
  */
 function getLocales(locales) {
   if (!locales) {
-    return localeSvc.getAppLocales();
+    return localeSvc.getAppLocalesAsBCP47();
   }
   return locales;
 }
 
 class MozIntl {
   constructor() {
     this._cache = {};
   }
--- a/toolkit/components/telemetry/TelemetryEnvironment.jsm
+++ b/toolkit/components/telemetry/TelemetryEnvironment.jsm
@@ -221,22 +221,22 @@ const SEARCH_SERVICE_TOPIC = "browser-se
 function enforceBoolean(aValue) {
   if (typeof(aValue) !== "number" && typeof(aValue) !== "boolean") {
     return null;
   }
   return (new Boolean(aValue)).valueOf();
 }
 
 /**
- * Get the current browser.
+ * Get the current browser locale.
  * @return a string with the locale or null on failure.
  */
 function getBrowserLocale() {
   try {
-    return Services.locale.getAppLocale();
+    return Services.locale.getAppLocaleAsLangTag();
   } catch (e) {
     return null;
   }
 }
 
 /**
  * Get the current OS locale.
  * @return a string with the OS locale or null on failure.