Bug 1502921 - Record telemetry environment data about locales r=chutten
authorMark Striemer <mstriemer@mozilla.com>
Wed, 23 Jan 2019 21:26:08 +0000
changeset 515190 de6426e0f4910ebe27666feb8153c3220d81edee
parent 515189 371bf42a4b472344a50f4e8adb3e72811f31151e
child 515191 4ef0f4957e8502de7b24fa2b6b88dd1c553dfc33
push id1953
push userffxbld-merge
push dateMon, 11 Mar 2019 12:10:20 +0000
treeherdermozilla-release@9c35dcbaa899 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschutten
bugs1502921
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1502921 - Record telemetry environment data about locales r=chutten Differential Revision: https://phabricator.services.mozilla.com/D15990
toolkit/components/telemetry/app/TelemetryEnvironment.jsm
toolkit/components/telemetry/docs/data/environment.rst
toolkit/components/telemetry/tests/unit/head.js
toolkit/components/telemetry/tests/unit/test_ChildEvents.js
toolkit/components/telemetry/tests/unit/test_ChildHistograms.js
toolkit/components/telemetry/tests/unit/test_ChildScalars.js
toolkit/components/telemetry/tests/unit/test_SubsessionChaining.js
toolkit/components/telemetry/tests/unit/test_TelemetryChildEvents_buildFaster.js
toolkit/components/telemetry/tests/unit/test_TelemetryController.js
toolkit/components/telemetry/tests/unit/test_TelemetryControllerShutdown.js
toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
toolkit/components/telemetry/tests/unit/test_TelemetryReportingPolicy.js
toolkit/components/telemetry/tests/unit/test_TelemetrySendOldPings.js
toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
toolkit/components/telemetry/tests/unit/test_TelemetryTimestamps.js
--- a/toolkit/components/telemetry/app/TelemetryEnvironment.jsm
+++ b/toolkit/components/telemetry/app/TelemetryEnvironment.jsm
@@ -43,16 +43,31 @@ const MAX_EXPERIMENT_BRANCH_LENGTH = 100
 const MAX_EXPERIMENT_TYPE_LENGTH = 20;
 
 /**
  * This is a policy object used to override behavior for testing.
  */
 // eslint-disable-next-line no-unused-vars
 var Policy = {
   now: () => new Date(),
+  _intlLoaded: false,
+  _browserDelayedStartup() {
+    if (Policy._intlLoaded) {
+      return Promise.resolve();
+    }
+    return new Promise(resolve => {
+      let startupTopic = "browser-delayed-startup-finished";
+      Services.obs.addObserver(function observer(subject, topic) {
+        if (topic == startupTopic) {
+          Services.obs.removeObserver(observer, startupTopic);
+          resolve();
+        }
+      }, startupTopic);
+    });
+  },
 };
 
 // This is used to buffer calls to setExperimentActive and friends, so that we
 // don't prematurely initialize our environment if it is called early during
 // startup.
 var gActiveExperimentStartupBuffer = new Map();
 
 var gGlobalEnvironment;
@@ -316,16 +331,59 @@ function getSystemLocale() {
              getService(Ci.mozIOSPreferences).
              systemLocale;
   } catch (e) {
     return null;
   }
 }
 
 /**
+ * Get the current OS locales.
+ * @return an array of strings with the OS locales or null on failure.
+ */
+function getSystemLocales() {
+  try {
+    return Cc["@mozilla.org/intl/ospreferences;1"].
+             getService(Ci.mozIOSPreferences).
+             systemLocales;
+  } catch (e) {
+    return null;
+  }
+}
+
+/**
+ * Get the current OS regional preference locales.
+ * @return an array of strings with the OS regional preference locales or null on failure.
+ */
+function getRegionalPrefsLocales() {
+  try {
+    return Cc["@mozilla.org/intl/ospreferences;1"].
+             getService(Ci.mozIOSPreferences).
+             regionalPrefsLocales;
+  } catch (e) {
+    return null;
+  }
+}
+
+function getIntlSettings() {
+  return {
+    requestedLocales: Services.locale.requestedLocales,
+    availableLocales: Services.locale.availableLocales,
+    appLocales: Services.locale.appLocalesAsBCP47,
+    systemLocales: getSystemLocales(),
+    regionalPrefsLocales: getRegionalPrefsLocales(),
+    acceptLanguages:
+      Services.prefs.getComplexValue("intl.accept_languages", Ci.nsIPrefLocalizedString)
+        .data
+        .split(",")
+        .map(str => str.trim()),
+  };
+}
+
+/**
  * Safely get a sysinfo property and return its value. If the property is not
  * available, return aDefault.
  *
  * @param aPropertyName the property name to get.
  * @param aDefault the value to return if aPropertyName is not available.
  * @return The property value, if available, or aDefault.
  */
 function getSysinfoProperty(aPropertyName, aDefault) {
@@ -899,16 +957,17 @@ function EnvironmentCache() {
   p = [ this._addonBuilder.init() ];
 
   this._currentEnvironment.profile = {};
   p.push(this._updateProfile());
   if (AppConstants.MOZ_BUILD_APP == "browser") {
     p.push(this._loadAttributionAsync());
   }
   p.push(this._loadAutoUpdateAsync());
+  p.push(this._loadIntlData());
 
   for (const [id, {branch, options}] of gActiveExperimentStartupBuffer.entries()) {
     this.setExperimentActive(id, branch, options);
   }
   gActiveExperimentStartupBuffer = null;
 
   let setup = () => {
     this._initTask = null;
@@ -1411,16 +1470,19 @@ EnvironmentCache.prototype = {
     } catch (e) {}
 
     this._currentEnvironment.settings = {
       blocklistEnabled: Services.prefs.getBoolPref(PREF_BLOCKLIST_ENABLED, true),
       e10sEnabled: Services.appinfo.browserTabsRemoteAutostart,
       e10sMultiProcesses: Services.appinfo.maxWebProcessCount,
       telemetryEnabled: Utils.isTelemetryEnabled,
       locale: getBrowserLocale(),
+      // We need to wait for browser-delayed-startup-finished to ensure that the locales
+      // have settled, once that's happened we can get the intl data directly.
+      intl: Policy._intlLoaded ? getIntlSettings() : {},
       update: {
         channel: updateChannel,
         enabled: !Services.policies || Services.policies.isAllowed("appUpdate"),
       },
       userPrefs: this._getPrefData(),
       sandbox: this._getSandboxData(),
     };
 
@@ -1533,16 +1595,27 @@ EnvironmentCache.prototype = {
   _updateAutoDownload() {
     if (this._updateAutoDownloadCache === undefined) {
       return;
     }
     this._currentEnvironment.settings.update.autoDownload = this._updateAutoDownloadCache;
   },
 
   /**
+  * Get i18n data about the system.
+  * @return A promise of completion.
+  */
+  async _loadIntlData() {
+    // Wait for the startup topic.
+    await Policy._browserDelayedStartup();
+    this._currentEnvironment.settings.intl = getIntlSettings();
+    Policy._intlLoaded = true;
+  },
+
+  /**
    * Get the partner data in object form.
    * @return Object containing the partner data.
    */
   _getPartner() {
     let partnerData = {
       distributionId: Services.prefs.getStringPref(PREF_DISTRIBUTION_ID, null),
       distributionVersion: Services.prefs.getStringPref(PREF_DISTRIBUTION_VERSION, null),
       partnerId: Services.prefs.getStringPref(PREF_PARTNER_ID, null),
--- a/toolkit/components/telemetry/docs/data/environment.rst
+++ b/toolkit/components/telemetry/docs/data/environment.rst
@@ -44,16 +44,24 @@ Structure:
           origin: <string>, // 'default', 'verified', 'unverified', or 'invalid'; based on the presence and validity of the engine's loadPath verification hash.
           submissionURL: <string> // set for default engines or well known search domains
         },
         searchCohort: <string>, // optional, contains an identifier for any active search A/B experiments
         launcherProcessState: <integer>, // optional, values correspond to values of mozilla::LauncherRegistryInfo::EnabledState enum
         e10sEnabled: <bool>, // whether e10s is on, i.e. browser tabs open by default in a different process
         telemetryEnabled: <bool>, // false on failure
         locale: <string>, // e.g. "it", null on failure
+        intl: {
+          requestedLocales: [ <string>, ... ], // The locales that are being requested.
+          availableLocales: [ <string>, ... ], // The locales that are available for use.
+          appLocales: [ <string>, ... ], // The negotiated locales that are being used.
+          systemLocales: [ <string>, ... ], // The locales for the OS.
+          regionalPrefsLocales: [ <string>, ... ], // The regional preferences for the OS.
+          acceptLanguages: [ <string>, ... ], // The languages for the Accept-Languages header.
+        },
         update: {
           channel: <string>, // e.g. "release", null on failure
           enabled: <bool>, // true on failure
           autoDownload: <bool>, // true on failure
         },
         userPrefs: {
           // Only prefs which are changed are listed in this block
           "pref.name.value": value // some prefs send the value
@@ -456,8 +464,9 @@ For each experiment we collect the ``id`
 
 
 Version History
 ---------------
 
 - Firefox 61:
 
   - Removed empty ``addons.activeExperiment`` (`bug 1452935 <https://bugzilla.mozilla.org/show_bug.cgi?id=1452935>`_).
+
--- a/toolkit/components/telemetry/tests/unit/head.js
+++ b/toolkit/components/telemetry/tests/unit/head.js
@@ -289,16 +289,23 @@ function fakeGzipCompressStringForNextPi
   };
 }
 
 function fakePrioEncode() {
   const m = ChromeUtils.import("resource://gre/modules/TelemetrySession.jsm", {});
   m.Policy.prioEncode = (batchID, prioParams) => prioParams;
 }
 
+function fakeIntlReady() {
+  const m = ChromeUtils.import("resource://gre/modules/TelemetryEnvironment.jsm", {});
+  m.Policy._intlLoaded = true;
+  // Dispatch the observer event in case the promise has been registered already.
+  Services.obs.notifyObservers(null, "browser-delayed-startup-finished");
+}
+
 // Return a date that is |offset| ms in the future from |date|.
 function futureDate(date, offset) {
   return new Date(date.getTime() + offset);
 }
 
 function truncateToDays(aMsec) {
   return Math.floor(aMsec / MILLISECONDS_PER_DAY);
 }
--- a/toolkit/components/telemetry/tests/unit/test_ChildEvents.js
+++ b/toolkit/components/telemetry/tests/unit/test_ChildEvents.js
@@ -70,16 +70,17 @@ add_task(async function() {
     do_send_remote_message(MESSAGE_CHILD_TEST_DONE);
     return;
   }
 
   // Setup.
   do_get_profile(true);
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
   finishAddonManagerStartup();
+  fakeIntlReady();
   await TelemetryController.testSetup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
   // Enable recording for the test event category.
   Telemetry.setEventRecordingEnabled("telemetry.test", true);
 
   // Register dynamic test events.
   Telemetry.registerEvents("telemetry.test.dynamic", {
--- a/toolkit/components/telemetry/tests/unit/test_ChildHistograms.js
+++ b/toolkit/components/telemetry/tests/unit/test_ChildHistograms.js
@@ -85,16 +85,17 @@ add_task(async function() {
     do_send_remote_message(MESSAGE_CHILD_TEST_DONE);
     return;
   }
 
   // Setup.
   do_get_profile(true);
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
   finishAddonManagerStartup();
+  fakeIntlReady();
   await TelemetryController.testSetup();
   if (runningInParent) {
     // Make sure we don't generate unexpected pings due to pref changes.
     await setEmptyPrefWatchlist();
   }
 
   // Run test in child, don't wait for it to finish.
   run_test_in_child("test_ChildHistograms.js");
--- a/toolkit/components/telemetry/tests/unit/test_ChildScalars.js
+++ b/toolkit/components/telemetry/tests/unit/test_ChildScalars.js
@@ -135,16 +135,17 @@ add_task(async function() {
     do_send_remote_message(MESSAGE_CHILD_TEST_DONE);
     return;
   }
 
   // Setup.
   do_get_profile(true);
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
   finishAddonManagerStartup();
+  fakeIntlReady();
   await TelemetryController.testSetup();
   if (runningInParent) {
     setParentScalars();
     // Make sure we don't generate unexpected pings due to pref changes.
     await setEmptyPrefWatchlist();
   }
 
   // Run test in child, don't wait for it to finish: just wait for the
--- a/toolkit/components/telemetry/tests/unit/test_SubsessionChaining.js
+++ b/toolkit/components/telemetry/tests/unit/test_SubsessionChaining.js
@@ -83,16 +83,17 @@ var promiseValidateArchivedPings = async
 
 add_task(async function test_setup() {
   do_test_pending();
 
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
   finishAddonManagerStartup();
+  fakeIntlReady();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 });
 
 add_task(async function test_subsessionsChaining() {
   if (gIsAndroid) {
     // We don't support subsessions yet on Android, so skip the next checks.
     return;
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryChildEvents_buildFaster.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryChildEvents_buildFaster.js
@@ -40,16 +40,17 @@ add_task(async function test_setup() {
     do_send_remote_message(MESSAGE_CHILD_TEST_DONE);
     return;
   }
 
   // Setup.
   do_get_profile(true);
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
   finishAddonManagerStartup();
+  fakeIntlReady();
   await TelemetryController.testSetup();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
   // Enable recording for the test event category.
 
   // Register some dynamic builtin test events.
   Telemetry.registerBuiltinEvents(TEST_EVENT_NAME, {
     "dynamic": {
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryController.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryController.js
@@ -89,16 +89,17 @@ function checkPingFormat(aPing, aType, a
   Assert.equal("environment" in aPing, aHasEnvironment);
 }
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
   finishAddonManagerStartup();
+  fakeIntlReady();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.FhrUploadEnabled, true);
 
   await new Promise(resolve =>
     Telemetry.asyncFetchTelemetryData(wrapWithExceptionHandler(resolve)));
 });
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryControllerShutdown.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryControllerShutdown.js
@@ -22,16 +22,17 @@ function contentHandler(metadata, respon
   response.setHeader("Content-Type", "text/plain");
 }
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
   finishAddonManagerStartup();
+  fakeIntlReady();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.FhrUploadEnabled, true);
 });
 
 /**
  * Ensures that TelemetryController does not hang processing shutdown
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
@@ -402,18 +402,19 @@ function checkBuildSection(data) {
   Assert.equal(data.build.updaterAvailable, AppConstants.MOZ_UPDATER,
                "build.updaterAvailable must equal AppConstants.MOZ_UPDATER");
 }
 
 function checkSettingsSection(data) {
   const EXPECTED_FIELDS_TYPES = {
     blocklistEnabled: "boolean",
     e10sEnabled: "boolean",
+    intl: "object",
+    locale: "string",
     telemetryEnabled: "boolean",
-    locale: "string",
     update: "object",
     userPrefs: "object",
   };
 
   Assert.ok("settings" in data, "There must be a settings section in Environment.");
 
   for (let f in EXPECTED_FIELDS_TYPES) {
     Assert.equal(typeof data.settings[f], EXPECTED_FIELDS_TYPES[f],
@@ -449,16 +450,44 @@ function checkSettingsSection(data) {
     checkString(data.settings.defaultSearchEngine);
     Assert.equal(typeof data.settings.defaultSearchEngineData, "object");
   }
 
   if (gIsWindows && AppConstants.MOZ_BUILD_APP == "browser") {
     Assert.equal(typeof data.settings.attribution, "object");
     Assert.equal(data.settings.attribution.source, "google.com");
   }
+
+  checkIntlSettings(data.settings);
+}
+
+function checkIntlSettings({intl}) {
+  let fields = [
+    "requestedLocales",
+    "availableLocales",
+    "appLocales",
+    "acceptLanguages",
+  ];
+
+  for (let field of fields) {
+    Assert.ok(Array.isArray(intl[field]), `${field} is an array`);
+  }
+
+  // These fields may be null if they aren't ready yet. This is mostly to deal
+  // with test failures on Android, but they aren't guaranteed to exist.
+  let optionalFields = [
+    "systemLocales",
+    "regionalPrefsLocales",
+  ];
+
+  for (let field of optionalFields) {
+    let isArray = Array.isArray(intl[field]);
+    let isNull = intl[field] === null;
+    Assert.ok(isArray || isNull, `${field} is an array or null`);
+  }
 }
 
 function checkProfileSection(data) {
   Assert.ok("profile" in data, "There must be a profile section in Environment.");
   Assert.equal(data.profile.creationDate, truncateToDays(PROFILE_CREATION_DATE_MS));
   Assert.equal(data.profile.resetDate, truncateToDays(PROFILE_RESET_DATE_MS));
   Assert.equal(data.profile.firstUseDate, truncateToDays(PROFILE_FIRST_USE_MS));
 }
@@ -909,21 +938,30 @@ add_task(async function setup() {
 
 add_task(async function test_checkEnvironment() {
   // During startup we have partial addon records.
   // First make sure we haven't yet read the addons DB, then test that
   // we have some partial addons data.
   Assert.equal(AddonManagerPrivate.isDBLoaded(), false,
                "addons database is not loaded");
 
-  checkAddonsSection(TelemetryEnvironment.currentEnvironment, false, true);
+  let data = TelemetryEnvironment.currentEnvironment;
+  checkAddonsSection(data, false, true);
+
+  // Check that settings.intl is lazily loaded.
+  Assert.equal(typeof data.settings.intl, "object", "intl is initially an object");
+  Assert.equal(Object.keys(data.settings.intl).length, 0, "intl is initially empty");
 
   // Now continue with startup.
   let initPromise = TelemetryEnvironment.onInitialized();
   finishAddonManagerStartup();
+
+  // Fake the delayed startup event for intl data to load.
+  fakeIntlReady();
+
   let environmentData = await initPromise;
   checkEnvironmentData(environmentData, {isInitial: true});
 
   spoofPartnerInfo();
   Services.obs.notifyObservers(null, DISTRIBUTION_CUSTOMIZATION_COMPLETE_TOPIC);
 
   environmentData = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(environmentData);
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryReportingPolicy.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryReportingPolicy.js
@@ -48,16 +48,17 @@ function setMinimumPolicyVersion(aNewPol
   Preferences.set(TelemetryUtils.Preferences.MinimumPolicyVersion, aNewPolicyVersion);
 }
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile(true);
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
   finishAddonManagerStartup();
+  fakeIntlReady();
 
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   // Don't bypass the notifications in this test, we'll fake it.
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.BypassNotification, false);
 
   TelemetryReportingPolicy.setup();
--- a/toolkit/components/telemetry/tests/unit/test_TelemetrySendOldPings.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetrySendOldPings.js
@@ -131,16 +131,17 @@ function pingHandler(aRequest) {
 }
 
 add_task(async function test_setup() {
   PingServer.start();
   PingServer.registerPingHandler(pingHandler);
   do_get_profile();
   loadAddonManager("xpcshell@tests.mozilla.org", "XPCShell", "1", "1.9.2");
   finishAddonManagerStartup();
+  fakeIntlReady();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setCharPref(TelemetryUtils.Preferences.Server,
                               "http://localhost:" + PingServer.port);
 });
 
 /**
--- a/toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetrySession.js
@@ -437,16 +437,17 @@ function write_fake_failedprofilelocks_f
   writeStringToFile(file, contents);
 }
 
 add_task(async function test_setup() {
   // Addon manager needs a profile directory
   do_get_profile();
   loadAddonManager(APP_ID, APP_NAME, APP_VERSION, PLATFORM_VERSION);
   finishAddonManagerStartup();
+  fakeIntlReady();
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   Services.prefs.setBoolPref(TelemetryUtils.Preferences.FhrUploadEnabled, true);
 
   // Make it look like we've previously failed to lock a profile a couple times.
   write_fake_failedprofilelocks_file();
 
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryTimestamps.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryTimestamps.js
@@ -17,16 +17,17 @@ var gGlobalScope = this;
 function getSimpleMeasurementsFromTelemetryController() {
   return TelemetrySession.getPayload().simpleMeasurements;
 }
 
 add_task(async function test_setup() {
   // Telemetry needs the AddonManager.
   loadAddonManager();
   finishAddonManagerStartup();
+  fakeIntlReady();
   // Make profile available for |TelemetryController.testShutdown()|.
   do_get_profile();
 
   // Make sure we don't generate unexpected pings due to pref changes.
   await setEmptyPrefWatchlist();
 
   await new Promise(resolve =>
     Services.telemetry.asyncFetchTelemetryData(resolve));