Bug 1164159 - Send information about the default search engines through Telemetry. r=markh, r=gfritzsche, a=sledru
authorFlorian Quèze <florian@queze.net>
Wed, 10 Jun 2015 11:17:31 +0200
changeset 273662 6c7c1fd1f20291fcac5c7f1773fc0a5f215b1903
parent 273661 cfc6f3c1c1f6a5e1faff52ca19a3ddd734d33420
child 273663 7fdd6ef122314c4998f1783919b81d1427e01b5f
push id4830
push userjlund@mozilla.com
push dateMon, 29 Jun 2015 20:18:48 +0000
treeherdermozilla-beta@4c2175bb0420 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmarkh, gfritzsche, sledru
bugs1164159
milestone40.0a2
Bug 1164159 - Send information about the default search engines through Telemetry. r=markh, r=gfritzsche, a=sledru
toolkit/components/search/nsSearchService.js
toolkit/components/telemetry/TelemetryEnvironment.jsm
toolkit/components/telemetry/docs/environment.rst
toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
--- a/toolkit/components/search/nsSearchService.js
+++ b/toolkit/components/search/nsSearchService.js
@@ -2813,16 +2813,85 @@ Engine.prototype = {
     ENSURE_WARN(this._file, "_id: no _file for non-JAR engine",
                 Cr.NS_ERROR_UNEXPECTED);
 
     // We're not in the profile or appdir, so this must be an extension-shipped
     // plugin. Use the full filename.
     return this.__id = this._file.path;
   },
 
+  // This indicates where we found the .xml file to load the engine,
+  // and attempts to hide user-identifiable data (such as username).
+  get _anonymizedLoadPath() {
+    /* Examples of expected output:
+     *   jar:[app]/omni.ja!browser/engine.xml
+     *     'browser' here is the name of the chrome package, not a folder.
+     *   [profile]/searchplugins/engine.xml
+     *   [distribution]/searchplugins/common/engine.xml
+     *   [other]/engine.xml
+     */
+
+    let leafName = this._getLeafName();
+    if (!leafName)
+      return "null";
+
+    let prefix = "", suffix = "";
+    let file = this._file;
+    if (!file) {
+      let uri = this._uri;
+      if (uri.schemeIs("chrome")) {
+        let packageName = uri.hostPort;
+        uri = gChromeReg.convertChromeURL(uri);
+        if (uri instanceof Ci.nsINestedURI) {
+          prefix = "jar:";
+          suffix = "!" + packageName + "/" + leafName;
+          uri = uri.innermostURI;
+        }
+        uri.QueryInterface(Ci.nsIFileURL)
+        file = uri.file;
+      } else {
+        return "[" + uri.scheme + "]/" + leafName;
+      }
+    }
+
+    let id;
+    let enginePath = file.path;
+
+    const NS_XPCOM_CURRENT_PROCESS_DIR = "XCurProcD";
+    const NS_APP_USER_PROFILE_50_DIR = "ProfD";
+    const XRE_APP_DISTRIBUTION_DIR = "XREAppDist";
+
+    const knownDirs = {
+      app: NS_XPCOM_CURRENT_PROCESS_DIR,
+      profile: NS_APP_USER_PROFILE_50_DIR,
+      distribution: XRE_APP_DISTRIBUTION_DIR
+    };
+
+    for (let key in knownDirs) {
+      let path;
+      try {
+        path = getDir(knownDirs[key]).path;
+      } catch(e) {
+        // Getting XRE_APP_DISTRIBUTION_DIR throws during unit tests.
+        continue;
+      }
+      if (enginePath.startsWith(path)) {
+        id = "[" + key + "]" + enginePath.slice(path.length).replace(/\\/g, "/");
+        break;
+      }
+    }
+
+    // If the folder doesn't have a known ancestor, don't record its path to
+    // avoid leaking user identifiable data.
+    if (!id)
+      id = "[other]/" + file.leafName;
+
+    return prefix + id + suffix;
+  },
+
   get _installLocation() {
     if (this.__installLocation === null) {
       if (!this._file) {
         ENSURE_WARN(this._uri, "Engines without files must have URIs",
                     Cr.NS_ERROR_UNEXPECTED);
         this.__installLocation = SEARCH_JAR;
       }
       else if (this._file.parent.equals(getDir(NS_APP_SEARCH_DIR)))
@@ -3195,16 +3264,20 @@ function SearchService() {
     LOG = DO_LOG;
 
   this._initObservers = Promise.defer();
 }
 
 SearchService.prototype = {
   classID: Components.ID("{7319788a-fe93-4db3-9f39-818cf08f4256}"),
 
+  get wrappedJSObject() {
+    return this;
+  },
+
   // The current status of initialization. Note that it does not determine if
   // initialization is complete, only if an error has been encountered so far.
   _initRV: Cr.NS_OK,
 
   // The boolean indicates that the initialization has started or not.
   _initStarted: null,
 
   // If initialization has not been completed yet, perform synchronous
@@ -4592,16 +4665,81 @@ SearchService.prototype = {
     }
 
     engineMetadataService.setGlobalAttr("current", newName);
     engineMetadataService.setGlobalAttr("hash", this._getVerificationHash(newName));
 
     notifyAction(this._currentEngine, SEARCH_ENGINE_CURRENT);
   },
 
+  getDefaultEngineInfo() {
+    let result = {};
+
+    let engine;
+    try {
+      engine = this.defaultEngine;
+    } catch(e) {
+      // The defaultEngine getter will throw if there's no engine at all,
+      // which shouldn't happen unless an add-on or a test deleted all of them.
+      // Our preferences UI doesn't let users do that.
+      Cu.reportError("getDefaultEngineInfo: No default engine");
+    }
+
+    if (!engine) {
+      result.name = "NONE";
+    } else {
+      if (engine.name)
+        result.name = engine.name;
+
+      result.loadPath = engine._anonymizedLoadPath;
+
+      // For privacy, we only collect the submission URL for engines
+      // from the application or distribution folder...
+      let sendSubmissionURL =
+        /^(?:jar:|\[app\]|\[distribution\])/.test(result.loadPath);
+
+      // ... or engines sorted by default near the top of the list.
+      if (!sendSubmissionURL) {
+        let extras =
+          Services.prefs.getChildList(BROWSER_SEARCH_PREF + "order.extra.");
+
+        for (let prefName of extras) {
+          try {
+            if (result.name == Services.prefs.getCharPref(prefName)) {
+              sendSubmissionURL = true;
+              break;
+            }
+          } catch(e) {}
+        }
+
+        let prefNameBase = getGeoSpecificPrefName(BROWSER_SEARCH_PREF + "order");
+        let i = 0;
+        while (!sendSubmissionURL) {
+          let prefName = prefNameBase + "." + (++i);
+          let engineName = getLocalizedPref(prefName);
+          if (!engineName)
+            break;
+          if (result.name == engineName) {
+            sendSubmissionURL = true;
+            break;
+          }
+        }
+      }
+
+      if (sendSubmissionURL) {
+        let uri = engine._getURLOfType("text/html")
+                        .getSubmission("", engine, "searchbar").uri;
+        uri.userPass = ""; // Avoid reporting a username or password.
+        result.submissionURL = uri.spec;
+      }
+    }
+
+    return result;
+  },
+
   /**
    * This map is built lazily after the available search engines change.  It
    * allows quick parsing of an URL representing a search submission into the
    * search engine name and original terms.
    *
    * The keys are strings containing the domain name and lowercase path of the
    * engine submission, for example "www.google.com/search".
    *
--- a/toolkit/components/telemetry/TelemetryEnvironment.jsm
+++ b/toolkit/components/telemetry/TelemetryEnvironment.jsm
@@ -871,16 +871,18 @@ EnvironmentCache.prototype = {
     if (!Services.search.isInitialized) {
       return;
     }
 
     // Make sure we have a settings section.
     this._currentEnvironment.settings = this._currentEnvironment.settings || {};
     // Update the search engine entry in the current environment.
     this._currentEnvironment.settings.defaultSearchEngine = this._getDefaultSearchEngine();
+    this._currentEnvironment.settings.defaultSearchEngineData =
+      Services.search.wrappedJSObject.getDefaultEngineInfo();
   },
 
   /**
    * Update the default search engine value and trigger the environment change.
    */
   _onSearchEngineChange: function () {
     this._log.trace("_onSearchEngineChange");
 
--- a/toolkit/components/telemetry/docs/environment.rst
+++ b/toolkit/components/telemetry/docs/environment.rst
@@ -30,16 +30,21 @@ Structure::
         platformVersion: <string>, // e.g. "35.0"
         xpcomAbi: <string>, // e.g. "x86-msvc"
         hotfixVersion: <string>, // e.g. "20141211.01"
       },
       settings: {
         blocklistEnabled: <bool>, // true on failure
         isDefaultBrowser: <bool>, // null on failure, not available on Android
         defaultSearchEngine: <string>, // e.g. "yahoo"
+        defaultSearchEngineData: {, // data about the current default engine
+          name: <string>, // engine name, e.g. "Yahoo"; or "NONE" if no default
+          loadPath: <string>, // where the engine line is located; missing if no default
+          submissionURL: <string> // missing if no default or for user-installed engines
+        },
         e10sEnabled: <bool>, // false on failure
         telemetryEnabled: <bool>, // false on failure
         locale: <string>, // e.g. "it", null on failure
         update: {
           channel: <string>, // e.g. "release", null on failure
           enabled: <bool>, // true on failure
           autoDownload: <bool>, // true on failure
         },
@@ -191,15 +196,38 @@ Structure::
       },
     }
 
 Settings
 --------
 
 defaultSearchEngine
 ~~~~~~~~~~~~~~~~~~~
+Note: Deprecated, use defaultSearchEngineData instead.
+
 Contains the string identifier or name of the default search engine provider. This will not be present in environment data collected before the Search Service initialization.
 
 The special value ``NONE`` could occur if there is no default search engine.
 
 The special value ``UNDEFINED`` could occur if a default search engine exists but its identifier could not be determined.
 
 This field's contents are ``Services.search.defaultEngine.identifier`` (if defined) or ``"other-"`` + ``Services.search.defaultEngine.name`` if not. In other words, search engines without an ``.identifier`` are prefixed with ``other-``.
+
+defaultSearchEngineData
+~~~~~~~~~~~~~~~~~~~~~~~
+Contains data identifying the engine currently set as the default.
+
+The object contains:
+
+- a ``name`` property with the name of the engine, or ``NONE`` if no
+  engine is currently set as the default.
+
+- a ``loadPath`` property: an anonymized path of the engine xml file, e.g.
+ jar:[app]/omni.ja!browser/engine.xml
+  (where 'browser' is the name of the chrome package, not a folder)
+ [profile]/searchplugins/engine.xml
+ [distribution]/searchplugins/common/engine.xml
+ [other]/engine.xml
+
+- a ``submissionURL`` property with the HTTP url we would use to search.
+  For privacy, we don't record this for user-installed engines.
+
+``loadPath`` and ``submissionURL`` are not present if ``name`` is ``NONE``.
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
@@ -279,16 +279,17 @@ function checkSettingsSection(data) {
   let update = data.settings.update;
   Assert.ok(checkNullOrString(update.channel));
   Assert.equal(typeof update.enabled, "boolean");
   Assert.equal(typeof update.autoDownload, "boolean");
 
   // Check "defaultSearchEngine" separately, as it can either be undefined or string.
   if ("defaultSearchEngine" in data.settings) {
     checkString(data.settings.defaultSearchEngine);
+    Assert.equal(typeof data.settings.defaultSearchEngineData, "object");
   }
 }
 
 function checkProfileSection(data) {
   if (gIsAndroid) {
     Assert.ok(!("profile" in data),
               "There must be no profile section in Environment on Android.");
     return;
@@ -944,16 +945,17 @@ add_task(function* test_changeThrottling
 });
 
 add_task(function* test_defaultSearchEngine() {
   // Check that no default engine is in the environment before the search service is
   // initialized.
   let data = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(data);
   Assert.ok(!("defaultSearchEngine" in data.settings));
+  Assert.ok(!("defaultSearchEngineData" in data.settings));
 
   // Load the engines definitions from a custom JAR file: that's needed so that
   // the search provider reports an engine identifier.
   let defaultBranch = Services.prefs.getDefaultBranch(null);
   defaultBranch.setCharPref("browser.search.jarURIs", "chrome://testsearchplugin/locale/searchplugins/");
   defaultBranch.setBoolPref("browser.search.loadFromJars", true);
 
   // Initialize the search service and disable geoip lookup, so we don't get unwanted
@@ -961,30 +963,37 @@ add_task(function* test_defaultSearchEng
   Preferences.set("browser.search.geoip.url", "");
   yield new Promise(resolve => Services.search.init(resolve));
 
   // Our default engine from the JAR file has an identifier. Check if it is correctly
   // reported.
   data = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(data);
   Assert.equal(data.settings.defaultSearchEngine, "telemetrySearchIdentifier");
+  let expectedSearchEngineData = {
+    name: "telemetrySearchIdentifier",
+    loadPath: "jar:[other]/searchTest.jar!testsearchplugin/telemetrySearchIdentifier.xml",
+    submissionURL: "http://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB?search=&sourceid=Mozilla-search"
+  };
+  Assert.deepEqual(data.settings.defaultSearchEngineData, expectedSearchEngineData);
 
   // Remove all the search engines.
   for (let engine of Services.search.getEngines()) {
     Services.search.removeEngine(engine);
   }
   // The search service does not notify "engine-default" when removing a default engine.
   // Manually force the notification.
   // TODO: remove this when bug 1165341 is resolved.
   Services.obs.notifyObservers(null, "browser-search-engine-modified", "engine-default");
 
   // Then check that no default engine is reported if none is available.
   data = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(data);
   Assert.equal(data.settings.defaultSearchEngine, "NONE");
+  Assert.deepEqual(data.settings.defaultSearchEngineData, {name:"NONE"});
 
   // Add a new search engine (this will have no engine identifier).
   const SEARCH_ENGINE_ID = "telemetry_default";
   const SEARCH_ENGINE_URL = "http://www.example.org/?search={searchTerms}";
   Services.search.addEngineWithDetails(SEARCH_ENGINE_ID, "", null, "", "get", SEARCH_ENGINE_URL);
 
   // Set the clock in the future so our changes don't get throttled.
   gNow = fakeNow(futureDate(gNow, 10 * MILLISECONDS_PER_MINUTE));
@@ -994,16 +1003,22 @@ add_task(function* test_defaultSearchEng
   Services.search.defaultEngine = Services.search.getEngineByName(SEARCH_ENGINE_ID);
   yield deferred.promise;
 
   data = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(data);
 
   const EXPECTED_SEARCH_ENGINE = "other-" + SEARCH_ENGINE_ID;
   Assert.equal(data.settings.defaultSearchEngine, EXPECTED_SEARCH_ENGINE);
+
+  const EXPECTED_SEARCH_ENGINE_DATA = {
+    name: "telemetry_default",
+    loadPath: "[profile]/searchplugins/telemetrydefault.xml"
+  };
+  Assert.deepEqual(data.settings.defaultSearchEngineData, EXPECTED_SEARCH_ENGINE_DATA);
   TelemetryEnvironment.unregisterChangeListener("testWatch_SearchDefault");
 
   // Define and reset the test preference.
   const PREF_TEST = "toolkit.telemetry.test.pref1";
   const PREFS_TO_WATCH = new Map([
     [PREF_TEST, TelemetryEnvironment.RECORD_PREF_STATE],
   ]);
   Preferences.reset(PREF_TEST);