Bug 1164159 - Send information about the default search engines through Telemetry, r=markh,gfritzsche.
authorFlorian Quèze <florian@queze.net>
Wed, 10 Jun 2015 11:17:31 +0200
changeset 248118 a01b1fbc61e9a91cb1c2ce17147a88ce15d10a9c
parent 248117 57ec499fafb57645f5aa26ab8a9e2247a258cfb2
child 248119 027ffd03bae2e88b17e12076c9147fdf08673b4a
push id60888
push userkwierso@gmail.com
push dateThu, 11 Jun 2015 01:38:38 +0000
treeherdermozilla-inbound@39e638ed06bf [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmarkh, gfritzsche
bugs1164159
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1164159 - Send information about the default search engines through Telemetry, r=markh,gfritzsche.
toolkit/components/search/nsSearchService.js
toolkit/components/telemetry/TelemetryEnvironment.jsm
toolkit/components/telemetry/docs/environment.rst
toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
--- a/toolkit/components/search/nsSearchService.js
+++ b/toolkit/components/search/nsSearchService.js
@@ -2813,16 +2813,85 @@ Engine.prototype = {
     ENSURE_WARN(this._file, "_id: no _file for non-JAR engine",
                 Cr.NS_ERROR_UNEXPECTED);
 
     // We're not in the profile or appdir, so this must be an extension-shipped
     // plugin. Use the full filename.
     return this.__id = this._file.path;
   },
 
+  // This indicates where we found the .xml file to load the engine,
+  // and attempts to hide user-identifiable data (such as username).
+  get _anonymizedLoadPath() {
+    /* Examples of expected output:
+     *   jar:[app]/omni.ja!browser/engine.xml
+     *     'browser' here is the name of the chrome package, not a folder.
+     *   [profile]/searchplugins/engine.xml
+     *   [distribution]/searchplugins/common/engine.xml
+     *   [other]/engine.xml
+     */
+
+    let leafName = this._getLeafName();
+    if (!leafName)
+      return "null";
+
+    let prefix = "", suffix = "";
+    let file = this._file;
+    if (!file) {
+      let uri = this._uri;
+      if (uri.schemeIs("chrome")) {
+        let packageName = uri.hostPort;
+        uri = gChromeReg.convertChromeURL(uri);
+        if (uri instanceof Ci.nsINestedURI) {
+          prefix = "jar:";
+          suffix = "!" + packageName + "/" + leafName;
+          uri = uri.innermostURI;
+        }
+        uri.QueryInterface(Ci.nsIFileURL)
+        file = uri.file;
+      } else {
+        return "[" + uri.scheme + "]/" + leafName;
+      }
+    }
+
+    let id;
+    let enginePath = file.path;
+
+    const NS_XPCOM_CURRENT_PROCESS_DIR = "XCurProcD";
+    const NS_APP_USER_PROFILE_50_DIR = "ProfD";
+    const XRE_APP_DISTRIBUTION_DIR = "XREAppDist";
+
+    const knownDirs = {
+      app: NS_XPCOM_CURRENT_PROCESS_DIR,
+      profile: NS_APP_USER_PROFILE_50_DIR,
+      distribution: XRE_APP_DISTRIBUTION_DIR
+    };
+
+    for (let key in knownDirs) {
+      let path;
+      try {
+        path = getDir(knownDirs[key]).path;
+      } catch(e) {
+        // Getting XRE_APP_DISTRIBUTION_DIR throws during unit tests.
+        continue;
+      }
+      if (enginePath.startsWith(path)) {
+        id = "[" + key + "]" + enginePath.slice(path.length).replace(/\\/g, "/");
+        break;
+      }
+    }
+
+    // If the folder doesn't have a known ancestor, don't record its path to
+    // avoid leaking user identifiable data.
+    if (!id)
+      id = "[other]/" + file.leafName;
+
+    return prefix + id + suffix;
+  },
+
   get _installLocation() {
     if (this.__installLocation === null) {
       if (!this._file) {
         ENSURE_WARN(this._uri, "Engines without files must have URIs",
                     Cr.NS_ERROR_UNEXPECTED);
         this.__installLocation = SEARCH_JAR;
       }
       else if (this._file.parent.equals(getDir(NS_APP_SEARCH_DIR)))
@@ -3195,16 +3264,20 @@ function SearchService() {
     LOG = DO_LOG;
 
   this._initObservers = Promise.defer();
 }
 
 SearchService.prototype = {
   classID: Components.ID("{7319788a-fe93-4db3-9f39-818cf08f4256}"),
 
+  get wrappedJSObject() {
+    return this;
+  },
+
   // The current status of initialization. Note that it does not determine if
   // initialization is complete, only if an error has been encountered so far.
   _initRV: Cr.NS_OK,
 
   // The boolean indicates that the initialization has started or not.
   _initStarted: null,
 
   // If initialization has not been completed yet, perform synchronous
@@ -4592,16 +4665,81 @@ SearchService.prototype = {
     }
 
     engineMetadataService.setGlobalAttr("current", newName);
     engineMetadataService.setGlobalAttr("hash", this._getVerificationHash(newName));
 
     notifyAction(this._currentEngine, SEARCH_ENGINE_CURRENT);
   },
 
+  getDefaultEngineInfo() {
+    let result = {};
+
+    let engine;
+    try {
+      engine = this.defaultEngine;
+    } catch(e) {
+      // The defaultEngine getter will throw if there's no engine at all,
+      // which shouldn't happen unless an add-on or a test deleted all of them.
+      // Our preferences UI doesn't let users do that.
+      Cu.reportError("getDefaultEngineInfo: No default engine");
+    }
+
+    if (!engine) {
+      result.name = "NONE";
+    } else {
+      if (engine.name)
+        result.name = engine.name;
+
+      result.loadPath = engine._anonymizedLoadPath;
+
+      // For privacy, we only collect the submission URL for engines
+      // from the application or distribution folder...
+      let sendSubmissionURL =
+        /^(?:jar:|\[app\]|\[distribution\])/.test(result.loadPath);
+
+      // ... or engines sorted by default near the top of the list.
+      if (!sendSubmissionURL) {
+        let extras =
+          Services.prefs.getChildList(BROWSER_SEARCH_PREF + "order.extra.");
+
+        for (let prefName of extras) {
+          try {
+            if (result.name == Services.prefs.getCharPref(prefName)) {
+              sendSubmissionURL = true;
+              break;
+            }
+          } catch(e) {}
+        }
+
+        let prefNameBase = getGeoSpecificPrefName(BROWSER_SEARCH_PREF + "order");
+        let i = 0;
+        while (!sendSubmissionURL) {
+          let prefName = prefNameBase + "." + (++i);
+          let engineName = getLocalizedPref(prefName);
+          if (!engineName)
+            break;
+          if (result.name == engineName) {
+            sendSubmissionURL = true;
+            break;
+          }
+        }
+      }
+
+      if (sendSubmissionURL) {
+        let uri = engine._getURLOfType("text/html")
+                        .getSubmission("", engine, "searchbar").uri;
+        uri.userPass = ""; // Avoid reporting a username or password.
+        result.submissionURL = uri.spec;
+      }
+    }
+
+    return result;
+  },
+
   /**
    * This map is built lazily after the available search engines change.  It
    * allows quick parsing of an URL representing a search submission into the
    * search engine name and original terms.
    *
    * The keys are strings containing the domain name and lowercase path of the
    * engine submission, for example "www.google.com/search".
    *
--- a/toolkit/components/telemetry/TelemetryEnvironment.jsm
+++ b/toolkit/components/telemetry/TelemetryEnvironment.jsm
@@ -875,16 +875,18 @@ EnvironmentCache.prototype = {
     if (!Services.search.isInitialized) {
       return;
     }
 
     // Make sure we have a settings section.
     this._currentEnvironment.settings = this._currentEnvironment.settings || {};
     // Update the search engine entry in the current environment.
     this._currentEnvironment.settings.defaultSearchEngine = this._getDefaultSearchEngine();
+    this._currentEnvironment.settings.defaultSearchEngineData =
+      Services.search.wrappedJSObject.getDefaultEngineInfo();
   },
 
   /**
    * Update the default search engine value and trigger the environment change.
    */
   _onSearchEngineChange: function () {
     this._log.trace("_onSearchEngineChange");
 
--- a/toolkit/components/telemetry/docs/environment.rst
+++ b/toolkit/components/telemetry/docs/environment.rst
@@ -30,16 +30,21 @@ Structure::
         platformVersion: <string>, // e.g. "35.0"
         xpcomAbi: <string>, // e.g. "x86-msvc"
         hotfixVersion: <string>, // e.g. "20141211.01"
       },
       settings: {
         blocklistEnabled: <bool>, // true on failure
         isDefaultBrowser: <bool>, // null on failure, not available on Android
         defaultSearchEngine: <string>, // e.g. "yahoo"
+        defaultSearchEngineData: {, // data about the current default engine
+          name: <string>, // engine name, e.g. "Yahoo"; or "NONE" if no default
+          loadPath: <string>, // where the engine line is located; missing if no default
+          submissionURL: <string> // missing if no default or for user-installed engines
+        },
         e10sEnabled: <bool>, // false on failure
         telemetryEnabled: <bool>, // false on failure
         locale: <string>, // e.g. "it", null on failure
         update: {
           channel: <string>, // e.g. "release", null on failure
           enabled: <bool>, // true on failure
           autoDownload: <bool>, // true on failure
         },
@@ -192,15 +197,38 @@ Structure::
       },
     }
 
 Settings
 --------
 
 defaultSearchEngine
 ~~~~~~~~~~~~~~~~~~~
+Note: Deprecated, use defaultSearchEngineData instead.
+
 Contains the string identifier or name of the default search engine provider. This will not be present in environment data collected before the Search Service initialization.
 
 The special value ``NONE`` could occur if there is no default search engine.
 
 The special value ``UNDEFINED`` could occur if a default search engine exists but its identifier could not be determined.
 
 This field's contents are ``Services.search.defaultEngine.identifier`` (if defined) or ``"other-"`` + ``Services.search.defaultEngine.name`` if not. In other words, search engines without an ``.identifier`` are prefixed with ``other-``.
+
+defaultSearchEngineData
+~~~~~~~~~~~~~~~~~~~~~~~
+Contains data identifying the engine currently set as the default.
+
+The object contains:
+
+- a ``name`` property with the name of the engine, or ``NONE`` if no
+  engine is currently set as the default.
+
+- a ``loadPath`` property: an anonymized path of the engine xml file, e.g.
+ jar:[app]/omni.ja!browser/engine.xml
+  (where 'browser' is the name of the chrome package, not a folder)
+ [profile]/searchplugins/engine.xml
+ [distribution]/searchplugins/common/engine.xml
+ [other]/engine.xml
+
+- a ``submissionURL`` property with the HTTP url we would use to search.
+  For privacy, we don't record this for user-installed engines.
+
+``loadPath`` and ``submissionURL`` are not present if ``name`` is ``NONE``.
--- a/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
+++ b/toolkit/components/telemetry/tests/unit/test_TelemetryEnvironment.js
@@ -279,16 +279,17 @@ function checkSettingsSection(data) {
   let update = data.settings.update;
   Assert.ok(checkNullOrString(update.channel));
   Assert.equal(typeof update.enabled, "boolean");
   Assert.equal(typeof update.autoDownload, "boolean");
 
   // Check "defaultSearchEngine" separately, as it can either be undefined or string.
   if ("defaultSearchEngine" in data.settings) {
     checkString(data.settings.defaultSearchEngine);
+    Assert.equal(typeof data.settings.defaultSearchEngineData, "object");
   }
 }
 
 function checkProfileSection(data) {
   if (gIsAndroid) {
     Assert.ok(!("profile" in data),
               "There must be no profile section in Environment on Android.");
     return;
@@ -989,16 +990,17 @@ add_task(function* test_changeThrottling
 });
 
 add_task(function* test_defaultSearchEngine() {
   // Check that no default engine is in the environment before the search service is
   // initialized.
   let data = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(data);
   Assert.ok(!("defaultSearchEngine" in data.settings));
+  Assert.ok(!("defaultSearchEngineData" in data.settings));
 
   // Load the engines definitions from a custom JAR file: that's needed so that
   // the search provider reports an engine identifier.
   let defaultBranch = Services.prefs.getDefaultBranch(null);
   defaultBranch.setCharPref("browser.search.jarURIs", "chrome://testsearchplugin/locale/searchplugins/");
   defaultBranch.setBoolPref("browser.search.loadFromJars", true);
 
   // Initialize the search service and disable geoip lookup, so we don't get unwanted
@@ -1006,30 +1008,37 @@ add_task(function* test_defaultSearchEng
   Preferences.set("browser.search.geoip.url", "");
   yield new Promise(resolve => Services.search.init(resolve));
 
   // Our default engine from the JAR file has an identifier. Check if it is correctly
   // reported.
   data = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(data);
   Assert.equal(data.settings.defaultSearchEngine, "telemetrySearchIdentifier");
+  let expectedSearchEngineData = {
+    name: "telemetrySearchIdentifier",
+    loadPath: "jar:[other]/searchTest.jar!testsearchplugin/telemetrySearchIdentifier.xml",
+    submissionURL: "http://ar.wikipedia.org/wiki/%D8%AE%D8%A7%D8%B5:%D8%A8%D8%AD%D8%AB?search=&sourceid=Mozilla-search"
+  };
+  Assert.deepEqual(data.settings.defaultSearchEngineData, expectedSearchEngineData);
 
   // Remove all the search engines.
   for (let engine of Services.search.getEngines()) {
     Services.search.removeEngine(engine);
   }
   // The search service does not notify "engine-default" when removing a default engine.
   // Manually force the notification.
   // TODO: remove this when bug 1165341 is resolved.
   Services.obs.notifyObservers(null, "browser-search-engine-modified", "engine-default");
 
   // Then check that no default engine is reported if none is available.
   data = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(data);
   Assert.equal(data.settings.defaultSearchEngine, "NONE");
+  Assert.deepEqual(data.settings.defaultSearchEngineData, {name:"NONE"});
 
   // Add a new search engine (this will have no engine identifier).
   const SEARCH_ENGINE_ID = "telemetry_default";
   const SEARCH_ENGINE_URL = "http://www.example.org/?search={searchTerms}";
   Services.search.addEngineWithDetails(SEARCH_ENGINE_ID, "", null, "", "get", SEARCH_ENGINE_URL);
 
   // Set the clock in the future so our changes don't get throttled.
   gNow = fakeNow(futureDate(gNow, 10 * MILLISECONDS_PER_MINUTE));
@@ -1039,16 +1048,22 @@ add_task(function* test_defaultSearchEng
   Services.search.defaultEngine = Services.search.getEngineByName(SEARCH_ENGINE_ID);
   yield deferred.promise;
 
   data = TelemetryEnvironment.currentEnvironment;
   checkEnvironmentData(data);
 
   const EXPECTED_SEARCH_ENGINE = "other-" + SEARCH_ENGINE_ID;
   Assert.equal(data.settings.defaultSearchEngine, EXPECTED_SEARCH_ENGINE);
+
+  const EXPECTED_SEARCH_ENGINE_DATA = {
+    name: "telemetry_default",
+    loadPath: "[profile]/searchplugins/telemetrydefault.xml"
+  };
+  Assert.deepEqual(data.settings.defaultSearchEngineData, EXPECTED_SEARCH_ENGINE_DATA);
   TelemetryEnvironment.unregisterChangeListener("testWatch_SearchDefault");
 
   // Define and reset the test preference.
   const PREF_TEST = "toolkit.telemetry.test.pref1";
   const PREFS_TO_WATCH = new Map([
     [PREF_TEST, TelemetryEnvironment.RECORD_PREF_STATE],
   ]);
   Preferences.reset(PREF_TEST);