Bug 1369028 - Import the Follow-on search telemetry system add-on v0.8.0. r=past
authorMark Banner <standard8@mozilla.com>
Mon, 05 Jun 2017 11:06:37 +0100
changeset 362973 6e0d544cc5cadbdb3090c60b532ba1b95fbb32cc
parent 362972 d1272c6ebb3f5cfe2e9bd19cf5d0fee8255d8383
child 362974 d2a4436a43499b7fbc2e47600cfbea3740b896a0
push id91215
push usermbanner@mozilla.com
push dateThu, 08 Jun 2017 14:58:00 +0000
treeherdermozilla-inbound@d2a4436a4349 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerspast
bugs1369028
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1369028 - Import the Follow-on search telemetry system add-on v0.8.0. r=past MozReview-Commit-ID: 6QBnVy3F32g
browser/extensions/followonsearch/.eslintrc.js
browser/extensions/followonsearch/bootstrap.js
browser/extensions/followonsearch/content/followonsearch-fs.js
browser/extensions/followonsearch/install.rdf
new file mode 100644
--- /dev/null
+++ b/browser/extensions/followonsearch/.eslintrc.js
@@ -0,0 +1,8 @@
+"use strict";
+
+module.exports = {
+  "env": {
+    "browser": true,
+    "node": false
+  }
+};
new file mode 100644
--- /dev/null
+++ b/browser/extensions/followonsearch/bootstrap.js
@@ -0,0 +1,241 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://gre/modules/Timer.jsm");
+
+XPCOMUtils.defineLazyModuleGetter(this, "UpdateUtils",
+  "resource://gre/modules/UpdateUtils.jsm");
+XPCOMUtils.defineLazyModuleGetter(this, "TelemetryEnvironment",
+  "resource://gre/modules/TelemetryEnvironment.jsm");
+
+// The amount of people to be part of the telemetry reporting.
+const REPORTING_THRESHOLD = {
+  // "default": 1.0, // 100% - self builds, linux distros etc.
+  "nightly": 0.1, // 10%
+  "beta": 0.1,  // 10%
+  "release": 0.1,  // 10%
+};
+
+// Preferences this add-on uses.
+const kPrefPrefix = "extensions.followonsearch.";
+const PREF_COHORT_SAMPLE = `${kPrefPrefix}cohortSample`;
+const PREF_LOGGING = `${kPrefPrefix}logging`;
+const PREF_CHANNEL_OVERRIDE = `${kPrefPrefix}override`;
+
+const kExtensionID = "followonsearch@mozilla.com";
+const kSaveTelemetryMsg = `${kExtensionID}:save-telemetry`;
+const kShutdownMsg = `${kExtensionID}:shutdown`;
+
+const frameScript = `chrome://followonsearch/content/followonsearch-fs.js?q=${Math.random()}`;
+
+const validSearchTypes = [
+  // A search is a follow-on search from an SAP.
+  "follow-on",
+  // The search is a "search access point".
+  "sap",
+];
+
+var gLoggingEnabled = false;
+var gTelemetryActivated = false;
+
+/**
+ * Logs a message to the console if logging is enabled.
+ *
+ * @param {String} message The message to log.
+ */
+function log(message) {
+  if (gLoggingEnabled) {
+    console.log("Follow-On Search", message);
+  }
+}
+
+/**
+ * Handles receiving a message from the content process to save telemetry.
+ *
+ * @param {Object} message The message received.
+ */
+function handleSaveTelemetryMsg(message) {
+  if (message.name != kSaveTelemetryMsg) {
+    throw new Error(`Unexpected message received: ${kSaveTelemetryMsg}`);
+  }
+
+  let info = message.data;
+
+  if (!validSearchTypes.includes(info.type)) {
+    throw new Error("Unexpected type!");
+  }
+
+  log(info);
+
+  let histogram = Services.telemetry.getKeyedHistogramById("SEARCH_COUNTS");
+  histogram.add(`${info.sap}.${info.type}:unknown:${info.code}`);
+}
+
+/**
+ * Activites recording of telemetry if it isn't already activated.
+ */
+function activateTelemetry() {
+  if (gTelemetryActivated) {
+    return;
+  }
+
+  gTelemetryActivated = true;
+
+  Services.mm.addMessageListener(kSaveTelemetryMsg, handleSaveTelemetryMsg);
+  Services.mm.loadFrameScript(frameScript, true);
+
+  // Record the fact we're saving the extra data as a telemetry environment
+  // value.
+  TelemetryEnvironment.setExperimentActive(kExtensionID, "active");
+}
+
+/**
+ * Deactivites recording of telemetry if it isn't already deactivated.
+ */
+function deactivateTelemetry() {
+  if (!gTelemetryActivated) {
+    return;
+  }
+
+  TelemetryEnvironment.setExperimentInactive(kExtensionID);
+
+  Services.mm.removeMessageListener(kSaveTelemetryMsg, handleSaveTelemetryMsg);
+  Services.mm.removeDelayedFrameScript(frameScript);
+  Services.mm.broadcastAsyncMessage(kShutdownMsg);
+
+  gTelemetryActivated = false;
+}
+
+/**
+ * cohortManager is used to decide which users to enable the add-on for.
+ */
+var cohortManager = {
+  // Indicates whether the telemetry should be enabled.
+  enableForUser: false,
+
+  // Records if we've already run init.
+  _definedThisSession: false,
+
+  /**
+   * Initialises the manager, working out if telemetry should be enabled
+   * for the user.
+   */
+  init() {
+    if (this._definedThisSession) {
+      return;
+    }
+
+    this._definedThisSession = true;
+    this.enableForUser = false;
+
+    try {
+      let distId = Services.prefs.getCharPref("distribution.id", "");
+      if (distId) {
+        log("It is a distribution, not setting up nor enabling.");
+        return;
+      }
+    } catch (e) {}
+
+    let cohortSample;
+    try {
+      cohortSample = Services.prefs.getFloatPref(PREF_COHORT_SAMPLE, undefined);
+    } catch (e) {}
+    if (!cohortSample) {
+      cohortSample = Math.random().toString().substr(0, 8);
+      cohortSample = Services.prefs.setCharPref(PREF_COHORT_SAMPLE, cohortSample);
+    }
+    log(`Cohort Sample value is ${cohortSample}`);
+
+    let updateChannel = UpdateUtils.getUpdateChannel(false);
+    log(`Update channel is ${updateChannel}`);
+    if (!(updateChannel in REPORTING_THRESHOLD)) {
+      let prefOverride = "default";
+      try {
+        prefOverride = Services.prefs.getCharPref(PREF_CHANNEL_OVERRIDE, "default");
+      } catch (e) {}
+      if (prefOverride in REPORTING_THRESHOLD) {
+        updateChannel = prefOverride;
+      } else {
+        // Don't enable, we don't know about the channel, and it isn't overriden.
+        return;
+      }
+    }
+
+    if (cohortSample <= REPORTING_THRESHOLD[updateChannel]) {
+      log("Enabling telemetry for user");
+      this.enableForUser = true;
+    } else {
+      log("Not enabling telemetry for user - outside threshold.");
+    }
+  },
+};
+
+/**
+ * Called when the add-on is installed.
+ *
+ * @param {Object} data Data about the add-on.
+ * @param {Number} reason Indicates why the extension is being installed.
+ */
+function install(data, reason) {
+  try {
+    gLoggingEnabled = Services.prefs.getBoolPref(PREF_LOGGING, false);
+  } catch (e) {
+    // Needed until Firefox 54
+  }
+
+  cohortManager.init();
+  if (cohortManager.enableForUser) {
+    activateTelemetry();
+  }
+}
+
+/**
+ * Called when the add-on is uninstalled.
+ *
+ * @param {Object} data Data about the add-on.
+ * @param {Number} reason Indicates why the extension is being uninstalled.
+ */
+function uninstall(data, reason) {
+  deactivateTelemetry();
+}
+
+/**
+ * Called when the add-on starts up.
+ *
+ * @param {Object} data Data about the add-on.
+ * @param {Number} reason Indicates why the extension is being started.
+ */
+function startup(data, reason) {
+  try {
+    gLoggingEnabled = Services.prefs.getBoolPref(PREF_LOGGING, false);
+  } catch (e) {
+    // Needed until Firefox 54
+  }
+
+  cohortManager.init();
+
+  if (cohortManager.enableForUser) {
+    // Workaround for bug 1202125
+    // We need to delay our loading so that when we are upgraded,
+    // our new script doesn't get the shutdown message.
+    setTimeout(() => {
+      activateTelemetry();
+    }, 1000);
+  }
+}
+
+/**
+ * Called when the add-on shuts down.
+ *
+ * @param {Object} data Data about the add-on.
+ * @param {Number} reason Indicates why the extension is being shut down.
+ */
+function shutdown(data, reason) {
+  deactivateTelemetry();
+}
new file mode 100644
--- /dev/null
+++ b/browser/extensions/followonsearch/content/followonsearch-fs.js
@@ -0,0 +1,283 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/* eslint-env mozilla/frame-script */
+
+"use strict";
+
+const {classes: Cc, interfaces: Ci, utils: Cu} = Components;
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.importGlobalProperties(["URLSearchParams"]);
+
+const kExtensionID = "followonsearch@mozilla.com";
+const kSaveTelemetryMsg = `${kExtensionID}:save-telemetry`;
+const kShutdownMsg = `${kExtensionID}:shutdown`;
+
+/**
+ * A map of search domains with their expected codes.
+ */
+let searchDomains = {
+  "search.yahoo.co.jp": {
+    "search": "p",
+    "followOnSearch": "ai",
+    "prefix": "fr",
+    "codes": ["mozff"],
+    "sap": "yahoo",
+  },
+  "www.bing.com": {
+    "search": "q",
+    "prefix": "pc",
+    "reportPrefix": "form",
+    "codes": ["MOZI"],
+    "sap": "bing",
+  },
+};
+
+// The yahoo domains to watch for.
+const yahooDomains = new Set([
+  "search.yahoo.com", "ca.search.yahoo.com", "hk.search.yahoo.com",
+  "tw.search.yahoo.com",
+]);
+
+// Add Yahoo domains to search domains
+for (let domain of yahooDomains) {
+  searchDomains[domain] = {
+    "search": "p",
+    "followOnSearch": "fr2",
+    "prefix": "hspart",
+    "reportPrefix": "hsimp",
+    "codes": ["mozilla"],
+    "sap": "yahoo",
+  };
+}
+
+const yahooLegacyDomains = new Set([
+  "no.search.yahoo.com", "ar.search.yahoo.com", "br.search.yahoo.com",
+  "ch.search.yahoo.com", "cl.search.yahoo.com", "de.search.yahoo.com",
+  "uk.search.yahoo.com", "es.search.yahoo.com", "espanol.search.yahoo.com",
+  "fi.search.yahoo.com", "fr.search.yahoo.com", "nl.search.yahoo.com",
+  "id.search.yahoo.com", "in.search.yahoo.com", "it.search.yahoo.com",
+  "mx.search.yahoo.com", "se.search.yahoo.com", "sg.search.yahoo.com",
+]);
+
+// Add Yahoo legacy domains to search domains
+for (let domain of yahooLegacyDomains) {
+  searchDomains[domain] = {
+    "search": "p",
+    "followOnSearch": "fr2",
+    "prefix": "fr",
+    "codes": ["moz35"],
+    "sap": "yahoo",
+  };
+}
+
+const googleDomains = new Set([
+  "www.google.com", "www.google.ac", "www.google.ad", "www.google.ae",
+  "www.google.com.af", "www.google.com.ag", "www.google.com.ai",
+  "www.google.al", "www.google.am", "www.google.co.ao", "www.google.com.ar",
+  "www.google.as", "www.google.at", "www.google.com.au", "www.google.az",
+  "www.google.ba", "www.google.com.bd", "www.google.be", "www.google.bf",
+  "www.google.bg", "www.google.com.bh", "www.google.bi", "www.google.bj",
+  "www.google.com.bn", "www.google.com.bo", "www.google.com.br",
+  "www.google.bs", "www.google.bt", "www.google.co.bw", "www.google.by",
+  "www.google.com.bz", "www.google.ca", "www.google.com.kh", "www.google.cc",
+  "www.google.cd", "www.google.cf", "www.google.cat", "www.google.cg",
+  "www.google.ch", "www.google.ci", "www.google.co.ck", "www.google.cl",
+  "www.google.cm", "www.google.cn", "www.google.com.co", "www.google.co.cr",
+  "www.google.com.cu", "www.google.cv", "www.google.cx", "www.google.com.cy",
+  "www.google.cz", "www.google.de", "www.google.dj", "www.google.dk",
+  "www.google.dm", "www.google.com.do", "www.google.dz", "www.google.com.ec",
+  "www.google.ee", "www.google.com.eg", "www.google.es", "www.google.com.et",
+  "www.google.eu", "www.google.fi", "www.google.com.fj", "www.google.fm",
+  "www.google.fr", "www.google.ga", "www.google.ge", "www.google.gf",
+  "www.google.gg", "www.google.com.gh", "www.google.com.gi", "www.google.gl",
+  "www.google.gm", "www.google.gp", "www.google.gr", "www.google.com.gt",
+  "www.google.gy", "www.google.com.hk", "www.google.hn", "www.google.hr",
+  "www.google.ht", "www.google.hu", "www.google.co.id", "www.google.iq",
+  "www.google.ie", "www.google.co.il", "www.google.im", "www.google.co.in",
+  "www.google.io", "www.google.is", "www.google.it", "www.google.je",
+  "www.google.com.jm", "www.google.jo", "www.google.co.jp", "www.google.co.ke",
+  "www.google.ki", "www.google.kg", "www.google.co.kr", "www.google.com.kw",
+  "www.google.kz", "www.google.la", "www.google.com.lb", "www.google.com.lc",
+  "www.google.li", "www.google.lk", "www.google.co.ls", "www.google.lt",
+  "www.google.lu", "www.google.lv", "www.google.com.ly", "www.google.co.ma",
+  "www.google.md", "www.google.me", "www.google.mg", "www.google.mk",
+  "www.google.ml", "www.google.com.mm", "www.google.mn", "www.google.ms",
+  "www.google.com.mt", "www.google.mu", "www.google.mv", "www.google.mw",
+  "www.google.com.mx", "www.google.com.my", "www.google.co.mz",
+  "www.google.com.na", "www.google.ne", "www.google.nf", "www.google.com.ng",
+  "www.google.com.ni", "www.google.nl", "www.google.no", "www.google.com.np",
+  "www.google.nr", "www.google.nu", "www.google.co.nz", "www.google.com.om",
+  "www.google.com.pk", "www.google.com.pa", "www.google.com.pe",
+  "www.google.com.ph", "www.google.pl", "www.google.com.pg", "www.google.pn",
+  "www.google.com.pr", "www.google.ps", "www.google.pt", "www.google.com.py",
+  "www.google.com.qa", "www.google.ro", "www.google.rs", "www.google.ru",
+  "www.google.rw", "www.google.com.sa", "www.google.com.sb", "www.google.sc",
+  "www.google.se", "www.google.com.sg", "www.google.sh", "www.google.si",
+  "www.google.sk", "www.google.com.sl", "www.google.sn", "www.google.sm",
+  "www.google.so", "www.google.st", "www.google.sr", "www.google.com.sv",
+  "www.google.td", "www.google.tg", "www.google.co.th", "www.google.com.tj",
+  "www.google.tk", "www.google.tl", "www.google.tm", "www.google.to",
+  "www.google.tn", "www.google.com.tr", "www.google.tt", "www.google.com.tw",
+  "www.google.co.tz", "www.google.com.ua", "www.google.co.ug",
+  "www.google.co.uk", "www.google.us", "www.google.com.uy", "www.google.co.uz",
+  "www.google.com.vc", "www.google.co.ve", "www.google.vg", "www.google.co.vi",
+  "www.google.com.vn", "www.google.vu", "www.google.ws", "www.google.co.za",
+  "www.google.co.zm", "www.google.co.zw",
+]);
+
+// Add Google domains to search domains
+for (let domain of googleDomains) {
+  searchDomains[domain] = {
+    "search": "q",
+    "prefix": "client",
+    "codes": ["firefox-b-ab", "firefox-b"],
+    "sap": "google",
+  };
+}
+
+/**
+ * Used for debugging to log messages.
+ *
+ * @param {String} message The message to log.
+ */
+function log(message) {
+  // console.log(message);
+}
+
+// Hack to handle the most common reload case.
+// If gLastSearch is the same as the current URL, ignore the search.
+// This also prevents us from handling reloads with hashes twice
+let gLastSearch = null;
+
+/**
+ * Since most codes are in the URL, we can handle them via
+ * a progress listener.
+ */
+var webProgressListener = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsIWebProgressListener, Ci.nsISupportsWeakReference]),
+  onLocationChange(aWebProgress, aRequest, aLocation, aFlags)
+  {
+    try {
+      if (!aWebProgress.isTopLevel ||
+          // Not a URL
+          (!aLocation.schemeIs("http") && !aLocation.schemeIs("https")) ||
+          // Not a domain we handle
+          !(aLocation.host in searchDomains) ||
+          // Doesn't have a query string or a ref
+          (!aLocation.query && !aLocation.ref) ||
+          // Is the same as our last search (avoids reloads)
+          aLocation.spec == gLastSearch) {
+        return;
+      }
+      let domainInfo = searchDomains[aLocation.host];
+
+      let queries = new URLSearchParams(aLocation.query);
+      let code = queries.get(domainInfo.prefix);
+      if (queries.get(domainInfo.search)) {
+        if (domainInfo.codes.includes(code)) {
+          if (domainInfo.reportPrefix &&
+              queries.get(domainInfo.reportPrefix)) {
+            code = queries.get(domainInfo.reportPrefix);
+          }
+          if (googleDomains.has(aLocation.host) && aLocation.ref) {
+            log(`${aLocation.host} search with code ${code} - Follow on`);
+            sendSaveTelemetryMsg(code, domainInfo.sap, "follow-on");
+          } else if (queries.get(domainInfo.followOnSearch)) {
+            log(`${aLocation.host} search with code ${code} - Follow on`);
+            sendSaveTelemetryMsg(code, domainInfo.sap, "follow-on");
+          } else {
+            log(`${aLocation.host} search with code ${code} - First search via Firefox`);
+            sendSaveTelemetryMsg(code, domainInfo.sap, "sap");
+          }
+          gLastSearch = aLocation.spec;
+        }
+      }
+    } catch (e) {
+      console.error(e);
+    }
+  },
+};
+
+/**
+ * Parses a cookie string into separate parts.
+ *
+ * @param {String} cookieString The string to parse.
+ * @param {Object} [params] An optional object to append the parameters to.
+ * @return {Object} An object containing the query keys and values.
+ */
+function parseCookies(cookieString, params = {}) {
+  var cookies = cookieString.split(/;\s*/);
+
+  for (var i in cookies) {
+    var kvp = cookies[i].split(/=(.+)/);
+    params[kvp[0]] = kvp[1];
+  }
+
+  return params;
+}
+
+/**
+ * Page load listener to handle loads www.bing.com only.
+ * We have to use a page load listener because we need
+ * to check cookies.
+ * @param {Object} event The page load event.
+ */
+function onPageLoad(event) {
+  var doc = event.target;
+  var win = doc.defaultView;
+  if (win != win.top) {
+    return;
+  }
+  var uri = doc.documentURIObject;
+  if (!(uri instanceof Ci.nsIStandardURL) ||
+      (!uri.schemeIs("http") && !uri.schemeIs("https")) ||
+       uri.host != "www.bing.com" ||
+      !doc.location.search ||
+      uri.spec == gLastSearch) {
+    return;
+  }
+  var queries = new URLSearchParams(doc.location.search);
+  // For Bing, QBRE form code is used for all follow-on search
+  if (queries.get("form") != "QBRE") {
+    return;
+  }
+  if (parseCookies(doc.cookie).SRCHS == "PC=MOZI") {
+    log(`${uri.host} search with code MOZI - Follow on`);
+    sendSaveTelemetryMsg("MOZI", "bing", "follow-on");
+    gLastSearch = uri.spec;
+  }
+}
+
+/**
+ * Sends a message to the process that added this script to tell it to save
+ * telemetry.
+ *
+ * @param {String} code The codes used for the search engine.
+ * @param {String} sap The SAP code.
+ * @param {String} type The type of search (sap/follow-on).
+ */
+function sendSaveTelemetryMsg(code, sap, type) {
+  sendAsyncMessage(kSaveTelemetryMsg, {
+    code,
+    sap,
+    type,
+  });
+}
+
+addEventListener("DOMContentLoaded", onPageLoad, false);
+docShell.QueryInterface(Ci.nsIInterfaceRequestor).getInterface(Ci.nsIWebProgress)
+        .addProgressListener(webProgressListener, Ci.nsIWebProgress.NOTIFY_LOCATION);
+
+let gDisabled = false;
+
+addMessageListener(kShutdownMsg, () => {
+  if (!gDisabled) {
+    removeEventListener("DOMContentLoaded", onPageLoad, false);
+    docShell.QueryInterface(Ci.nsIInterfaceRequestor).getInterface(Ci.nsIWebProgress)
+            .removeProgressListener(webProgressListener);
+    gDisabled = true;
+  }
+});
new file mode 100644
--- /dev/null
+++ b/browser/extensions/followonsearch/install.rdf
@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<!-- This Source Code Form is subject to the terms of the Mozilla Public
+   - License, v. 2.0. If a copy of the MPL was not distributed with this
+   - file, You can obtain one at http://mozilla.org/MPL/2.0/. -->
+<RDF xmlns="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+     xmlns:em="http://www.mozilla.org/2004/em-rdf#">
+  <Description about="urn:mozilla:install-manifest">
+    <em:id>followonsearch@mozilla.com</em:id>
+    <em:name>Follow-on Search Telemetry</em:name>
+    <em:version>0.8.0</em:version>
+    <em:type>2</em:type>
+    <em:bootstrap>true</em:bootstrap>
+    <em:multiprocessCompatible>true</em:multiprocessCompatible>
+    <em:targetApplication>
+      <Description>
+        <em:id>{ec8030f7-c20a-464f-9b0e-13a3a9e97384}</em:id>
+        <em:minVersion>52.0</em:minVersion>
+        <em:maxVersion>59.*</em:maxVersion>
+      </Description>
+    </em:targetApplication>
+  </Description>
+</RDF>