Bug 1637402 - Add pref to compare MLS results r=chutten,mikedeboer
authorDale Harvey <dale@arandomurl.com>
Tue, 02 Jun 2020 00:01:31 +0000
changeset 533404 34aa06cff443833e1e0bdb8759c98cf3c4b7bf7d
parent 533403 9c1ad8bba1a069e3ff102c91986d4fab44e8cd76
child 533405 a478e84fec4878cd9165bbb0c06dde7a394c5884
push id37470
push userrmaries@mozilla.com
push dateTue, 02 Jun 2020 03:24:46 +0000
treeherdermozilla-central@34aa06cff443 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerschutten, mikedeboer
bugs1637402
milestone79.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1637402 - Add pref to compare MLS results r=chutten,mikedeboer Differential Revision: https://phabricator.services.mozilla.com/D74953
dom/base/LocationHelper.jsm
dom/system/NetworkGeolocationProvider.jsm
dom/system/tests/location_service.sjs
dom/system/tests/location_services_parent.js
dom/system/tests/mochitest.ini
dom/system/tests/test_location_services_telemetry.html
modules/libpref/init/all.js
testing/profiles/common/user.js
testing/profiles/xpcshell/user.js
toolkit/components/telemetry/Histograms.json
--- a/dom/base/LocationHelper.jsm
+++ b/dom/base/LocationHelper.jsm
@@ -27,9 +27,32 @@ function encode(ap) {
 
 class LocationHelper {
   static formatWifiAccessPoints(accessPoints) {
     return accessPoints
       .filter(isPublic)
       .sort(sort)
       .map(encode);
   }
+
+  /**
+   * Calculate the distance between 2 points using the Haversine formula.
+   * https://en.wikipedia.org/wiki/Haversine_formula
+   */
+  static distance(p1, p2) {
+    let rad = x => (x * Math.PI) / 180;
+    // Radius of the earth.
+    let R = 6371e3;
+    let lat = rad(p2.lat - p1.lat);
+    let lng = rad(p2.lng - p1.lng);
+
+    let a =
+      Math.sin(lat / 2) * Math.sin(lat / 2) +
+      Math.cos(rad(p1.lat)) *
+        Math.cos(rad(p2.lat)) *
+        Math.sin(lng / 2) *
+        Math.sin(lng / 2);
+
+    let c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
+
+    return R * c;
+  }
 }
--- a/dom/system/NetworkGeolocationProvider.jsm
+++ b/dom/system/NetworkGeolocationProvider.jsm
@@ -5,23 +5,26 @@
 "use strict";
 
 const { XPCOMUtils } = ChromeUtils.import(
   "resource://gre/modules/XPCOMUtils.jsm"
 );
 const { Services } = ChromeUtils.import("resource://gre/modules/Services.jsm");
 
 XPCOMUtils.defineLazyModuleGetters(this, {
+  clearTimeout: "resource://gre/modules/Timer.jsm",
   LocationHelper: "resource://gre/modules/LocationHelper.jsm",
+  setTimeout: "resource://gre/modules/Timer.jsm",
 });
 
-XPCOMUtils.defineLazyGlobalGetters(this, ["XMLHttpRequest"]);
+XPCOMUtils.defineLazyGlobalGetters(this, ["fetch"]);
 
 // GeolocationPositionError has no interface object, so we can't use that here.
 const POSITION_UNAVAILABLE = 2;
+const TELEMETRY_KEY = "REGION_LOCATION_SERVICES_DIFFERENCE";
 
 XPCOMUtils.defineLazyPreferenceGetter(
   this,
   "gLoggingEnabled",
   "geo.provider.network.logging.enabled",
   false
 );
 
@@ -269,16 +272,23 @@ function NetworkGeolocationProvider() {
 
   XPCOMUtils.defineLazyPreferenceGetter(
     this,
     "_wifiScanningEnabledCountry",
     "geo.provider-country.network.scan",
     true
   );
 
+  XPCOMUtils.defineLazyPreferenceGetter(
+    this,
+    "_wifiCompareURL",
+    "geo.provider.network.compare.url",
+    null
+  );
+
   this.wifiService = null;
   this.timer = null;
   this.started = false;
 }
 
 NetworkGeolocationProvider.prototype = {
   classID: Components.ID("{77DA64D3-7458-4920-9491-86CC9914F904}"),
   QueryInterface: ChromeUtils.generateQI([
@@ -413,17 +423,17 @@ NetworkGeolocationProvider.prototype = {
    *                          in the following structure:
    *                          <code>
    *                          [
    *                            { macAddress: <mac1>, signalStrength: <signal1> },
    *                            { macAddress: <mac2>, signalStrength: <signal2> }
    *                          ]
    *                          </code>
    */
-  sendLocationRequest(wifiData) {
+  async sendLocationRequest(wifiData) {
     let data = { cellTowers: undefined, wifiAccessPoints: undefined };
     if (wifiData && wifiData.length >= 2) {
       data.wifiAccessPoints = wifiData;
     }
 
     let useCached = isCachedRequestMoreAccurateThanServerRequest(
       data.cellTowers,
       data.wifiAccessPoints
@@ -438,69 +448,83 @@ NetworkGeolocationProvider.prototype = {
       }
       return;
     }
 
     // From here on, do a network geolocation request //
     let url = Services.urlFormatter.formatURLPref("geo.provider.network.url");
     LOG("Sending request");
 
-    let xhr = new XMLHttpRequest();
-    this.onStatus(false, "xhr-start");
+    let result;
     try {
-      xhr.open("POST", url, true);
-      xhr.channel.loadFlags = Ci.nsIChannel.LOAD_ANONYMOUS;
-    } catch (e) {
-      this.onStatus(true, "xhr-error");
-      return;
-    }
-    xhr.setRequestHeader("Content-Type", "application/json; charset=UTF-8");
-    xhr.responseType = "json";
-    xhr.mozBackgroundRequest = true;
-    // Allow deprecated HTTP request from SystemPrincipal
-    xhr.channel.loadInfo.allowDeprecatedSystemRequests = true;
-    xhr.timeout = Services.prefs.getIntPref("geo.provider.network.timeout");
-    xhr.ontimeout = () => {
-      LOG("Location request XHR timed out.");
-      this.onStatus(true, "xhr-timeout");
-    };
-    xhr.onerror = () => {
-      this.onStatus(true, "xhr-error");
-    };
-    xhr.onload = () => {
+      result = await this.makeRequest(url, wifiData);
       LOG(
-        "server returned status: " +
-          xhr.status +
-          " --> " +
-          JSON.stringify(xhr.response)
+        `geo provider reported: ${result.location.lng}:${result.location.lat}`
       );
-      if (
-        (xhr.channel instanceof Ci.nsIHttpChannel && xhr.status != 200) ||
-        !xhr.response
-      ) {
-        this.onStatus(true, !xhr.response ? "xhr-empty" : "xhr-error");
-        return;
-      }
-
       let newLocation = new NetworkGeoPositionObject(
-        xhr.response.location.lat,
-        xhr.response.location.lng,
-        xhr.response.accuracy
+        result.location.lat,
+        result.location.lng,
+        result.accuracy
       );
 
       if (this.listener) {
         this.listener.update(newLocation);
       }
 
       gCachedRequest = new CachedRequest(
         newLocation,
         data.cellTowers,
         data.wifiAccessPoints
       );
+    } catch (err) {
+      LOG("Location request hit error: " + err.name);
+      Cu.reportError(err);
+      if (err.name == "AbortError") {
+        this.onStatus(true, "xhr-timeout");
+      } else {
+        this.onStatus(true, "xhr-error");
+      }
+    }
+
+    if (!this._wifiCompareURL) {
+      return;
+    }
+
+    let compareUrl = Services.urlFormatter.formatURL(this._wifiCompareURL);
+    let compare = await this.makeRequest(compareUrl, wifiData);
+    let distance = LocationHelper.distance(result.location, compare.location);
+    LOG(
+      `compare reported reported: ${compare.location.lng}:${compare.location.lat}`
+    );
+    LOG(`distance between results: ${distance}`);
+    if (!isNaN(distance)) {
+      Services.telemetry.getHistogramById(TELEMETRY_KEY).add(distance);
+    }
+  },
+
+  async makeRequest(url, wifiData) {
+    this.onStatus(false, "xhr-start");
+
+    let fetchController = new AbortController();
+    let fetchOpts = {
+      method: "POST",
+      headers: { "Content-Type": "application/json; charset=UTF-8" },
+      credentials: "omit",
+      signal: fetchController.signal,
     };
 
-    var requestData = JSON.stringify(data);
-    LOG("sending " + requestData);
-    xhr.send(requestData);
+    if (wifiData) {
+      fetchOpts.body = JSON.stringify({ wifiAccessPoints: wifiData });
+    }
+
+    let timeoutId = setTimeout(
+      () => fetchController.abort(),
+      Services.prefs.getIntPref("geo.provider.network.timeout")
+    );
+
+    let req = await fetch(url, fetchOpts);
+    clearTimeout(timeoutId);
+    let result = req.json();
+    return result;
   },
 };
 
 var EXPORTED_SYMBOLS = ["NetworkGeolocationProvider"];
new file mode 100644
--- /dev/null
+++ b/dom/system/tests/location_service.sjs
@@ -0,0 +1,39 @@
+function parseQueryString(str) {
+  if (str == "") {
+    return {};
+  }
+
+  var paramArray = str.split("&");
+  var regex = /^([^=]+)=(.*)$/;
+  var params = {};
+  for (var i = 0, sz = paramArray.length; i < sz; i++) {
+    var match = regex.exec(paramArray[i]);
+    if (!match) {
+      throw new Error("Bad parameter in queryString!  '" + paramArray[i] + "'");
+    }
+    params[decodeURIComponent(match[1])] = decodeURIComponent(match[2]);
+  }
+
+  return params;
+}
+
+function getPosition(params) {
+  var response = {
+    status: "OK",
+    accuracy: 100,
+    location: {
+      lat: params.lat,
+      lng: params.lng,
+    },
+  };
+
+  return JSON.stringify(response);
+}
+
+function handleRequest(request, response) {
+  let params = parseQueryString(request.queryString);
+  response.setStatusLine("1.0", 200, "OK");
+  response.setHeader("Cache-Control", "no-cache", false);
+  response.setHeader("Content-Type", "application/x-javascript", false);
+  response.write(getPosition(params));
+}
new file mode 100644
--- /dev/null
+++ b/dom/system/tests/location_services_parent.js
@@ -0,0 +1,20 @@
+/**
+ * Loaded as a frame script fetch telemetry for
+ * test_location_services_telemetry.html
+ */
+
+/* global addMessageListener, sendAsyncMessage */
+
+"use strict";
+
+const HISTOGRAM_KEY = "REGION_LOCATION_SERVICES_DIFFERENCE";
+
+addMessageListener("getTelemetryEvents", options => {
+  let result = Services.telemetry.getHistogramById(HISTOGRAM_KEY).snapshot();
+  sendAsyncMessage("getTelemetryEvents", result);
+});
+
+addMessageListener("clear", options => {
+  Services.telemetry.getHistogramById(HISTOGRAM_KEY).clear();
+  sendAsyncMessage("clear", true);
+});
--- a/dom/system/tests/mochitest.ini
+++ b/dom/system/tests/mochitest.ini
@@ -1,5 +1,11 @@
 [DEFAULT]
+scheme = https
+
 support-files =
   file_bug1197901.html
+  location_services_parent.js
+  location_service.sjs
 
 [test_bug1197901.html]
+[test_location_services_telemetry.html]
+skip-if = os == "android"
new file mode 100644
--- /dev/null
+++ b/dom/system/tests/test_location_services_telemetry.html
@@ -0,0 +1,143 @@
+<!DOCTYPE HTML>
+<html>
+<!--
+https://bugzilla.mozilla.org/show_bug.cgi?id=1637402
+-->
+<head>
+  <meta charset="utf-8">
+  <title>Test for Bug 1637402</title>
+  <script src="/tests/SimpleTest/SimpleTest.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css"/>
+  <script type="application/javascript">
+
+  SimpleTest.requestLongerTimeout(2);
+
+  const BASE_GEO_URL = "http://mochi.test:8888/tests/dom/system/tests/location_service.sjs";
+
+  const GEO_PREF = "geo.provider.network.url";
+  const BACKUP_PREF = "geo.provider.network.compare.url";
+
+  const PARENT = SpecialPowers.loadChromeScript(SimpleTest.getTestFileURL("location_services_parent.js"));
+
+  function sendToParent(msg, options) {
+    return new Promise(resolve => {
+      PARENT.addMessageListener(msg, events => {
+        PARENT.removeMessageListener(msg);
+        resolve(events);
+      });
+      PARENT.sendAsyncMessage(msg, options);
+    });
+  }
+
+  function getCurrentPosition() {
+    return new Promise(function(resolve, reject) {
+      navigator.geolocation.getCurrentPosition(resolve, reject);
+    });
+  }
+
+  let tries = 0;
+  let MAX_RETRIES = 500;
+  async function waitFor(fun) {
+    let passing = false;
+    while (!passing && ++tries < MAX_RETRIES) {
+      passing = await fun();
+    }
+    tries = 0;
+    if (!passing) {
+      ok(false, "waitFor condition never passed");
+    }
+  }
+
+  // Keeps track of how many telemetry results we have
+  // seen so we can wait for new ones.
+  let telemetryResultCount = 0;
+  async function newTelemetryResult() {
+    let results = await sendToParent("getTelemetryEvents");
+    let total = Object.values(results.values)
+      .reduce((val, acc) => acc + val, 0);
+    if (total <= telemetryResultCount) {
+      return false;
+    }
+    telemetryResultCount++;
+    return true;
+  }
+
+  SimpleTest.waitForExplicitFinish();
+  window.onload = () => {
+    SimpleTest.waitForFocus(() => {
+      SpecialPowers.pushPrefEnv({"set":
+        [
+          ["geo.prompt.testing", true],
+          ["geo.prompt.testing.allow", true],
+          ["geo.provider.network.logging.enabled", true],
+          ["geo.provider.network.debug.requestCache.enabled", false]
+        ],
+      }, doTest);
+    }, window);
+  };
+
+  const BASE_LOCATION = {lat: 55.867055, lng: -4.271041};
+  const LOCATIONS = [
+    {lat: "foo", lng: "bar", skipWait: true}, // Nan
+    {lat: 55.867055, lng: -4.271041},   // 0M
+    {lat: 50.8251639, lng: -0.1622551}, // 623KM
+    {lat: 55.9438948, lng: -3.1845417}, // 68KM
+    {lat: 39.4780911, lng: -0.3821706}, // 1844KM
+    {lat: 55.867160, lng: -4.271041},   // 10M
+    {lat: 41.8769913, lng: 12.4835351}, // 1969KM
+    {lat: 55.867055, lng: -4.271041},   // 0M
+  ]
+
+  async function setLocations(main, backup) {
+    await SpecialPowers.setCharPref(
+      GEO_PREF,
+      `${BASE_GEO_URL}?lat=${main.lat}&lng=${main.lng}`
+    );
+    await SpecialPowers.setCharPref(
+      BACKUP_PREF,
+      `${BASE_GEO_URL}?lat=${backup.lat}&lng=${backup.lng}`
+    );
+  }
+
+  async function doTest() {
+    // Not all treeherder builds can collect telemetry.
+    if (!SpecialPowers.Services.telemetry.canRecordPrereleaseData) {
+      ok(true, "Cant run any tests without telemetry");
+      SimpleTest.finish();
+      return;
+    }
+    await sendToParent("clear");
+
+    for (let location of LOCATIONS) {
+      await setLocations(BASE_LOCATION, location);
+      await getCurrentPosition();
+      // Not all requests (NaN) will report telemetry.
+      if (!location.skipWait) {
+        await waitFor(newTelemetryResult, "");
+      }
+    }
+
+    let res = await sendToParent("getTelemetryEvents");
+    let total = Object.values(res.values)
+      .reduce((val, acc) => acc + val, 0);
+
+    is(total, 7, "Should have correct number of results");
+    is(res.values["0"], 2, "Two results were same location");
+    // Telemetry could change how exact bucketing
+    // implementation, so check the low bucket
+    // and that the rest are spead out.
+    is(
+      Object.keys(res.values).length,
+      6,
+      "Split the rest of the results across buckets"
+    );
+
+    SimpleTest.finish();
+  }
+</script>
+</head>
+<body>
+<a target="_blank" href="https://bugzilla.mozilla.org/show_bug.cgi?id=1637402">Mozilla Bug </a>
+<pre id="test"></pre>
+</body>
+</html>
--- a/modules/libpref/init/all.js
+++ b/modules/libpref/init/all.js
@@ -3949,16 +3949,19 @@ pref("network.psl.onUpdate_notify", fals
 
 // All the Geolocation preferences are here.
 //
 #ifndef EARLY_BETA_OR_EARLIER
   pref("geo.provider.network.url", "https://www.googleapis.com/geolocation/v1/geolocate?key=%GOOGLE_LOCATION_SERVICE_API_KEY%");
 #else
   // Use MLS on Nightly and early Beta.
   pref("geo.provider.network.url", "https://location.services.mozilla.com/v1/geolocate?key=%MOZILLA_API_KEY%");
+  // On Nightly and early Beta, make duplicate location services requests
+  // to google so we can compare results.
+  pref("geo.provider.network.compare.url", "https://www.googleapis.com/geolocation/v1/geolocate?key=%GOOGLE_LOCATION_SERVICE_API_KEY%");
 #endif
 
 // Timeout to wait before sending the location request.
 pref("geo.provider.network.timeToWaitBeforeSending", 5000);
 // Timeout for outbound network geolocation provider.
 pref("geo.provider.network.timeout", 60000);
 
 #ifdef XP_MACOSX
--- a/testing/profiles/common/user.js
+++ b/testing/profiles/common/user.js
@@ -62,12 +62,12 @@ user_pref("media.autoplay.blocking_polic
 user_pref("media.autoplay.ask-permission", false);
 user_pref("media.autoplay.block-webaudio", false);
 user_pref("media.allowed-to-play.enabled", true);
 // Ensure media can always play without delay
 user_pref("media.block-autoplay-until-in-foreground", false);
 user_pref("toolkit.telemetry.coverage.endpoint.base", "http://localhost");
 // Don't ask for a request in testing unless explicitly set this as true.
 user_pref("media.geckoview.autoplay.request", false);
-// user_pref("geo.provider.network.url", "http://localhost/geoip-dummy");
+user_pref("geo.provider.network.compare.url", "");
 user_pref("browser.region.network.url", "http://localhost/geoip-dummy");
 // Do not unload tabs on low memory when testing
 user_pref("browser.tabs.unloadOnLowMemory", false);
--- a/testing/profiles/xpcshell/user.js
+++ b/testing/profiles/xpcshell/user.js
@@ -5,16 +5,17 @@ user_pref("app.normandy.api_url", "https
 user_pref("browser.safebrowsing.downloads.remote.url", "https://%(server)s/safebrowsing-dummy");
 user_pref("browser.search.geoSpecificDefaults", false);
 user_pref("extensions.systemAddon.update.url", "http://%(server)s/dummy-system-addons.xml");
 // Treat WebExtension API/schema warnings as errors.
 user_pref("extensions.webextensions.warnings-as-errors", true);
 // Always use network provider for geolocation tests
 // so we bypass the OSX dialog raised by the corelocation provider
 user_pref("geo.provider.testing", true);
+user_pref("geo.provider.network.compare.url", "");
 user_pref("media.gmp-manager.updateEnabled", false);
 user_pref("media.gmp-manager.url.override", "http://%(server)s/dummy-gmp-manager.xml");
 user_pref("toolkit.telemetry.server", "https://%(server)s/telemetry-dummy");
 // Prevent Remote Settings to issue non local connections.
 user_pref("services.settings.server", "http://localhost/remote-settings-dummy/v1");
 // Prevent intermediate preloads to be downloaded on Remote Settings polling.
 user_pref("security.remote_settings.intermediates.enabled", false);
 // The process priority manager only shifts priorities when it has at least
--- a/toolkit/components/telemetry/Histograms.json
+++ b/toolkit/components/telemetry/Histograms.json
@@ -9104,16 +9104,27 @@
   "SEARCH_SERVICE_NONUS_COUNTRY_MISMATCHED_PLATFORM_WIN": {
     "record_in_processes": ["main", "content"],
     "products": ["firefox", "fennec", "geckoview", "thunderbird"],
     "alert_emails": ["mdeboer@mozilla.com", "fx-search@mozilla.com"],
     "expires_in_version": "never",
     "kind": "boolean",
     "description": "If we are on Windows and neither the Windows countryCode nor the geoip countryCode indicates we are in the US, set to false if they both agree on the value or true otherwise"
   },
+  "REGION_LOCATION_SERVICES_DIFFERENCE": {
+    "record_in_processes": ["main", "content"],
+    "products": ["firefox"],
+    "expires_in_version": "84",
+    "kind": "exponential",
+    "n_buckets": 20,
+    "high": 12742000,
+    "bug_numbers": [1637402],
+    "alert_emails": ["dharvey@mozilla.com"],
+    "description": "The distance(m) between the the result of 2 services that detect the users location"
+  },
   "TOUCH_ENABLED_DEVICE": {
     "record_in_processes": ["main"],
     "products": ["firefox", "fennec", "geckoview"],
     "expires_in_version": "never",
     "kind": "boolean",
     "releaseChannelCollection": "opt-out",
     "bug_numbers": [795307, 1390269],
     "alert_emails": ["jimm@mozilla.com"],