Bug 1227956 - Implement Kinto.js OneCRL client r=rnewman
authorMark Goodwin <mgoodwin@mozilla.com>
Tue, 09 Feb 2016 18:51:08 +0000
changeset 283798 56e66f43d7ee4e52da7fc7add39ca86743225cc2
parent 283797 86f89d86b235c8cfdadd77f96c731dcdd3191b05
child 283799 a4f070f356772d4559516cf6f51fe3aa2e39ebaf
push id29991
push usercbook@mozilla.com
push dateThu, 11 Feb 2016 10:52:20 +0000
treeherdermozilla-central@d4d72e7b30da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersrnewman
bugs1227956
milestone47.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1227956 - Implement Kinto.js OneCRL client r=rnewman
browser/app/profile/firefox.js
mobile/android/app/mobile.js
services/common/KintoCertificateBlocklist.js
services/common/moz.build
services/common/tests/unit/test_kintoCertBlocklist.js
services/common/tests/unit/xpcshell.ini
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -57,16 +57,22 @@ pref("extensions.blocklist.enabled", tru
 pref("extensions.blocklist.interval", 86400);
 // Controls what level the blocklist switches from warning about items to forcibly
 // blocking them.
 pref("extensions.blocklist.level", 2);
 pref("extensions.blocklist.url", "https://blocklist.addons.mozilla.org/blocklist/3/%APP_ID%/%APP_VERSION%/%PRODUCT%/%BUILD_ID%/%BUILD_TARGET%/%LOCALE%/%CHANNEL%/%OS_VERSION%/%DISTRIBUTION%/%DISTRIBUTION_VERSION%/%PING_COUNT%/%TOTAL_PING_COUNT%/%DAYS_SINCE_LAST_PING%/");
 pref("extensions.blocklist.detailsURL", "https://www.mozilla.org/%LOCALE%/blocklist/");
 pref("extensions.blocklist.itemURL", "https://blocklist.addons.mozilla.org/%LOCALE%/%APP%/blocked/%blockID%");
 
+// Kinto blocklist preferences
+pref("services.kinto.base", "https://firefox.settings.services.mozilla.com/v1");
+pref("services.kinto.bucket", "blocklists");
+pref("services.kinto.onecrl.collection", "certificates");
+pref("services.kinto.onecrl.checked", 0);
+
 pref("extensions.update.autoUpdateDefault", true);
 
 pref("extensions.hotfix.id", "firefox-hotfix@mozilla.org");
 pref("extensions.hotfix.cert.checkAttributes", true);
 pref("extensions.hotfix.certs.1.sha1Fingerprint", "91:53:98:0C:C1:86:DF:47:8F:35:22:9E:11:C9:A7:31:04:49:A1:AA");
 pref("extensions.hotfix.certs.2.sha1Fingerprint", "39:E7:2B:7A:5B:CF:37:78:F9:5D:4A:E0:53:2D:2F:3D:68:53:C5:60");
 
 // Check AUS for system add-on updates.
--- a/mobile/android/app/mobile.js
+++ b/mobile/android/app/mobile.js
@@ -233,16 +233,22 @@ pref("extensions.compatability.locales.b
 /* blocklist preferences */
 pref("extensions.blocklist.enabled", true);
 // OneCRL freshness checking depends on this value, so if you change it,
 // please also update security.onecrl.maximum_staleness_in_seconds.
 pref("extensions.blocklist.interval", 86400);
 pref("extensions.blocklist.url", "https://blocklist.addons.mozilla.org/blocklist/3/%APP_ID%/%APP_VERSION%/%PRODUCT%/%BUILD_ID%/%BUILD_TARGET%/%LOCALE%/%CHANNEL%/%OS_VERSION%/%DISTRIBUTION%/%DISTRIBUTION_VERSION%/%PING_COUNT%/%TOTAL_PING_COUNT%/%DAYS_SINCE_LAST_PING%/");
 pref("extensions.blocklist.detailsURL", "https://www.mozilla.com/%LOCALE%/blocklist/");
 
+// Kinto blocklist preferences
+pref("services.kinto.base", "https://firefox.settings.services.mozilla.com/v1");
+pref("services.kinto.bucket", "blocklists");
+pref("services.kinto.onecrl.collection", "certificates");
+pref("services.kinto.onecrl.checked", 0);
+
 /* Don't let XPIProvider install distribution add-ons; we do our own thing on mobile. */
 pref("extensions.installDistroAddons", false);
 
 /* block popups by default, and notify the user about blocked popups */
 pref("dom.disable_open_during_load", true);
 pref("privacy.popups.showBrowserMessage", true);
 
 /* disable opening windows with the dialog feature */
new file mode 100644
--- /dev/null
+++ b/services/common/KintoCertificateBlocklist.js
@@ -0,0 +1,115 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+this.EXPORTED_SYMBOLS = ["OneCRLClient"];
+
+const { classes: Cc, interfaces: Ci, utils: Cu } = Components;
+
+Cu.import("resource://services-common/moz-kinto-client.js");
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/Task.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+XPCOMUtils.defineLazyServiceGetter(this, "uuidgen",
+                                   "@mozilla.org/uuid-generator;1",
+                                   "nsIUUIDGenerator");
+
+const PREF_KINTO_BASE = "services.kinto.base";
+const PREF_KINTO_BUCKET = "services.kinto.bucket";
+const PREF_KINTO_ONECRL_COLLECTION = "services.kinto.onecrl.collection";
+const PREF_KINTO_ONECRL_CHECKED_SECONDS = "services.kinto.onecrl.checked";
+
+const RE_UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+// Kinto.js assumes version 4 UUIDs but allows you to specify custom
+// validators and generators. The tooling that generates records in the
+// certificates collection currently uses a version 1 UUID so we must
+// specify a validator that's less strict. We must also supply a generator
+// since Kinto.js does not allow one without the other.
+function makeIDSchema() {
+  return {
+    validate: RE_UUID.test.bind(RE_UUID),
+    generate: function() {
+      return uuidgen.generateUUID().toString();
+    }
+  };
+}
+
+// A Kinto based client to keep the OneCRL certificate blocklist up to date.
+function CertBlocklistClient() {
+  // maybe sync the collection of certificates with remote data.
+  // lastModified - the lastModified date (on the server, milliseconds since
+  // epoch) of data in the remote collection
+  // serverTime - the time on the server (milliseconds since epoch)
+  // returns a promise which rejects on sync failure
+  this.maybeSync = function(lastModified, serverTime) {
+    let base = Services.prefs.getCharPref(PREF_KINTO_BASE);
+    let bucket = Services.prefs.getCharPref(PREF_KINTO_BUCKET);
+
+    let Kinto = loadKinto();
+
+    let FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+
+
+    let certList = Cc["@mozilla.org/security/certblocklist;1"]
+                     .getService(Ci.nsICertBlocklist);
+
+    // Future blocklist clients can extract the sync-if-stale logic. For
+    // now, since this is currently the only client, we'll do this here.
+    let config = {
+      remote: base,
+      bucket: bucket,
+      adapter: FirefoxAdapter,
+    };
+
+    let db = new Kinto(config);
+    let collectionName = Services.prefs.getCharPref(PREF_KINTO_ONECRL_COLLECTION,
+                                                    "certificates");
+    let blocklist = db.collection(collectionName,
+                                  { idSchema: makeIDSchema() });
+
+    let updateLastCheck = function() {
+      let checkedServerTimeInSeconds = Math.round(serverTime / 1000);
+      Services.prefs.setIntPref(PREF_KINTO_ONECRL_CHECKED_SECONDS,
+                                checkedServerTimeInSeconds);
+    }
+
+    return Task.spawn(function* () {
+      try {
+        yield blocklist.db.open();
+        let collectionLastModified = yield blocklist.db.getLastModified();
+        // if the data is up to date, there's no need to sync. We still need
+        // to record the fact that a check happened.
+        if (lastModified <= collectionLastModified) {
+          updateLastCheck();
+          return;
+        }
+        yield blocklist.sync();
+        let list = yield blocklist.list();
+        for (let item of list.data) {
+          if (item.issuerName && item.serialNumber) {
+            certList.revokeCertByIssuerAndSerial(item.issuerName,
+                                                 item.serialNumber);
+          } else if (item.subject && item.pubKeyHash) {
+            certList.revokeCertBySubjectAndPubKey(item.subject,
+                                                  item.pubKeyHash);
+          } else {
+            throw new Error("Cert blocklist record has incomplete data");
+          }
+        }
+        // We explicitly do not want to save entries or update the
+        // last-checked time if sync fails
+        certList.saveEntries();
+        updateLastCheck();
+      } finally {
+        blocklist.db.close()
+      }
+    });
+  }
+}
+
+this.OneCRLClient = new CertBlocklistClient();
--- a/services/common/moz.build
+++ b/services/common/moz.build
@@ -10,16 +10,17 @@ with Files('**'):
 TEST_DIRS += ['tests']
 
 EXTRA_COMPONENTS += [
     'servicesComponents.manifest',
 ]
 
 EXTRA_JS_MODULES['services-common'] += [
     'async.js',
+    'KintoCertificateBlocklist.js',
     'logmanager.js',
     'moz-kinto-client.js',
     'observers.js',
     'rest.js',
     'stringbundle.js',
     'utils.js',
 ]
 
new file mode 100644
--- /dev/null
+++ b/services/common/tests/unit/test_kintoCertBlocklist.js
@@ -0,0 +1,189 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+const { Constructor: CC } = Components;
+
+Cu.import("resource://services-common/KintoCertificateBlocklist.js");
+Cu.import("resource://services-common/moz-kinto-client.js")
+Cu.import("resource://testing-common/httpd.js");
+
+const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+  "nsIBinaryInputStream", "setInputStream");
+
+var server;
+
+// set up what we need to make storage adapters
+const Kinto = loadKinto();
+const FirefoxAdapter = Kinto.adapters.FirefoxAdapter;
+const kintoFilename = "kinto.sqlite";
+
+let kintoClient;
+
+function do_get_kinto_collection(collectionName) {
+  if (!kintoClient) {
+    let config = {
+      // Set the remote to be some server that will cause test failure when
+      // hit since we should never hit the server directly, only via maybeSync()
+      remote: "https://firefox.settings.services.mozilla.com/v1/",
+      // Set up the adapter and bucket as normal
+      adapter: FirefoxAdapter,
+      bucket: "blocklists"
+    };
+    kintoClient = new Kinto(config);
+  }
+  return kintoClient.collection(collectionName);
+}
+
+// Some simple tests to demonstrate that the logic inside maybeSync works
+// correctly and that simple kinto operations are working as expected. There
+// are more tests for core Kinto.js (and its storage adapter) in the
+// xpcshell tests under /services/common
+add_task(function* test_something(){
+  const configPath = "/v1/";
+  const recordsPath = "/v1/buckets/blocklists/collections/certificates/records";
+
+  Services.prefs.setCharPref("services.kinto.base",
+                             `http://localhost:${server.identity.primaryPort}/v1`);
+
+  // register a handler
+  function handleResponse (request, response) {
+    try {
+      const sampled = getSampleResponse(request, server.identity.primaryPort);
+      if (!sampled) {
+        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+      }
+
+      response.setStatusLine(null, sampled.status.status,
+                             sampled.status.statusText);
+      // send the headers
+      for (let headerLine of sampled.sampleHeaders) {
+        let headerElements = headerLine.split(':');
+        response.setHeader(headerElements[0], headerElements[1].trimLeft());
+      }
+      response.setHeader("Date", (new Date()).toUTCString());
+
+      response.write(sampled.responseBody);
+    } catch (e) {
+      dump(`${e}\n`);
+    }
+  }
+  server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(recordsPath, handleResponse);
+
+  // Test an empty db populates
+  let result = yield OneCRLClient.maybeSync(2000, Date.now());
+
+  // Open the collection, verify it's been populated:
+  // Our test data has a single record; it should be in the local collection
+  let collection = do_get_kinto_collection("certificates");
+  yield collection.db.open();
+  let list = yield collection.list();
+  do_check_eq(list.data.length, 1);
+  yield collection.db.close();
+
+  // Test the db is updated when we call again with a later lastModified value
+  result = yield OneCRLClient.maybeSync(4000, Date.now());
+
+  // Open the collection, verify it's been updated:
+  // Our test data now has two records; both should be in the local collection
+  collection = do_get_kinto_collection("certificates");
+  yield collection.db.open();
+  list = yield collection.list();
+  do_check_eq(list.data.length, 3);
+  yield collection.db.close();
+
+  // Try to maybeSync with the current lastModified value - no connection
+  // should be attempted.
+  // Clear the kinto base pref so any connections will cause a test failure
+  Services.prefs.clearUserPref("services.kinto.base");
+  yield OneCRLClient.maybeSync(4000, Date.now());
+
+  // Try again with a lastModified value at some point in the past
+  yield OneCRLClient.maybeSync(3000, Date.now());
+
+  // Check the OneCRL check time pref is modified, even if the collection
+  // hasn't changed
+  Services.prefs.setIntPref("services.kinto.onecrl.checked", 0);
+  yield OneCRLClient.maybeSync(3000, Date.now());
+  let newValue = Services.prefs.getIntPref("services.kinto.onecrl.checked");
+  do_check_neq(newValue, 0);
+});
+
+function run_test() {
+  // Set up an HTTP Server
+  server = new HttpServer();
+  server.start(-1);
+
+  run_next_test();
+
+  do_register_cleanup(function() {
+    server.stop(function() { });
+  });
+}
+
+// get a response for a given request from sample data
+function getSampleResponse(req, port) {
+  const responses = {
+    "OPTIONS": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
+        "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
+        "Access-Control-Allow-Origin: *",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": "null"
+    },
+    "GET:/v1/?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress"
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"settings":{"cliquet.batch_max_requests":25}, "url":`http://localhost:${port}/v1/`, "documentation":"https://kinto.readthedocs.org/", "version":"1.5.1", "commit":"cbc6f58", "hello":"kinto"})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"3000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName": "MEQxCzAJBgNVBAYTAlVTMRUwEwYDVQQKEwx0aGF3dGUsIEluYy4xHjAcBgNVBAMTFXRoYXd0ZSBFViBTU0wgQ0EgLSBHMw==",
+        "serialNumber":"CrTHPEE6AZSfI3jysin2bA==",
+        "id":"78cf8900-fdea-4ce5-f8fb-b78710617718",
+        "last_modified":3000
+      }]})
+    },
+    "GET:/v1/buckets/blocklists/collections/certificates/records?_since=3000": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"4000\""
+      ],
+      "status": {status: 200, statusText: "OK"},
+      "responseBody": JSON.stringify({"data":[{
+        "issuerName":"MFkxCzAJBgNVBAYTAk5MMR4wHAYDVQQKExVTdGFhdCBkZXIgTmVkZXJsYW5kZW4xKjAoBgNVBAMTIVN0YWF0IGRlciBOZWRlcmxhbmRlbiBPdmVyaGVpZCBDQQ",
+        "serialNumber":"ATFpsA==",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02c",
+        "last_modified":4000
+      },{
+        "subject":"MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
+        "pubKeyHash":"VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
+        "id":"dabafde9-df4a-ddba-2548-748da04cc02d",
+        "last_modified":4000
+      }]})
+    }
+  };
+  return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[req.method];
+
+}
--- a/services/common/tests/unit/xpcshell.ini
+++ b/services/common/tests/unit/xpcshell.ini
@@ -5,16 +5,17 @@ firefox-appdir = browser
 skip-if = toolkit == 'gonk'
 support-files =
   test_storage_adapter/**
 
 # Test load modules first so syntax failures are caught early.
 [test_load_modules.js]
 
 [test_kinto.js]
+[test_kintoCertBlocklist.js]
 [test_storage_adapter.js]
 
 [test_utils_atob.js]
 [test_utils_convert_string.js]
 [test_utils_dateprefs.js]
 [test_utils_deepCopy.js]
 [test_utils_encodeBase32.js]
 [test_utils_encodeBase64URL.js]