Bug 1453692 - Add RemoteSettings inspect() method r=Gijs
authorMathieu Leplatre <mathieu@mozilla.com>
Fri, 08 Jun 2018 17:08:33 +0200
changeset 479624 693ff336db30ae54237b1b6c64ac23b4e3d0b9af
parent 479623 0ad07da6e0db35808ce3fbc66bdc704144f5593b
child 479625 4432fb67de2d1e2658a36fed800979c1968c1f94
push id1757
push userffxbld-merge
push dateFri, 24 Aug 2018 17:02:43 +0000
treeherdermozilla-release@736023aebdb1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersGijs
bugs1453692
milestone62.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1453692 - Add RemoteSettings inspect() method r=Gijs MozReview-Commit-ID: FRHvXnGzBq0
services/common/tests/unit/test_blocklist_clients.js
services/settings/remote-settings.js
services/settings/test/unit/test_remote_settings.js
--- a/services/common/tests/unit/test_blocklist_clients.js
+++ b/services/common/tests/unit/test_blocklist_clients.js
@@ -1,16 +1,17 @@
 const { Constructor: CC } = Components;
 
 ChromeUtils.import("resource://gre/modules/AppConstants.jsm");
 ChromeUtils.import("resource://gre/modules/Services.jsm");
 ChromeUtils.import("resource://testing-common/httpd.js");
 const { FileUtils } = ChromeUtils.import("resource://gre/modules/FileUtils.jsm", {});
 const { OS } = ChromeUtils.import("resource://gre/modules/osfile.jsm", {});
 
+const { RemoteSettings } = ChromeUtils.import("resource://services-settings/remote-settings.js", {});
 const BlocklistClients = ChromeUtils.import("resource://services-common/blocklist-clients.js", {});
 
 const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
   "nsIBinaryInputStream", "setInputStream");
 
 const IS_ANDROID = AppConstants.platform == "android";
 
 
@@ -80,20 +81,22 @@ function run_test() {
 
       response.write(responseBody);
       response.finish();
     } catch (e) {
       info(e);
     }
   }
   const configPath = "/v1/";
+  const monitorChangesPath = "/v1/buckets/monitor/collections/changes/records";
   const addonsRecordsPath  = "/v1/buckets/blocklists/collections/addons/records";
   const gfxRecordsPath     = "/v1/buckets/blocklists/collections/gfx/records";
   const pluginsRecordsPath = "/v1/buckets/blocklists/collections/plugins/records";
   server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(monitorChangesPath, handleResponse);
   server.registerPathHandler(addonsRecordsPath, handleResponse);
   server.registerPathHandler(gfxRecordsPath, handleResponse);
   server.registerPathHandler(pluginsRecordsPath, handleResponse);
 
 
   run_next_test();
 
   registerCleanupFunction(function() {
@@ -285,16 +288,56 @@ add_task(async function test_entries_are
     await collection.db.saveLastModified(42); // Prevent from loading JSON dump.
     const list = await client.get();
     equal(list.length, 4);
     ok(list.every(e => e.willMatch));
   }
 });
 add_task(clear_state);
 
+add_task(async function test_inspect_with_blocklist_clients() {
+  const rsSigner = "remote-settings.content-signature.mozilla.org";
+  const {
+    serverTimestamp,
+    localTimestamp,
+    lastCheck,
+    collections
+  } = await RemoteSettings.inspect();
+
+  equal(serverTimestamp, '"3000"');
+  equal(localTimestamp, null); // not yet synchronized.
+  equal(lastCheck, 0); // not yet synchronized.
+
+  const defaults = {
+    bucket: "blocklists",
+    localTimestamp: null,
+    lastCheck: 0,
+    signerName: rsSigner,
+  };
+  equal(collections.length, 3);
+  deepEqual(collections[0], { ...defaults, collection: "gfx", serverTimestamp: 3000 });
+  deepEqual(collections[1], { ...defaults, collection: "addons", serverTimestamp: 2900 });
+  deepEqual(collections[2], { ...defaults, collection: "plugins", serverTimestamp: 2800 });
+
+  // Now synchronize, and check that values were updated.
+  Services.prefs.setBoolPref("services.settings.load_dump", false);
+  const currentTime = Math.floor(Date.now() / 1000);
+  await RemoteSettings.pollChanges();
+
+  const inspected = await RemoteSettings.inspect();
+
+  equal(inspected.localTimestamp, '"3000"');
+  ok(inspected.lastCheck >= currentTime, `last check ${inspected.lastCheck}`);
+  for (const c of inspected.collections) {
+    equal(c.localTimestamp, 3000);
+    ok(c.lastCheck >= currentTime, `${c.collectionName} last check ${c.lastCheck}`);
+  }
+});
+add_task(clear_state);
+
 
 // get a response for a given request from sample data
 function getSampleResponse(req, port) {
   const responses = {
     "OPTIONS": {
       "sampleHeaders": [
         "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
         "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
@@ -319,16 +362,55 @@ function getSampleResponse(req, port) {
         },
         "url": `http://localhost:${port}/v1/`,
         "documentation": "https://kinto.readthedocs.org/",
         "version": "1.5.1",
         "commit": "cbc6f58",
         "hello": "kinto"
       })
     },
+    "GET:/v1/buckets/monitor/collections/changes/records": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        `Date: ${new Date().toUTCString()}`,
+        "Etag: \"3000\""
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": [{
+          "id": "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
+          "bucket": "blocklists",
+          "collection": "gfx",
+          "last_modified": 3000
+        }, {
+          "id": "36b2ebab-d691-4796-b36b-f7a06df38b26",
+          "bucket": "blocklists",
+          "collection": "addons",
+          "last_modified": 2900
+        }, {
+          "id": "42aea14b-4b35-4308-94d9-8562412a2fef",
+          "bucket": "blocklists",
+          "collection": "plugins",
+          "last_modified": 2800
+        }, {
+          "id": "50f4ef31-7788-4be8-b073-114440be4d8d",
+          "bucket": "main",
+          "collection": "passwords",
+          "last_modified": 2700
+        }, {
+          "id": "d2f08123-b815-4bbf-a0b7-a948a65ecafa",
+          "bucket": "pinning-preview",
+          "collection": "pins",
+          "last_modified": 2600
+        }]
+      })
+    },
     "GET:/v1/buckets/blocklists/collections/addons/records?_sort=-last_modified": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         "Etag: \"3000\""
       ],
--- a/services/settings/remote-settings.js
+++ b/services/settings/remote-settings.js
@@ -596,61 +596,111 @@ async function hasLocalDump(bucket, coll
 
 function remoteSettingsFunction() {
   const _clients = new Map();
 
   // If not explicitly specified, use the default bucket name and signer.
   const mainBucket = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_BUCKET);
   const defaultSigner = Services.prefs.getCharPref(PREF_SETTINGS_DEFAULT_SIGNER);
 
+  /**
+   * RemoteSettings constructor.
+   *
+   * @param {String} collectionName The remote settings identifier
+   * @param {Object} options Advanced options
+   * @returns {RemoteSettingsClient} An instance of a Remote Settings client.
+   */
   const remoteSettings = function(collectionName, options) {
     // Get or instantiate a remote settings client.
     const rsOptions = {
       bucketName: mainBucket,
       signerName: defaultSigner,
       ...options
     };
     const { bucketName } = rsOptions;
     const key = `${bucketName}/${collectionName}`;
     if (!_clients.has(key)) {
       const c = new RemoteSettingsClient(collectionName, rsOptions);
       _clients.set(key, c);
     }
     return _clients.get(key);
   };
 
-  // This is called by the ping mechanism.
-  // returns a promise that rejects if something goes wrong
+  Object.defineProperty(remoteSettings, "pollingEndpoint", {
+    get() {
+      const kintoServer = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
+      const changesPath = Services.prefs.getCharPref(PREF_SETTINGS_CHANGES_PATH);
+      return kintoServer + changesPath;
+    }
+  });
+
+  /**
+   * Internal helper to retrieve existing instances of clients or new instances
+   * with default options if possible, or `null` if bucket/collection are unknown.
+   */
+  async function _client(bucketName, collectionName) {
+    // Check if a client was registered for this bucket/collection. Potentially
+    // with some specific options like signer, filter function etc.
+    const key = `${bucketName}/${collectionName}`;
+    const client = _clients.get(key);
+    if (client) {
+      // If the bucket name was changed manually on the client instance and does not
+      // match, don't return it.
+      if (client.bucketName == bucketName) {
+        return client;
+      }
+
+    // There was no client registered for this bucket/collection, but it's the main bucket,
+    // therefore we can instantiate a client with the default options.
+    // So if we have a local database or if we ship a JSON dump, then it means that
+    // this client is known but it was not registered yet (eg. calling module not "imported" yet).
+    } else if (bucketName == mainBucket) {
+      const [dbExists, localDump] = await Promise.all([
+        databaseExists(bucketName, collectionName),
+        hasLocalDump(bucketName, collectionName)
+      ]);
+      if (dbExists || localDump) {
+        return new RemoteSettingsClient(collectionName, { bucketName, signerName: defaultSigner });
+      }
+    }
+    // Else, we cannot return a client insttance because we are not able to synchronize data in specific buckets.
+    // Mainly because we cannot guess which `signerName` has to be used for example.
+    // And we don't want to synchronize data for collections in the main bucket that are
+    // completely unknown (ie. no database and no JSON dump).
+    return null;
+  }
+
+  /**
+   * Main polling method, called by the ping mechanism.
+   *
+   * @returns {Promise} or throws error if something goes wrong.
+   */
   remoteSettings.pollChanges = async () => {
     // Check if the server backoff time is elapsed.
     if (Services.prefs.prefHasUserValue(PREF_SETTINGS_SERVER_BACKOFF)) {
       const backoffReleaseTime = Services.prefs.getCharPref(PREF_SETTINGS_SERVER_BACKOFF);
       const remainingMilliseconds = parseInt(backoffReleaseTime, 10) - Date.now();
       if (remainingMilliseconds > 0) {
         // Backoff time has not elapsed yet.
         UptakeTelemetry.report(TELEMETRY_HISTOGRAM_KEY,
                                UptakeTelemetry.STATUS.BACKOFF);
         throw new Error(`Server is asking clients to back off; retry in ${Math.ceil(remainingMilliseconds / 1000)}s.`);
       } else {
         Services.prefs.clearUserPref(PREF_SETTINGS_SERVER_BACKOFF);
       }
     }
 
-    // Right now, we only use the collection name and the last modified info
-    const kintoBase = Services.prefs.getCharPref(PREF_SETTINGS_SERVER);
-    const changesEndpoint = kintoBase + Services.prefs.getCharPref(PREF_SETTINGS_CHANGES_PATH);
-
     let lastEtag;
     if (Services.prefs.prefHasUserValue(PREF_SETTINGS_LAST_ETAG)) {
       lastEtag = Services.prefs.getCharPref(PREF_SETTINGS_LAST_ETAG);
     }
 
     let pollResult;
     try {
-      pollResult = await fetchLatestChanges(changesEndpoint, lastEtag);
+      pollResult = await fetchLatestChanges(remoteSettings.pollingEndpoint, lastEtag);
     } catch (e) {
       // Report polling error to Uptake Telemetry.
       let report;
       if (/Server/.test(e.message)) {
         report = UptakeTelemetry.STATUS.SERVER_ERROR;
       } else if (/NetworkError/.test(e.message)) {
         report = UptakeTelemetry.STATUS.NETWORK_ERROR;
       } else {
@@ -677,54 +727,31 @@ function remoteSettingsFunction() {
     // Record new update time and the difference between local and server time.
     // Negative clockDifference means local time is behind server time
     // by the absolute of that value in seconds (positive means it's ahead)
     const clockDifference = Math.floor((Date.now() - serverTimeMillis) / 1000);
     Services.prefs.setIntPref(PREF_SETTINGS_CLOCK_SKEW_SECONDS, clockDifference);
     Services.prefs.setIntPref(PREF_SETTINGS_LAST_UPDATE, serverTimeMillis / 1000);
 
     const loadDump = Services.prefs.getBoolPref(PREF_SETTINGS_LOAD_DUMP, true);
+
     // Iterate through the collections version info and initiate a synchronization
     // on the related remote settings client.
     let firstError;
     for (const change of changes) {
-      const {bucket: bucketName, collection, last_modified: lastModified} = change;
-      const key = `${bucketName}/${collection}`;
-
-      let client;
-      // Check if a client was registered for this bucket/collection. Potentially
-      // with some specific options like bucket, signer, etc.
-      if (_clients.has(key)) {
-        client = _clients.get(key);
-        // If the bucket name was changed manually on the client instance and does not
-        // match, it should be skipped.
-        if (client.bucketName != bucketName) {
-          continue;
-        }
+      const { bucket, collection, last_modified } = change;
 
-      // There was no client registered for this bucket/collection, but it's the main bucket,
-      // therefore we can instantiate a client with the default options.
-      // So if we have a local database or if we ship a JSON dump, then it means that
-      // this client is known but it was not registered yet (eg. calling module not "imported" yet).
-      } else if (bucketName == mainBucket && (await databaseExists(bucketName, collection) ||
-                                              await hasLocalDump(bucketName, collection))) {
-        client = new RemoteSettingsClient(collection, {bucketName, signerName: defaultSigner});
-
-      // We are not able to synchronize data for clients in specific buckets since we cannot
-      // guess which `signerName` has to be used for example.
-      // And we don't want to synchronize data for collections in the main bucket that are
-      // completely unknown (ie. no database and no JSON dump).
-      } else {
+      const client = await _client(bucket, collection);
+      if (!client) {
         continue;
       }
-
       // Start synchronization! It will be a no-op if the specified `lastModified` equals
       // the one in the local database.
       try {
-        await client.maybeSync(lastModified, serverTimeMillis, {loadDump});
+        await client.maybeSync(last_modified, serverTimeMillis, {loadDump});
       } catch (e) {
         if (!firstError) {
           firstError = e;
           firstError.details = change;
         }
       }
     }
     if (firstError) {
@@ -735,30 +762,66 @@ function remoteSettingsFunction() {
     // Save current Etag for next poll.
     if (currentEtag) {
       Services.prefs.setCharPref(PREF_SETTINGS_LAST_ETAG, currentEtag);
     }
 
     Services.obs.notifyObservers(null, "remote-settings-changes-polled");
   };
 
+  /**
+   * Returns an object with polling status information and the list of
+   * known remote settings collections.
+   */
+  remoteSettings.inspect = async () => {
+    const { changes, currentEtag: serverTimestamp } = await fetchLatestChanges(remoteSettings.pollingEndpoint);
+
+    const collections = await Promise.all(changes.map(async (change) => {
+      const { bucket, collection, last_modified: serverTimestamp } = change;
+      const client = await _client(bucket, collection);
+      if (!client) {
+        return null;
+      }
+      const kintoCol = await client.openCollection();
+      const localTimestamp = await kintoCol.db.getLastModified();
+      const lastCheck = Services.prefs.getIntPref(client.lastCheckTimePref, 0);
+      return {
+        bucket,
+        collection,
+        localTimestamp,
+        serverTimestamp,
+        lastCheck,
+        signerName: client.signerName
+      };
+    }));
+
+    return {
+      serverURL: Services.prefs.getCharPref(PREF_SETTINGS_SERVER),
+      serverTimestamp,
+      localTimestamp: Services.prefs.getCharPref(PREF_SETTINGS_LAST_ETAG, null),
+      lastCheck: Services.prefs.getIntPref(PREF_SETTINGS_LAST_UPDATE, 0),
+      mainBucket,
+      defaultSigner,
+      collections: collections.filter(c => !!c)
+    };
+  };
+
 
   const broadcastID = "remote-settings/monitor_changes";
   // When we start on a new profile there will be no ETag stored.
   // Use an arbitrary ETag that is guaranteed not to occur.
   // This will trigger a broadcast message but that's fine because we
   // will check the changes on each collection and retrieve only the
   // changes (e.g. nothing if we have a dump with the same data).
   const currentVersion = Services.prefs.getStringPref(PREF_SETTINGS_LAST_ETAG, "\"0\"");
   const moduleInfo = {
     moduleURI: __URI__,
     symbolName: "remoteSettingsBroadcastHandler",
   };
-  pushBroadcastService.addListener(broadcastID, currentVersion,
-                                   moduleInfo);
+  pushBroadcastService.addListener(broadcastID, currentVersion, moduleInfo);
 
   return remoteSettings;
 }
 
 var RemoteSettings = remoteSettingsFunction();
 
 var remoteSettingsBroadcastHandler = {
   async receivedBroadcastMessage(data, broadcastID) {
--- a/services/settings/test/unit/test_remote_settings.js
+++ b/services/settings/test/unit/test_remote_settings.js
@@ -56,18 +56,20 @@ function run_test() {
 
       response.write(JSON.stringify(sample.responseBody));
       response.finish();
     } catch (e) {
       info(e);
     }
   }
   const configPath = "/v1/";
+  const changesPath = "/v1/buckets/monitor/collections/changes/records";
   const recordsPath  = "/v1/buckets/main/collections/password-fields/records";
   server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(changesPath, handleResponse);
   server.registerPathHandler(recordsPath, handleResponse);
 
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(() => { });
   });
 }
@@ -140,16 +142,39 @@ add_task(async function test_sync_event_
   equal(eventData.current.length, 1);
   equal(eventData.created.length, 0);
   equal(eventData.updated.length, 0);
   equal(eventData.deleted.length, 1);
   equal(eventData.deleted[0].website, "https://www.other.org/signin");
 });
 add_task(clear_state);
 
+add_task(async function test_inspect_method() {
+  const serverTime = Date.now();
+
+  // Synchronize the `password-fields` collection.
+  await client.maybeSync(Infinity, serverTime);
+
+  const inspected = await RemoteSettings.inspect();
+
+  const { mainBucket, serverURL, defaultSigner, collections } = inspected;
+  const rsSigner = "remote-settings.content-signature.mozilla.org";
+  equal(mainBucket, "main");
+  equal(serverURL, `http://localhost:${server.identity.primaryPort}/v1`);
+  equal(defaultSigner, rsSigner);
+
+  equal(inspected.serverTimestamp, '"4000"');
+  equal(collections.length, 1);
+  // password-fields was synchronized and has local data.
+  equal(collections[0].collection, "password-fields");
+  equal(collections[0].serverTimestamp, 3000);
+  equal(collections[0].localTimestamp, 3000);
+});
+add_task(clear_state);
+
 add_task(async function test_all_listeners_are_executed_if_one_fails() {
   const serverTime = Date.now();
 
   let count = 0;
   client.on("sync", () => { count += 1; });
   client.on("sync", () => { throw new Error("boom"); });
   client.on("sync", () => { count += 2; });
 
@@ -270,16 +295,40 @@ function getSampleResponse(req, port) {
         },
         "url": `http://localhost:${port}/v1/`,
         "documentation": "https://kinto.readthedocs.org/",
         "version": "1.5.1",
         "commit": "cbc6f58",
         "hello": "kinto"
       }
     },
+    "GET:/v1/buckets/monitor/collections/changes/records": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        `Date: ${new Date().toUTCString()}`,
+        "Etag: \"4000\""
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": {
+        "data": [{
+          "id": "4676f0c7-9757-4796-a0e8-b40a5a37a9c9",
+          "bucket": "main",
+          "collection": "unknown",
+          "last_modified": 4000
+        }, {
+          "id": "0af8da0b-3e03-48fb-8d0d-2d8e4cb7514d",
+          "bucket": "main",
+          "collection": "password-fields",
+          "last_modified": 3000
+        }]
+      }
+    },
     "GET:/v1/buckets/main/collections/password-fields/records?_sort=-last_modified": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         "Etag: \"3000\""
       ],