Bug 1552199 - Include records from dump in Remote Settings sync event created data r=glasserc
☠☠ backed out by 4a73759aafa6 ☠ ☠
authorMathieu Leplatre <mathieu@mozilla.com>
Mon, 20 May 2019 14:34:10 +0000
changeset 474531 6519e35004283dc67e83ce49090480c724fac5bf
parent 474530 10cac54d1bb994f1a54c316faa7857f0fb22ad93
child 474532 82b9eaa4679754eb3ff38e6472a3d9f77600affa
push id36042
push userdvarga@mozilla.com
push dateTue, 21 May 2019 04:19:40 +0000
treeherdermozilla-central@ca560ff55451 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglasserc
bugs1552199
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1552199 - Include records from dump in Remote Settings sync event created data r=glasserc Differential Revision: https://phabricator.services.mozilla.com/D31599
services/settings/RemoteSettingsClient.jsm
services/settings/test/unit/test_remote_settings.js
--- a/services/settings/RemoteSettingsClient.jsm
+++ b/services/settings/RemoteSettingsClient.jsm
@@ -277,30 +277,35 @@ class RemoteSettingsClient extends Event
    * @param {Object} options           additional advanced options.
    * @param {bool}   options.loadDump  load initial dump from disk on first sync (default: true)
    * @param {string} options.trigger   label to identify what triggered this sync (eg. ``"timer"``, default: `"manual"`)
    * @return {Promise}                 which rejects on sync or process failure.
    */
   async maybeSync(expectedTimestamp, options = {}) {
     const { loadDump = true, trigger = "manual" } = options;
 
+    let importedFromDump = [];
     const startedAt = new Date();
     let reportStatus = null;
     try {
       // Synchronize remote data into a local DB using Kinto.
       const kintoCollection = await this.openCollection();
       let collectionLastModified = await kintoCollection.db.getLastModified();
 
       // If there is no data currently in the collection, attempt to import
       // initial data from the application defaults.
       // This allows to avoid synchronizing the whole collection content on
       // cold start.
       if (!collectionLastModified && loadDump) {
         try {
-          await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
+          const imported = await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
+          // The worker only returns an integer. List the imported records to build the sync event.
+          if (imported > 0) {
+            ({ data: importedFromDump } = await kintoCollection.list());
+          }
           collectionLastModified = await kintoCollection.db.getLastModified();
         } catch (e) {
           // Report but go-on.
           Cu.reportError(e);
         }
       }
 
       // If the data is up to date, there's no need to sync. We still need
@@ -332,16 +337,19 @@ class RemoteSettingsClient extends Event
       try {
         // Fetch changes from server, and make sure we overwrite local data.
         const strategy = Kinto.syncStrategy.SERVER_WINS;
         syncResult = await kintoCollection.sync({ remote: gServerURL, strategy, expectedTimestamp });
         if (!syncResult.ok) {
           // With SERVER_WINS, there cannot be any conflicts, but don't silent it anyway.
           throw new Error("Synced failed");
         }
+        // The records imported from the dump should be considered as "created" for the
+        // listeners.
+        syncResult.created = importedFromDump.concat(syncResult.created);
       } catch (e) {
         if (e instanceof RemoteSettingsClient.InvalidSignatureError) {
           // Signature verification failed during synchronization.
           reportStatus = UptakeTelemetry.STATUS.SIGNATURE_ERROR;
           // If sync fails with a signature error, it's likely that our
           // local data has been modified in some way.
           // We will attempt to fix this by retrieving the whole
           // remote collection.
--- a/services/settings/test/unit/test_remote_settings.js
+++ b/services/settings/test/unit/test_remote_settings.js
@@ -46,49 +46,22 @@ function run_test() {
                              `http://localhost:${server.identity.primaryPort}/v1`);
 
   client = RemoteSettings("password-fields");
   client.verifySignature = false;
 
   clientWithDump = RemoteSettings("language-dictionaries");
   clientWithDump.verifySignature = false;
 
-  // Setup server fake responses.
-  function handleResponse(request, response) {
-    try {
-      const sample = getSampleResponse(request, server.identity.primaryPort);
-      if (!sample) {
-        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
-      }
-
-      response.setStatusLine(null, sample.status.status,
-                             sample.status.statusText);
-      // send the headers
-      for (let headerLine of sample.sampleHeaders) {
-        let headerElements = headerLine.split(":");
-        response.setHeader(headerElements[0], headerElements[1].trimLeft());
-      }
-      response.setHeader("Date", (new Date()).toUTCString());
-
-      const body = typeof sample.responseBody == "string" ? sample.responseBody
-                                                          : JSON.stringify(sample.responseBody);
-      response.write(body);
-      response.finish();
-    } catch (e) {
-      info(e);
-    }
-  }
-  const configPath = "/v1/";
-  const changesPath = "/v1/buckets/monitor/collections/changes/records";
-  const metadataPath = "/v1/buckets/main/collections/password-fields";
-  const recordsPath  = "/v1/buckets/main/collections/password-fields/records";
-  server.registerPathHandler(configPath, handleResponse);
-  server.registerPathHandler(changesPath, handleResponse);
-  server.registerPathHandler(metadataPath, handleResponse);
-  server.registerPathHandler(recordsPath, handleResponse);
+  server.registerPathHandler("/v1/", handleResponse);
+  server.registerPathHandler("/v1/buckets/monitor/collections/changes/records", handleResponse);
+  server.registerPathHandler("/v1/buckets/main/collections/password-fields", handleResponse);
+  server.registerPathHandler("/v1/buckets/main/collections/password-fields/records", handleResponse);
+  server.registerPathHandler("/v1/buckets/main/collections/language-dictionaries", handleResponse);
+  server.registerPathHandler("/v1/buckets/main/collections/language-dictionaries/records", handleResponse);
   server.registerPathHandler("/fake-x5u", handleResponse);
 
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(() => { });
   });
 }
@@ -99,16 +72,32 @@ add_task(async function test_records_obt
 
   // Open the collection, verify it's been populated:
   // Our test data has a single record; it should be in the local collection
   const list = await client.get();
   equal(list.length, 1);
 });
 add_task(clear_state);
 
+add_task(async function test_records_from_dump_are_listed_as_created_in_event() {
+  let received;
+  clientWithDump.on("sync", ({ data }) => received = data);
+  // Use a timestamp superior to latest record in dump.
+  const timestamp = 5000000000000; // Fri Jun 11 2128
+
+  await clientWithDump.maybeSync(timestamp);
+
+  const list = await clientWithDump.get();
+  ok(list.length > 20, "The dump was loaded");
+  equal(received.created[received.created.length - 1].id, "xx", "Last record comes from the sync.");
+  equal(received.created.length, list.length, "The list of created records contains the dump");
+  equal(received.current.length, received.created.length);
+});
+add_task(clear_state);
+
 add_task(async function test_records_can_have_local_fields() {
   const c = RemoteSettings("password-fields", { localFields: ["accepted"] });
   await c.maybeSync(2000);
 
   const col = await c.openCollection();
   await col.update({ id: "9d500963-d80e-3a91-6e74-66f3811b99cc", accepted: true });
 
   await c.maybeSync(2000); // Does not fail.
@@ -482,17 +471,40 @@ add_task(async function test_inspect_cha
   const { collections: after, mainBucket } = await RemoteSettings.inspect();
 
   // These two collections are listed in the main bucket in monitor/changes (both are registered).
   deepEqual(after.map(c => c.collection).sort(), ["crash-rate", "password-fields"]);
   equal(mainBucket, "main-preview");
 });
 add_task(clear_state);
 
-// get a response for a given request from sample data
+function handleResponse(request, response) {
+  try {
+    const sample = getSampleResponse(request, server.identity.primaryPort);
+    if (!sample) {
+      do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+    }
+
+    response.setStatusLine(null, sample.status.status, sample.status.statusText);
+    // send the headers
+    for (let headerLine of sample.sampleHeaders) {
+      let headerElements = headerLine.split(":");
+      response.setHeader(headerElements[0], headerElements[1].trimLeft());
+    }
+    response.setHeader("Date", (new Date()).toUTCString());
+
+    const body = typeof sample.responseBody == "string" ? sample.responseBody
+      : JSON.stringify(sample.responseBody);
+    response.write(body);
+    response.finish();
+  } catch (e) {
+    info(e);
+  }
+}
+
 function getSampleResponse(req, port) {
   const responses = {
     "OPTIONS": {
       "sampleHeaders": [
         "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
         "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
         "Access-Control-Allow-Origin: *",
         "Content-Type: application/json; charset=UTF-8",
@@ -736,16 +748,53 @@ wNuvFqc=
         "data": [{
           "id": "312cc78d-9c1f-4291-a4fa-a1be56f6cc69",
           "last_modified": 3000,
           "website": "https://some-website.com",
           "selector": "#webpage[field-pwd]",
         }],
       },
     },
+    "GET:/v1/buckets/main/collections/language-dictionaries": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1234\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": {
+          "id": "language-dictionaries",
+          "last_modified": 1234,
+          "signature": {
+            "signature": "xyz",
+            "x5u": `http://localhost:${port}/fake-x5u`,
+          },
+        },
+      }),
+    },
+    "GET:/v1/buckets/main/collections/language-dictionaries/records": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"5000000000000\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": {
+        "data": [{
+          "id": "xx",
+          "last_modified": 5000000000000,
+          "dictionaries": ["xx-XX@dictionaries.addons.mozilla.org"],
+        }],
+      },
+    },
     "GET:/v1/buckets/monitor/collections/changes/records?collection=no-mocked-responses&bucket=main": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         `Date: ${new Date().toUTCString()}`,
         "Etag: \"713705\"",