Bug 1552199 - Include records from dump in Remote Settings sync event created data r=glasserc
authorMathieu Leplatre <mathieu@mozilla.com>
Tue, 21 May 2019 07:38:29 +0000
changeset 474691 a57a160207c492159c70a7bf4d41b710cbac7eaa
parent 474690 0e382475292d9801921403db032eb0e9dba31c12
child 474692 cc344bcbf49d0f99d8a76fa15bdbb91a934728cb
push id113168
push userrmaries@mozilla.com
push dateTue, 21 May 2019 16:39:23 +0000
treeherdermozilla-inbound@3c0f78074b72 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglasserc
bugs1552199
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1552199 - Include records from dump in Remote Settings sync event created data r=glasserc Differential Revision: https://phabricator.services.mozilla.com/D31599
services/settings/RemoteSettingsClient.jsm
services/settings/test/unit/test_remote_settings.js
--- a/services/settings/RemoteSettingsClient.jsm
+++ b/services/settings/RemoteSettingsClient.jsm
@@ -277,30 +277,35 @@ class RemoteSettingsClient extends Event
    * @param {Object} options           additional advanced options.
    * @param {bool}   options.loadDump  load initial dump from disk on first sync (default: true)
    * @param {string} options.trigger   label to identify what triggered this sync (eg. ``"timer"``, default: `"manual"`)
    * @return {Promise}                 which rejects on sync or process failure.
    */
   async maybeSync(expectedTimestamp, options = {}) {
     const { loadDump = true, trigger = "manual" } = options;
 
+    let importedFromDump = [];
     const startedAt = new Date();
     let reportStatus = null;
     try {
       // Synchronize remote data into a local DB using Kinto.
       const kintoCollection = await this.openCollection();
       let collectionLastModified = await kintoCollection.db.getLastModified();
 
       // If there is no data currently in the collection, attempt to import
       // initial data from the application defaults.
       // This allows to avoid synchronizing the whole collection content on
       // cold start.
       if (!collectionLastModified && loadDump) {
         try {
-          await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
+          const imported = await RemoteSettingsWorker.importJSONDump(this.bucketName, this.collectionName);
+          // The worker only returns an integer. List the imported records to build the sync event.
+          if (imported > 0) {
+            ({ data: importedFromDump } = await kintoCollection.list());
+          }
           collectionLastModified = await kintoCollection.db.getLastModified();
         } catch (e) {
           // Report but go-on.
           Cu.reportError(e);
         }
       }
 
       // If the data is up to date, there's no need to sync. We still need
@@ -332,16 +337,19 @@ class RemoteSettingsClient extends Event
       try {
         // Fetch changes from server, and make sure we overwrite local data.
         const strategy = Kinto.syncStrategy.SERVER_WINS;
         syncResult = await kintoCollection.sync({ remote: gServerURL, strategy, expectedTimestamp });
         if (!syncResult.ok) {
           // With SERVER_WINS, there cannot be any conflicts, but don't silent it anyway.
           throw new Error("Synced failed");
         }
+        // The records imported from the dump should be considered as "created" for the
+        // listeners.
+        syncResult.created = importedFromDump.concat(syncResult.created);
       } catch (e) {
         if (e instanceof RemoteSettingsClient.InvalidSignatureError) {
           // Signature verification failed during synchronization.
           reportStatus = UptakeTelemetry.STATUS.SIGNATURE_ERROR;
           // If sync fails with a signature error, it's likely that our
           // local data has been modified in some way.
           // We will attempt to fix this by retrieving the whole
           // remote collection.
--- a/services/settings/test/unit/test_remote_settings.js
+++ b/services/settings/test/unit/test_remote_settings.js
@@ -46,49 +46,22 @@ function run_test() {
                              `http://localhost:${server.identity.primaryPort}/v1`);
 
   client = RemoteSettings("password-fields");
   client.verifySignature = false;
 
   clientWithDump = RemoteSettings("language-dictionaries");
   clientWithDump.verifySignature = false;
 
-  // Setup server fake responses.
-  function handleResponse(request, response) {
-    try {
-      const sample = getSampleResponse(request, server.identity.primaryPort);
-      if (!sample) {
-        do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
-      }
-
-      response.setStatusLine(null, sample.status.status,
-                             sample.status.statusText);
-      // send the headers
-      for (let headerLine of sample.sampleHeaders) {
-        let headerElements = headerLine.split(":");
-        response.setHeader(headerElements[0], headerElements[1].trimLeft());
-      }
-      response.setHeader("Date", (new Date()).toUTCString());
-
-      const body = typeof sample.responseBody == "string" ? sample.responseBody
-                                                          : JSON.stringify(sample.responseBody);
-      response.write(body);
-      response.finish();
-    } catch (e) {
-      info(e);
-    }
-  }
-  const configPath = "/v1/";
-  const changesPath = "/v1/buckets/monitor/collections/changes/records";
-  const metadataPath = "/v1/buckets/main/collections/password-fields";
-  const recordsPath  = "/v1/buckets/main/collections/password-fields/records";
-  server.registerPathHandler(configPath, handleResponse);
-  server.registerPathHandler(changesPath, handleResponse);
-  server.registerPathHandler(metadataPath, handleResponse);
-  server.registerPathHandler(recordsPath, handleResponse);
+  server.registerPathHandler("/v1/", handleResponse);
+  server.registerPathHandler("/v1/buckets/monitor/collections/changes/records", handleResponse);
+  server.registerPathHandler("/v1/buckets/main/collections/password-fields", handleResponse);
+  server.registerPathHandler("/v1/buckets/main/collections/password-fields/records", handleResponse);
+  server.registerPathHandler("/v1/buckets/main/collections/language-dictionaries", handleResponse);
+  server.registerPathHandler("/v1/buckets/main/collections/language-dictionaries/records", handleResponse);
   server.registerPathHandler("/fake-x5u", handleResponse);
 
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(() => { });
   });
 }
@@ -99,16 +72,36 @@ add_task(async function test_records_obt
 
   // Open the collection, verify it's been populated:
   // Our test data has a single record; it should be in the local collection
   const list = await client.get();
   equal(list.length, 1);
 });
 add_task(clear_state);
 
+add_task(async function test_records_from_dump_are_listed_as_created_in_event() {
+  if (IS_ANDROID) {
+    // Skip test: we don't ship remote settings dumps on Android (see package-manifest).
+    return;
+  }
+  let received;
+  clientWithDump.on("sync", ({ data }) => received = data);
+  // Use a timestamp superior to latest record in dump.
+  const timestamp = 5000000000000; // Fri Jun 11 2128
+
+  await clientWithDump.maybeSync(timestamp);
+
+  const list = await clientWithDump.get();
+  ok(list.length > 20, "The dump was loaded");
+  equal(received.created[received.created.length - 1].id, "xx", "Last record comes from the sync.");
+  equal(received.created.length, list.length, "The list of created records contains the dump");
+  equal(received.current.length, received.created.length);
+});
+add_task(clear_state);
+
 add_task(async function test_records_can_have_local_fields() {
   const c = RemoteSettings("password-fields", { localFields: ["accepted"] });
   await c.maybeSync(2000);
 
   const col = await c.openCollection();
   await col.update({ id: "9d500963-d80e-3a91-6e74-66f3811b99cc", accepted: true });
 
   await c.maybeSync(2000); // Does not fail.
@@ -482,17 +475,40 @@ add_task(async function test_inspect_cha
   const { collections: after, mainBucket } = await RemoteSettings.inspect();
 
   // These two collections are listed in the main bucket in monitor/changes (both are registered).
   deepEqual(after.map(c => c.collection).sort(), ["crash-rate", "password-fields"]);
   equal(mainBucket, "main-preview");
 });
 add_task(clear_state);
 
-// get a response for a given request from sample data
+function handleResponse(request, response) {
+  try {
+    const sample = getSampleResponse(request, server.identity.primaryPort);
+    if (!sample) {
+      do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
+    }
+
+    response.setStatusLine(null, sample.status.status, sample.status.statusText);
+    // send the headers
+    for (let headerLine of sample.sampleHeaders) {
+      let headerElements = headerLine.split(":");
+      response.setHeader(headerElements[0], headerElements[1].trimLeft());
+    }
+    response.setHeader("Date", (new Date()).toUTCString());
+
+    const body = typeof sample.responseBody == "string" ? sample.responseBody
+      : JSON.stringify(sample.responseBody);
+    response.write(body);
+    response.finish();
+  } catch (e) {
+    info(e);
+  }
+}
+
 function getSampleResponse(req, port) {
   const responses = {
     "OPTIONS": {
       "sampleHeaders": [
         "Access-Control-Allow-Headers: Content-Length,Expires,Backoff,Retry-After,Last-Modified,Total-Records,ETag,Pragma,Cache-Control,authorization,content-type,if-none-match,Alert,Next-Page",
         "Access-Control-Allow-Methods: GET,HEAD,OPTIONS,POST,DELETE,OPTIONS",
         "Access-Control-Allow-Origin: *",
         "Content-Type: application/json; charset=UTF-8",
@@ -736,16 +752,53 @@ wNuvFqc=
         "data": [{
           "id": "312cc78d-9c1f-4291-a4fa-a1be56f6cc69",
           "last_modified": 3000,
           "website": "https://some-website.com",
           "selector": "#webpage[field-pwd]",
         }],
       },
     },
+    "GET:/v1/buckets/main/collections/language-dictionaries": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1234\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": {
+          "id": "language-dictionaries",
+          "last_modified": 1234,
+          "signature": {
+            "signature": "xyz",
+            "x5u": `http://localhost:${port}/fake-x5u`,
+          },
+        },
+      }),
+    },
+    "GET:/v1/buckets/main/collections/language-dictionaries/records": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"5000000000000\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": {
+        "data": [{
+          "id": "xx",
+          "last_modified": 5000000000000,
+          "dictionaries": ["xx-XX@dictionaries.addons.mozilla.org"],
+        }],
+      },
+    },
     "GET:/v1/buckets/monitor/collections/changes/records?collection=no-mocked-responses&bucket=main": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         `Date: ${new Date().toUTCString()}`,
         "Etag: \"713705\"",