Bug 1547995 - Upgrade kinto-offline-client.js to v12.4.0 r=glasserc
authorMathieu Leplatre <mathieu@mozilla.com>
Mon, 13 May 2019 09:30:54 +0000
changeset 473743 3840128adf0a6c129ef34650fdf4d5cc0a79db38
parent 473742 b23f1b4655818d6d64517ddd7fa74fae1fbd9507
child 473744 8fb278dd620a5dab42e6ecc175ab7418ec62a72a
push id36013
push usercsabou@mozilla.com
push dateTue, 14 May 2019 16:01:08 +0000
treeherdermozilla-central@230016dbba05 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglasserc
bugs1547995
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1547995 - Upgrade kinto-offline-client.js to v12.4.0 r=glasserc Differential Revision: https://phabricator.services.mozilla.com/D30356
security/manager/ssl/tests/unit/test_intermediate_preloads.js
services/common/kinto-offline-client.js
services/common/kinto-storage-adapter.js
services/common/tests/unit/test_blocklist_onecrl.js
services/common/tests/unit/test_blocklist_pinning.js
services/common/tests/unit/test_kinto.js
services/common/tests/unit/test_storage_adapter.js
services/common/tests/unit/test_storage_adapter/v1.sqlite
services/settings/RemoteSettingsWorker.js
services/settings/test/unit/test_remote_settings.js
toolkit/components/extensions/test/xpcshell/test_ext_storage_sync.js
--- a/security/manager/ssl/tests/unit/test_intermediate_preloads.js
+++ b/security/manager/ssl/tests/unit/test_intermediate_preloads.js
@@ -87,16 +87,17 @@ function setupKintoPreloadServer(certGen
   attachmentCB: null,
   hashFunc: null,
   lengthFunc: null,
 }) {
   const dummyServerURL = `http://localhost:${server.identity.primaryPort}/v1`;
   Services.prefs.setCharPref("services.settings.server", dummyServerURL);
 
   const configPath = "/v1/";
+  const metadataPath = "/v1/buckets/security-state/collections/intermediates";
   const recordsPath = "/v1/buckets/security-state/collections/intermediates/records";
   const attachmentsPath = "/attachments/";
 
   if (options.hashFunc == null) {
     options.hashFunc = getHash;
   }
   if (options.lengthFunc == null) {
     options.lengthFunc = arr => arr.length;
@@ -106,32 +107,34 @@ function setupKintoPreloadServer(certGen
     for (let headerLine of headers) {
       let headerElements = headerLine.split(":");
       response.setHeader(headerElements[0], headerElements[1].trimLeft());
     }
     response.setHeader("Date", (new Date()).toUTCString());
   }
 
   // Basic server information, all static
-  server.registerPathHandler(configPath, (request, response) => {
+  const handler = (request, response) => {
     try {
       const respData = getResponseData(request, server.identity.primaryPort);
       if (!respData) {
         do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
         return;
       }
 
       response.setStatusLine(null, respData.status.status,
                              respData.status.statusText);
       setHeader(response, respData.responseHeaders);
       response.write(respData.responseBody);
     } catch (e) {
       info(e);
     }
-  });
+  };
+  server.registerPathHandler(configPath, handler);
+  server.registerPathHandler(metadataPath, handler);
 
   // Lists of certs
   server.registerPathHandler(recordsPath, (request, response) => {
     response.setStatusLine(null, 200, "OK");
     setHeader(response, [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
@@ -500,14 +503,30 @@ function getResponseData(req, port) {
         "hello": "kinto",
         "capabilities": {
           "attachments": {
             "base_url": `http://localhost:${port}/attachments/`,
           },
         },
       }),
     },
+    "GET:/v1/buckets/security-state/collections/intermediates?": {
+      "responseHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1234\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": {
+          "id": "intermediates",
+          "last_modified": 1234,
+        },
+      }),
+    },
   };
   let result = cannedResponses[`${req.method}:${req.path}?${req.queryString}`] ||
                cannedResponses[`${req.method}:${req.path}`] ||
                cannedResponses[req.method];
   return result;
 }
--- a/services/common/kinto-offline-client.js
+++ b/services/common/kinto-offline-client.js
@@ -28,17 +28,17 @@
 //
 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1394556#c3 for
 // more details.
 const global = this;
 
 var EXPORTED_SYMBOLS = ["Kinto"];
 
 /*
- * Version 12.3.0 - f7a9e81
+ * Version 12.4.0 - 896d337
  */
 
 (function(f){if(typeof exports==="object"&&typeof module!=="undefined"){module.exports=f()}else if(typeof define==="function"&&define.amd){define([],f)}else{var g;if(typeof window!=="undefined"){g=window}else if(typeof global!=="undefined"){g=global}else if(typeof self!=="undefined"){g=self}else{g=this}g.Kinto = f()}})(function(){var define,module,exports;return (function(){function r(e,n,t){function o(i,f){if(!n[i]){if(!e[i]){var c="function"==typeof require&&require;if(!f&&c)return c(i,!0);if(u)return u(i,!0);var a=new Error("Cannot find module '"+i+"'");throw a.code="MODULE_NOT_FOUND",a}var p=n[i]={exports:{}};e[i][0].call(p.exports,function(r){var n=e[i][1][r];return o(n||r)},p,p.exports,r,e,n,t)}return n[i].exports}for(var u="function"==typeof require&&require,i=0;i<t.length;i++)o(t[i]);return o}return r})()({1:[function(require,module,exports){
 /*
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
  * You may obtain a copy of the License at
@@ -64,17 +64,19 @@ var _base = _interopRequireDefault(requi
 
 var _IDB = _interopRequireDefault(require("../src/adapters/IDB"));
 
 var _utils = require("../src/utils");
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
 ChromeUtils.import("resource://gre/modules/Timer.jsm", global);
-const {XPCOMUtils} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
+const {
+  XPCOMUtils
+} = ChromeUtils.import("resource://gre/modules/XPCOMUtils.jsm");
 XPCOMUtils.defineLazyGlobalGetters(global, ["fetch", "indexedDB"]);
 ChromeUtils.defineModuleGetter(global, "EventEmitter", "resource://gre/modules/EventEmitter.jsm"); // Use standalone kinto-http module landed in FFx.
 
 ChromeUtils.defineModuleGetter(global, "KintoHttpClient", "resource://services-common/kinto-http-client.js");
 XPCOMUtils.defineLazyGetter(global, "generateUUID", () => {
   const {
     generateUUID
   } = Cc["@mozilla.org/uuid-generator;1"].getService(Ci.nsIUUIDGenerator);
@@ -497,16 +499,25 @@ function createListRequest(cid, store, f
 
 
   const indexField = filterFields.find(field => {
     return INDEXED_FIELDS.includes(field);
   });
 
   if (!indexField) {
     // Iterate on all records for this collection (ie. cid)
+    const isSubQuery = Object.keys(filters).some(key => key.includes(".")); // (ie. filters: {"article.title": "hello"})
+
+    if (isSubQuery) {
+      const newFilter = (0, _utils.transformSubObjectFilters)(filters);
+      const request = store.index("cid").openCursor(IDBKeyRange.only(cid));
+      request.onsuccess = cursorHandlers.all(newFilter, done);
+      return request;
+    }
+
     const request = store.index("cid").openCursor(IDBKeyRange.only(cid));
     request.onsuccess = cursorHandlers.all(filters, done);
     return request;
   } // If `indexField` was used already, don't filter again.
 
 
   const remainingFilters = (0, _utils.omitKeys)(filters, [indexField]); // value specified in the filter (eg. `filters: { _status: ["created", "updated"] }`)
 
@@ -583,34 +594,44 @@ class IDB extends _base.default {
     } // In previous versions, we used to have a database with name `${bid}/${cid}`.
     // Check if it exists, and migrate data once new schema is in place.
     // Note: the built-in migrations from IndexedDB can only be used if the
     // database name does not change.
 
 
     const dataToMigrate = this._options.migrateOldData ? await migrationRequired(this.cid) : null;
     this._db = await open(this.dbName, {
-      version: 1,
+      version: 2,
       onupgradeneeded: event => {
-        const db = event.target.result; // Records store
-
-        const recordsStore = db.createObjectStore("records", {
-          keyPath: ["_cid", "id"]
-        }); // An index to obtain all the records in a collection.
-
-        recordsStore.createIndex("cid", "_cid"); // Here we create indices for every known field in records by collection.
-        // Local record status ("synced", "created", "updated", "deleted")
-
-        recordsStore.createIndex("_status", ["_cid", "_status"]); // Last modified field
-
-        recordsStore.createIndex("last_modified", ["_cid", "last_modified"]); // Timestamps store
-
-        db.createObjectStore("timestamps", {
-          keyPath: "cid"
-        });
+        const db = event.target.result;
+
+        if (event.oldVersion < 1) {
+          // Records store
+          const recordsStore = db.createObjectStore("records", {
+            keyPath: ["_cid", "id"]
+          }); // An index to obtain all the records in a collection.
+
+          recordsStore.createIndex("cid", "_cid"); // Here we create indices for every known field in records by collection.
+          // Local record status ("synced", "created", "updated", "deleted")
+
+          recordsStore.createIndex("_status", ["_cid", "_status"]); // Last modified field
+
+          recordsStore.createIndex("last_modified", ["_cid", "last_modified"]); // Timestamps store
+
+          db.createObjectStore("timestamps", {
+            keyPath: "cid"
+          });
+        }
+
+        if (event.oldVersion < 2) {
+          // Collections store
+          db.createObjectStore("collections", {
+            keyPath: "cid"
+          });
+        }
       }
     });
 
     if (dataToMigrate) {
       const {
         records,
         timestamp
       } = dataToMigrate;
@@ -932,16 +953,42 @@ class IDB extends _base.default {
       }
 
       return records;
     } catch (e) {
       this._handleError("importBulk", e);
     }
   }
 
+  async saveMetadata(metadata) {
+    try {
+      await this.prepare("collections", store => store.put({
+        cid: this.cid,
+        metadata
+      }), {
+        mode: "readwrite"
+      });
+      return metadata;
+    } catch (e) {
+      this._handleError("saveMetadata", e);
+    }
+  }
+
+  async getMetadata() {
+    try {
+      let entry = null;
+      await this.prepare("collections", store => {
+        store.get(this.cid).onsuccess = e => entry = e.target.result;
+      });
+      return entry ? entry.metadata : null;
+    } catch (e) {
+      this._handleError("getMetadata", e);
+    }
+  }
+
 }
 /**
  * IDB transaction proxy.
  *
  * @param  {IDB} adapter        The call IDB adapter
  * @param  {IDBStore} store     The IndexedDB database store.
  * @param  {Array}    preloaded The list of records to make available to
  *                              get() (default: []).
@@ -1150,27 +1197,36 @@ class BaseAdapter {
    * @return {Promise}
    */
 
 
   loadDump(records) {
     throw new Error("Not Implemented.");
   }
 
+  saveMetadata(metadata) {
+    throw new Error("Not Implemented.");
+  }
+
+  getMetadata() {
+    throw new Error("Not Implemented.");
+  }
+
 }
 
 exports.default = BaseAdapter;
 
 },{}],6:[function(require,module,exports){
 "use strict";
 
 Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.recordsEqual = recordsEqual;
+exports.createKeyValueStoreIdSchema = createKeyValueStoreIdSchema;
 exports.CollectionTransaction = exports.default = exports.ServerWasFlushedError = exports.SyncResultObject = void 0;
 
 var _base = _interopRequireDefault(require("./adapters/base"));
 
 var _IDB = _interopRequireDefault(require("./adapters/IDB"));
 
 var _utils = require("./utils");
 
@@ -1312,16 +1368,40 @@ function createUUIDSchema() {
     },
 
     validate(id) {
       return typeof id == "string" && _utils.RE_RECORD_ID.test(id);
     }
 
   };
 }
+/**
+ * IDSchema for when using kinto.js as a key-value store.
+ * Using this IDSchema requires you to set a property as the id.
+ * This will be the property used to retrieve this record.
+ *
+ * @example
+ * const exampleCollection = db.collection("example", { idSchema: createKeyValueStoreIdSchema() })
+ * await exampleCollection.create({ title: "How to tie a tie", favoriteColor: "blue", id: "user123" }, { useRecordId: true })
+ * await exampleCollection.getAny("user123")
+ */
+
+
+function createKeyValueStoreIdSchema() {
+  return {
+    generate() {
+      throw new Error("createKeyValueStoreIdSchema() does not generate an id");
+    },
+
+    validate() {
+      return true;
+    }
+
+  };
+}
 
 function markStatus(record, status) {
   return { ...record,
     _status: status
   };
 }
 
 function markDeleted(record) {
@@ -1670,16 +1750,17 @@ class Collection {
    * never synced.
    *
    * @return {Promise}
    */
 
 
   async clear() {
     await this.db.clear();
+    await this.db.saveMetadata(null);
     await this.db.saveLastModified(null);
     return {
       data: [],
       permissions: {}
     };
   }
   /**
    * Encodes a record.
@@ -2551,17 +2632,19 @@ class Collection {
       const seconds = Math.ceil(this.api.backoff / 1000);
       return Promise.reject(new Error(`Server is asking clients to back off; retry in ${seconds}s or use the ignoreBackoff option.`));
     }
 
     const client = this.api.bucket(options.bucket).collection(options.collection);
     const result = new SyncResultObject();
 
     try {
-      // Fetch last changes from the server.
+      // Fetch collection metadata.
+      await this.pullMetadata(client, options); // Fetch last changes from the server.
+
       await this.pullChanges(client, result, options);
       const {
         lastModified
       } = result; // Fetch local changes
 
       const toSync = await this.gatherLocalChanges(); // Publish local changes and pull local resolutions
 
       await this.pushChanges(client, toSync, result, options); // Publish local resolution of push conflicts to server (on CLIENT_WINS)
@@ -2675,16 +2758,33 @@ class Collection {
       localRecord._status === "synced" && // And was synced from server
       localRecord.last_modified !== undefined && // And is older than imported one.
       record.last_modified > localRecord.last_modified;
       return shouldKeep;
     });
     return await this.db.importBulk(newRecords.map(markSynced));
   }
 
+  async pullMetadata(client, options = {}) {
+    const {
+      expectedTimestamp
+    } = options;
+    const query = expectedTimestamp ? {
+      query: {
+        _expected: expectedTimestamp
+      }
+    } : undefined;
+    const metadata = await client.getData(query);
+    return this.db.saveMetadata(metadata);
+  }
+
+  async metadata() {
+    return this.db.getMetadata();
+  }
+
 }
 /**
  * A Collection-oriented wrapper for an adapter's transaction.
  *
  * This defines the high-level functions available on a collection.
  * The collection itself offers functions of the same name. These will
  * perform just one operation in its own transaction.
  */
@@ -3053,16 +3153,17 @@ Object.defineProperty(exports, "__esModu
 });
 exports.sortObjects = sortObjects;
 exports.filterObject = filterObject;
 exports.filterObjects = filterObjects;
 exports.waterfall = waterfall;
 exports.deepEqual = deepEqual;
 exports.omitKeys = omitKeys;
 exports.arrayEqual = arrayEqual;
+exports.transformSubObjectFilters = transformSubObjectFilters;
 exports.RE_RECORD_ID = void 0;
 const RE_RECORD_ID = /^[a-zA-Z0-9][a-zA-Z0-9_-]*$/;
 /**
  * Checks if a value is undefined.
  * @param  {Any}  value
  * @return {Boolean}
  */
 
@@ -3110,16 +3211,21 @@ function sortObjects(order, list) {
 
 
 function filterObject(filters, entry) {
   return Object.keys(filters).every(filter => {
     const value = filters[filter];
 
     if (Array.isArray(value)) {
       return value.some(candidate => candidate === entry[filter]);
+    } else if (typeof value === "object") {
+      return filterObject(value, entry[filter]);
+    } else if (!entry.hasOwnProperty(filter)) {
+      console.error(`The property ${filter} does not exist`);
+      return false;
     }
 
     return entry[filter] === value;
   });
 }
 /**
  * Filters records in a list matching all given filters.
  *
@@ -3217,11 +3323,36 @@ function arrayEqual(a, b) {
     if (a[i] !== b[i]) {
       return false;
     }
   }
 
   return true;
 }
 
+function makeNestedObjectFromArr(arr, val, nestedFiltersObj) {
+  const last = arr.length - 1;
+  return arr.reduce((acc, cv, i) => {
+    if (i === last) {
+      return acc[cv] = val;
+    } else if (acc.hasOwnProperty(cv)) {
+      return acc[cv];
+    } else {
+      return acc[cv] = {};
+    }
+  }, nestedFiltersObj);
+}
+
+function transformSubObjectFilters(filtersObj) {
+  const transformedFilters = {};
+
+  for (const key in filtersObj) {
+    const keysArr = key.split(".");
+    const val = filtersObj[key];
+    makeNestedObjectFromArr(keysArr, val, transformedFilters);
+  }
+
+  return transformedFilters;
+}
+
 },{}]},{},[1])(1)
 });
 
--- a/services/common/kinto-storage-adapter.js
+++ b/services/common/kinto-storage-adapter.js
@@ -105,17 +105,18 @@ const statements = {
       collection_name TEXT,
       record_id TEXT,
       record TEXT
     );`,
 
   "createCollectionMetadata": `
     CREATE TABLE collection_metadata (
       collection_name TEXT PRIMARY KEY,
-      last_modified INTEGER
+      last_modified INTEGER,
+      metadata TEXT
     ) WITHOUT ROWID;`,
 
   "createCollectionDataRecordIdIndex": `
     CREATE UNIQUE INDEX unique_collection_record
       ON collection_data(collection_name, record_id);`,
 
   "clearData": `
     DELETE FROM collection_data
@@ -130,24 +131,35 @@ const statements = {
       VALUES (:collection_name, :record_id, :record);`,
 
   "deleteData": `
     DELETE FROM collection_data
       WHERE collection_name = :collection_name
       AND record_id = :record_id;`,
 
   "saveLastModified": `
-    REPLACE INTO collection_metadata (collection_name, last_modified)
-      VALUES (:collection_name, :last_modified);`,
+    INSERT INTO collection_metadata(collection_name, last_modified)
+      VALUES(:collection_name, :last_modified)
+        ON CONFLICT(collection_name) DO UPDATE SET last_modified = :last_modified`,
 
   "getLastModified": `
     SELECT last_modified
       FROM collection_metadata
         WHERE collection_name = :collection_name;`,
 
+  "saveMetadata": `
+    INSERT INTO collection_metadata(collection_name, metadata)
+      VALUES(:collection_name, :metadata)
+        ON CONFLICT(collection_name) DO UPDATE SET metadata = :metadata`,
+
+  "getMetadata": `
+    SELECT metadata
+      FROM collection_metadata
+        WHERE collection_name = :collection_name;`,
+
   "getRecord": `
     SELECT record
       FROM collection_data
         WHERE collection_name = :collection_name
         AND record_id = :record_id;`,
 
   "listRecords": `
     SELECT record
@@ -169,25 +181,29 @@ const statements = {
   "scanAllRecords": `SELECT * FROM collection_data;`,
 
   "clearCollectionMetadata": `DELETE FROM collection_metadata;`,
 
   "calculateStorage": `
     SELECT collection_name, SUM(LENGTH(record)) as size, COUNT(record) as num_records
       FROM collection_data
         GROUP BY collection_name;`,
+
+  "addMetadataColumn": `
+    ALTER TABLE collection_metadata
+      ADD COLUMN metadata TEXT;`,
 };
 
 const createStatements = [
   "createCollectionData",
   "createCollectionMetadata",
   "createCollectionDataRecordIdIndex",
 ];
 
-const currentSchemaVersion = 1;
+const currentSchemaVersion = 2;
 
 /**
  * Firefox adapter.
  *
  * Uses Sqlite as a backing store.
  *
  * Options:
  *  - sqliteHandle: a handle to the Sqlite database this adapter will
@@ -211,19 +227,21 @@ class FirefoxAdapter extends Kinto.adapt
   static async _init(connection) {
     await connection.executeTransaction(async function doSetup() {
       const schema = await connection.getSchemaVersion();
 
       if (schema == 0) {
         for (let statementName of createStatements) {
           await connection.execute(statements[statementName]);
         }
-
         await connection.setSchemaVersion(currentSchemaVersion);
-      } else if (schema != 1) {
+      } else if (schema == 1) {
+        await connection.execute(statements.addMetadataColumn);
+        await connection.setSchemaVersion(currentSchemaVersion);
+      } else if (schema != 2) {
         throw new Error("Unknown database schema: " + schema);
       }
     });
     return connection;
   }
 
   _executeStatement(statement, params) {
     return this._connection.executeCached(statement, params);
@@ -396,16 +414,36 @@ class FirefoxAdapter extends Kinto.adapt
       .then(result => {
         if (result.length == 0) {
           return 0;
         }
         return result[0].getResultByName("last_modified");
       });
   }
 
+  async saveMetadata(metadata) {
+    const params = {
+      collection_name: this.collection,
+      metadata: JSON.stringify(metadata),
+    };
+    await this._executeStatement(statements.saveMetadata, params);
+    return metadata;
+  }
+
+  async getMetadata() {
+    const params = {
+      collection_name: this.collection,
+    };
+    const result = await this._executeStatement(statements.getMetadata, params);
+    if (result.length == 0) {
+      return null;
+    }
+    return JSON.parse(result[0].getResultByName("metadata"));
+  }
+
   calculateStorage() {
     return this._executeStatement(statements.calculateStorage, {})
       .then(result => {
         return Array.from(result, row => ({
           collectionName: row.getResultByName("collection_name"),
           size: row.getResultByName("size"),
           numRecords: row.getResultByName("num_records"),
         }));
--- a/services/common/tests/unit/test_blocklist_onecrl.js
+++ b/services/common/tests/unit/test_blocklist_onecrl.js
@@ -8,19 +8,16 @@ const BinaryInputStream = CC("@mozilla.o
 
 let server;
 
 // Some simple tests to demonstrate that the logic inside maybeSync works
 // correctly and that simple kinto operations are working as expected. There
 // are more tests for core Kinto.js (and its storage adapter) in the
 // xpcshell tests under /services/common
 add_task(async function test_something() {
-  const configPath = "/v1/";
-  const recordsPath = "/v1/buckets/security-state/collections/onecrl/records";
-
   const dummyServerURL = `http://localhost:${server.identity.primaryPort}/v1`;
   Services.prefs.setCharPref("services.settings.server", dummyServerURL);
 
   const {OneCRLBlocklistClient} = BlocklistClients.initialize({verifySignature: false});
 
   // register a handler
   function handleResponse(request, response) {
     try {
@@ -38,18 +35,19 @@ add_task(async function test_something()
       }
       response.setHeader("Date", (new Date()).toUTCString());
 
       response.write(sample.responseBody);
     } catch (e) {
       info(e);
     }
   }
-  server.registerPathHandler(configPath, handleResponse);
-  server.registerPathHandler(recordsPath, handleResponse);
+  server.registerPathHandler("/v1/", handleResponse);
+  server.registerPathHandler("/v1/buckets/security-state/collections/onecrl", handleResponse);
+  server.registerPathHandler("/v1/buckets/security-state/collections/onecrl/records", handleResponse);
 
   // Test an empty db populates from JSON dump.
   await OneCRLBlocklistClient.maybeSync(42);
 
   // Open the collection, verify it's been populated:
   const list = await OneCRLBlocklistClient.get();
   // We know there will be initial values from the JSON dump.
   // (at least as many as in the dump shipped when this test was written).
@@ -143,16 +141,32 @@ function getSampleResponse(req, port) {
         },
         "url": `http://localhost:${port}/v1/`,
         "documentation": "https://kinto.readthedocs.org/",
         "version": "1.5.1",
         "commit": "cbc6f58",
         "hello": "kinto",
       }),
     },
+    "GET:/v1/buckets/security-state/collections/onecrl": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1234\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": {
+          "id": "onecrl",
+          "last_modified": 1234,
+        },
+      }),
+    },
     "GET:/v1/buckets/security-state/collections/onecrl/records?_expected=2000&_sort=-last_modified&_since=1000": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         "Etag: \"3000\"",
       ],
@@ -208,10 +222,11 @@ function getSampleResponse(req, port) {
         "subject": "MCIxIDAeBgNVBAMMF0Fub3RoZXIgVGVzdCBFbmQtZW50aXR5",
         "pubKeyHash": "VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=",
         "id": "dabafde9-df4a-ddba-2548-748da04cc02g",
         "last_modified": 5000,
       }]}),
     },
   };
   return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[`${req.method}:${req.path}`] ||
          responses[req.method];
 }
--- a/services/common/tests/unit/test_blocklist_pinning.js
+++ b/services/common/tests/unit/test_blocklist_pinning.js
@@ -26,19 +26,16 @@ let server;
 
 // Some simple tests to demonstrate that the core preload sync operations work
 // correctly and that simple kinto operations are working as expected.
 add_task(async function test_something() {
   const {
     PinningBlocklistClient: PinningPreloadClient,
   } = BlocklistClients.initialize({ verifySignature: false });
 
-  const configPath = "/v1/";
-  const recordsPath = "/v1/buckets/pinning/collections/pins/records";
-
   Services.prefs.setCharPref("services.settings.server",
                              `http://localhost:${server.identity.primaryPort}/v1`);
 
   // register a handler
   function handleResponse(request, response) {
     try {
       const sample = getSampleResponse(request, server.identity.primaryPort);
       if (!sample) {
@@ -54,18 +51,19 @@ add_task(async function test_something()
       }
       response.setHeader("Date", (new Date()).toUTCString());
 
       response.write(sample.responseBody);
     } catch (e) {
       info(e);
     }
   }
-  server.registerPathHandler(configPath, handleResponse);
-  server.registerPathHandler(recordsPath, handleResponse);
+  server.registerPathHandler("/v1/", handleResponse);
+  server.registerPathHandler("/v1/buckets/pinning/collections/pins", handleResponse);
+  server.registerPathHandler("/v1/buckets/pinning/collections/pins/records", handleResponse);
 
   let sss = Cc["@mozilla.org/ssservice;1"]
               .getService(Ci.nsISiteSecurityService);
 
   // ensure our pins are all missing before we start
   ok(!sss.isSecureURI(sss.HEADER_HPKP,
                       Services.io.newURI("https://one.example.com"), 0));
   ok(!sss.isSecureURI(sss.HEADER_HPKP,
@@ -179,16 +177,32 @@ function getSampleResponse(req, port) {
         },
         "url": `http://localhost:${port}/v1/`,
         "documentation": "https://kinto.readthedocs.org/",
         "version": "1.5.1",
         "commit": "cbc6f58",
         "hello": "kinto",
       }),
     },
+    "GET:/v1/buckets/pinning/collections/pins": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1234\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": {
+          "id": "pins",
+          "last_modified": 1234,
+        },
+      }),
+    },
     "GET:/v1/buckets/pinning/collections/pins/records?_expected=2000&_sort=-last_modified": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         "Etag: \"3000\"",
       ],
@@ -293,10 +307,11 @@ function getSampleResponse(req, port) {
         "expires": new Date().getTime() + 1000000,
         "versions": [Services.appinfo.version, "some version that won't match"],
         "id": "dabafde9-df4a-ddba-2548-748da04cc032",
         "last_modified": 5000,
       }]}),
     },
   };
   return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[`${req.method}:${req.path}`] ||
          responses[req.method];
 }
--- a/services/common/tests/unit/test_kinto.js
+++ b/services/common/tests/unit/test_kinto.js
@@ -280,16 +280,17 @@ add_task(async function test_loadDump_sh
 
 add_task(clear_collection);
 
 // Now do some sanity checks against a server - we're not looking to test
 // core kinto.js functionality here (there is excellent test coverage in
 // kinto.js), more making sure things are basically working as expected.
 add_task(async function test_kinto_sync() {
   const configPath = "/v1/";
+  const metadataPath = "/v1/buckets/default/collections/test_collection";
   const recordsPath = "/v1/buckets/default/collections/test_collection/records";
   // register a handler
   function handleResponse(request, response) {
     try {
       const sampled = getSampleResponse(request, server.identity.primaryPort);
       if (!sampled) {
         do_throw(`unexpected ${request.method} request for ${request.path}?${request.queryString}`);
       }
@@ -304,16 +305,17 @@ add_task(async function test_kinto_sync(
       response.setHeader("Date", (new Date()).toUTCString());
 
       response.write(sampled.responseBody);
     } catch (e) {
       dump(`${e}\n`);
     }
   }
   server.registerPathHandler(configPath, handleResponse);
+  server.registerPathHandler(metadataPath, handleResponse);
   server.registerPathHandler(recordsPath, handleResponse);
 
   // create an empty collection, sync to populate
   let sqliteHandle;
   try {
     let result;
     sqliteHandle = await do_get_kinto_sqliteHandle();
     const collection = do_get_kinto_collection(sqliteHandle);
@@ -387,16 +389,32 @@ function getSampleResponse(req, port) {
         },
         "url": `http://localhost:${port}/v1/`,
         "documentation": "https://kinto.readthedocs.org/",
         "version": "1.5.1",
         "commit": "cbc6f58",
         "hello": "kinto",
       }),
     },
+    "GET:/v1/buckets/default/collections/test_collection": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1234\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": {
+          "id": "test_collection",
+          "last_modified": 1234,
+        },
+      }),
+    },
     "GET:/v1/buckets/default/collections/test_collection/records?_sort=-last_modified": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         "Etag: \"1445606341071\"",
       ],
@@ -448,10 +466,11 @@ function getSampleResponse(req, port) {
           "done": true,
           "id": "some-manually-chosen-id",
           "title": "New record with custom ID",
         }],
       }),
     },
   };
   return responses[`${req.method}:${req.path}?${req.queryString}`] ||
+         responses[`${req.method}:${req.path}`] ||
          responses[req.method];
 }
--- a/services/common/tests/unit/test_storage_adapter.js
+++ b/services/common/tests/unit/test_storage_adapter.js
@@ -1,11 +1,12 @@
 /* Any copyright is dedicated to the Public Domain.
    http://creativecommons.org/publicdomain/zero/1.0/ */
 
+const {Sqlite} = ChromeUtils.import("resource://gre/modules/Sqlite.jsm");
 const {FirefoxAdapter} = ChromeUtils.import("resource://services-common/kinto-storage-adapter.js");
 
 // set up what we need to make storage adapters
 const kintoFilename = "kinto.sqlite";
 
 function do_get_kinto_connection() {
   return FirefoxAdapter.openConnection({path: kintoFilename});
 }
@@ -216,16 +217,29 @@ function test_collection_operations() {
     await adapter.loadDump([
       {id: 1, foo: "bar", last_modified: 1457896541},
       {id: 2, foo: "baz", last_modified: 1458796542},
     ]);
     let lastModified = await adapter.getLastModified();
     Assert.equal(lastModified, 1458796543);
     await sqliteHandle.close();
   });
+
+  add_task(async function test_save_metadata_preserves_lastModified() {
+    let sqliteHandle = await do_get_kinto_connection();
+
+    let adapter = do_get_kinto_adapter(sqliteHandle);
+    await adapter.saveLastModified(42);
+
+    await adapter.saveMetadata({id: "col"});
+
+    let lastModified = await adapter.getLastModified();
+    Assert.equal(lastModified, 42);
+    await sqliteHandle.close();
+  });
 }
 
 // test kinto db setup and operations in various scenarios
 // test from scratch - no current existing database
 add_test(function test_db_creation() {
   add_test(function test_create_from_scratch() {
     // ensure the file does not exist in the profile
     let kintoDB = do_get_kinto_db();
@@ -252,8 +266,40 @@ add_test(function test_creation_from_emp
     run_next_test();
   });
 
   test_collection_operations();
 
   cleanup_kinto();
   run_next_test();
 });
+
+// test schema version upgrade at v2
+add_test(function test_migration_from_v1_to_v2() {
+  add_test(function test_migrate_from_v1_to_v2() {
+    // place an empty kinto db file in the profile
+    let profile = do_get_profile();
+
+    let v1DB = do_get_file("test_storage_adapter/v1.sqlite");
+    v1DB.copyTo(profile, kintoFilename);
+
+    run_next_test();
+  });
+
+  add_test(async function schema_is_update_from_1_to_2() {
+    // The `v1.sqlite` has schema version 1.
+    let sqliteHandle = await Sqlite.openConnection({ path: kintoFilename });
+    Assert.equal(await sqliteHandle.getSchemaVersion(), 1);
+    await sqliteHandle.close();
+
+    // The `.openConnection()` migrates it to version 2.
+    sqliteHandle = await FirefoxAdapter.openConnection({ path: kintoFilename });
+    Assert.equal(await sqliteHandle.getSchemaVersion(), 2);
+    await sqliteHandle.close();
+
+    run_next_test();
+  });
+
+  test_collection_operations();
+
+  cleanup_kinto();
+  run_next_test();
+});
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..8482b8b31d045e1d98a3906254ac02708862956d
GIT binary patch
literal 131072
zc%1FhUrSSA90%}cDhP&G5#Bx&g`!2^O$33ahZ>p7HBpHe%N=3iHnbxIf!l&!N*BF_
zUP^bplQvx`LlZ0+(dTnG=kPq|_j}Iw{CT;yQp?h~)7-CDvN#{wA(smaF@|t!BZM#(
zPK*wWbN&4{x@+9K|8TO`e0Vdo-X4DKo&W#<000000000000000000000000000000
z00000{!yJ9qvKPLW^<iFtx-+iA2e!v2Wfe`xx1TgXSHUdyq|72_p7HSs+FuVwp2_O
zOG#W`&9AK|F~7Q;Y{v6?C*qO9Y8+T)`qa2lsi$-CxS?FD&UEgMj8Dza=RW4L%A0P{
zNo76FD!uMArbhd*N{f$Il6XczobHDixN|Heo2B@on18-l+=|bVt+{^PZslE8t~aZ-
zomyJ$K4&R;k`#}w&BTp->1koT6pMw8{Bq}dZhUIC`yO8p<{ik|8;+cp^TLcjbNzZz
zXUrZ=9cP^Eed2Zq*{dP{W&i*H0000000000000000000000000000000Pw%L_IcO|
z004kfCVGc<2(1sTrQ>bLfdBvi00000000000000000000000000000000000xB{*3
zVc`$^Z(m&WB|Ny~`kyY@_xDWn4(;wkd>e8Q000000000000000000000000000000
R000000002EDt9g}`2mowv?Krk
--- a/services/settings/RemoteSettingsWorker.js
+++ b/services/settings/RemoteSettingsWorker.js
@@ -10,17 +10,17 @@
  * A worker dedicated to Remote Settings.
  */
 
 importScripts("resource://gre/modules/workers/require.js",
               "resource://gre/modules/CanonicalJSON.jsm",
               "resource://gre/modules/third_party/jsesc/jsesc.js");
 
 const IDB_NAME = "remote-settings";
-const IDB_VERSION = 1;
+const IDB_VERSION = 2;
 const IDB_RECORDS_STORE = "records";
 const IDB_TIMESTAMPS_STORE = "timestamps";
 
 const Agent = {
   /**
    * Return the canonical JSON serialization of the changes
    * applied to the local records.
    * It has to match what is done on the server (See Kinto/kinto-signer).
--- a/services/settings/test/unit/test_remote_settings.js
+++ b/services/settings/test/unit/test_remote_settings.js
@@ -73,19 +73,21 @@ function run_test() {
       response.write(body);
       response.finish();
     } catch (e) {
       info(e);
     }
   }
   const configPath = "/v1/";
   const changesPath = "/v1/buckets/monitor/collections/changes/records";
+  const metadataPath = "/v1/buckets/main/collections/password-fields";
   const recordsPath  = "/v1/buckets/main/collections/password-fields/records";
   server.registerPathHandler(configPath, handleResponse);
   server.registerPathHandler(changesPath, handleResponse);
+  server.registerPathHandler(metadataPath, handleResponse);
   server.registerPathHandler(recordsPath, handleResponse);
 
   run_next_test();
 
   registerCleanupFunction(function() {
     server.stop(() => { });
   });
 }
@@ -515,16 +517,36 @@ function getSampleResponse(req, port) {
         }, {
           "id": "58697bd1-315f-4185-9bee-3371befc2585",
           "bucket": "main-preview",
           "collection": "crash-rate",
           "last_modified": 1000,
         }],
       },
     },
+    "GET:/v1/buckets/main/collections/password-fields": {
+      "sampleHeaders": [
+        "Access-Control-Allow-Origin: *",
+        "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
+        "Content-Type: application/json; charset=UTF-8",
+        "Server: waitress",
+        "Etag: \"1234\"",
+      ],
+      "status": { status: 200, statusText: "OK" },
+      "responseBody": JSON.stringify({
+        "data": {
+          "id": "password-fields",
+          "last_modified": 1234,
+          "signature": {
+            "signature": "abcdef",
+            "x5u": "http://localhost/dummy",
+          },
+        },
+      }),
+    },
     "GET:/v1/buckets/main/collections/password-fields/records?_expected=2000&_sort=-last_modified": {
       "sampleHeaders": [
         "Access-Control-Allow-Origin: *",
         "Access-Control-Expose-Headers: Retry-After, Content-Length, Alert, Backoff",
         "Content-Type: application/json; charset=UTF-8",
         "Server: waitress",
         "Etag: \"3000\"",
       ],
--- a/toolkit/components/extensions/test/xpcshell/test_ext_storage_sync.js
+++ b/toolkit/components/extensions/test/xpcshell/test_ext_storage_sync.js
@@ -29,16 +29,20 @@ function handleCannedResponse(cannedResp
     let headerElements = headerLine.split(":");
     response.setHeader(headerElements[0], headerElements[1].trimLeft());
   }
   response.setHeader("Date", (new Date()).toUTCString());
 
   response.write(cannedResponse.responseBody);
 }
 
+function collectionPath(collectionId) {
+  return `/buckets/default/collections/${collectionId}`;
+}
+
 function collectionRecordsPath(collectionId) {
   return `/buckets/default/collections/${collectionId}/records`;
 }
 
 class KintoServer {
   constructor() {
     // Set up an HTTP Server
     this.httpServer = new HttpServer();
@@ -251,20 +255,33 @@ class KintoServer {
    * @param {string} collectionId   the collection whose route we
    *    should set up.
    */
   installCollection(collectionId) {
     if (this.collections.has(collectionId)) {
       return;
     }
     this.collections.add(collectionId);
+    const remoteCollectionPath = "/v1" + collectionPath(encodeURIComponent(collectionId));
+    this.httpServer.registerPathHandler(remoteCollectionPath, this.handleGetCollection.bind(this, collectionId));
     const remoteRecordsPath = "/v1" + collectionRecordsPath(encodeURIComponent(collectionId));
     this.httpServer.registerPathHandler(remoteRecordsPath, this.handleGetRecords.bind(this, collectionId));
   }
 
+  handleGetCollection(collectionId, request, response) {
+    response.setStatusLine(null, 200, "OK");
+    response.setHeader("Content-Type", "application/json; charset=UTF-8");
+    response.setHeader("Date", (new Date()).toUTCString());
+    response.write(JSON.stringify({
+      data: {
+        id: collectionId,
+      },
+    }));
+  }
+
   handleGetRecords(collectionId, request, response) {
     if (this.checkAuth(request, response)) {
       return;
     }
 
     if (request.method != "GET") {
       do_throw(`only GET is supported on ${request.path}`);
     }