first cut at some sync stubs draft
authorThom Chiovoloni <tchiovoloni@mozilla.com>
Tue, 25 Apr 2017 14:51:55 -0700
changeset 600698 2c1a73cb5b6912dc5f342002113a9ebe1c4cf44b
parent 600697 a975e078842992e6f7245f248eb0a5e01ca23c69
child 600699 47cb875bdeaf761d5280a6f5892b75e09031a29d
push id65844
push userbmo:kit@mozilla.com
push dateTue, 27 Jun 2017 18:57:45 +0000
milestone56.0a1
first cut at some sync stubs MozReview-Commit-ID: BcwS86YhfBc
browser/extensions/formautofill/FormAutofillSync.jsm
browser/extensions/formautofill/test/unit/head.js
browser/extensions/formautofill/test/unit/test_sync.js
browser/extensions/formautofill/test/unit/xpcshell.ini
services/sync/modules/engines.js
services/sync/modules/record.js
services/sync/modules/service.js
services/sync/modules/telemetry.js
services/sync/services-sync.js
tools/lint/eslint/modules.json
new file mode 100644
--- /dev/null
+++ b/browser/extensions/formautofill/FormAutofillSync.jsm
@@ -0,0 +1,418 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+"use strict";
+
+const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+
+this.EXPORTED_SYMBOLS = ["AddressesEngine", "CreditCardsEngine"];
+
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+Cu.import("resource://services-sync/engines.js");
+Cu.import("resource://services-sync/record.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-common/async.js");
+
+XPCOMUtils.defineLazyModuleGetter(this, "profileStorage",
+                                  "resource://formautofill/ProfileStorage.jsm");
+
+
+// XXX - this will end up in ProfileStorage - here for a POC (and at the top
+// of the file for eslint)
+function findDuplicateGUID(record) {
+  for (let profile of profileStorage.addresses.getAll()) {
+    let keys = new Set(Object.keys(record));
+    for (let key of Object.keys(profile)) {
+      keys.add(key);
+    }
+    let same = true;
+    for (let key of keys) {
+      if (!same) {
+        break;
+      }
+      if (profile.hasOwnProperty(key) && record.hasOwnProperty(key)) {
+        same = profile[key] == record[key];
+      }
+      // else something smarter, possibly using version field?
+    }
+    if (same) {
+      return profile.guid;
+    }
+  }
+  return null;
+}
+
+// A helper to sanitize address and creditcard records suitable for logging.
+function sanitizeStorageObject(ob) {
+  const whitelist = ["timeCreated", "timeLastUsed", "timeLastModified"];
+  let result = {};
+  for (let key of Object.keys(ob)) {
+    let origVal = ob[key];
+    if (whitelist.includes(key)) {
+      result[key] = origVal;
+    } else if (typeof origVal == "string") {
+      result[key] = "X".repeat(origVal.length);
+    } else {
+      result[key] = typeof(origVal); // *shrug*
+    }
+  }
+  return result;
+}
+
+// Three years in seconds
+// XXX - stolen from forms engine, possibly should be different
+const AUTOFILL_TTL = 3 * 365 * 24 * 60 * 60;
+
+function AutofillRecord(collection, id) {
+  CryptoWrapper.call(this, collection, id);
+}
+
+AutofillRecord.prototype = {
+  __proto__: CryptoWrapper.prototype,
+  ttl: AUTOFILL_TTL,
+
+  cleartextToString() {
+    // And a helper so logging a *Sync* record auto sanitizes.
+    let record = this.cleartext;
+    let result = {entry: {}};
+    if (record.entry) {
+      result.entry = sanitizeStorageObject(record.entry);
+    }
+    return JSON.stringify(result);
+  },
+};
+
+// Profile data is stored in the "entry" object of the record.
+Utils.deferGetSet(AutofillRecord, "cleartext", ["entry"]);
+
+function FormAutofillStore(name, engine) {
+  Store.call(this, name, engine);
+}
+
+FormAutofillStore.prototype = {
+  __proto__: Store.prototype,
+
+  _subStorageName: null, // overridden below.
+  _storage: null,
+
+  get storage() {
+    if (!this._storage) {
+      Async.promiseSpinningly(profileStorage.initialize());
+      this._storage = profileStorage[this._subStorageName];
+    }
+    return this._storage;
+  },
+
+  getAllIDs() {
+    let result = {};
+    for (let {guid} of this.storage.getAll({includeDeleted: true})) {
+      result[guid] = true;
+    }
+    return result;
+  },
+
+  changeItemID(oldID, newID) {
+    this.storage.changeGUID(oldID, newID);
+  },
+
+  // Note: this function should return false in cases where we only have a
+  // (local) tombstone - and profileStorage.get() filters them for us.
+  itemExists(id) {
+    return Boolean(this.storage.get(id));
+  },
+
+  applyIncoming(remoteRecord) {
+    if (remoteRecord.deleted) {
+      this._log.trace("Deleting record", remoteRecord);
+      this.storage.remove(remoteRecord.id, {sourceSync: true});
+      return;
+    }
+
+    if (this.itemExists(remoteRecord.id)) {
+      // We will never get a tombstone here, so we are updating a real record.
+      this._doUpdateRecord(remoteRecord);
+      return;
+    }
+
+    // No matching local record. Try to dedupe a NEW local record.
+    let localDupeID = /* this.storage. */findDuplicateGUID(remoteRecord.entry);
+    if (localDupeID) {
+      this._log.trace(`Deduping local record ${localDupeID} to remote`, remoteRecord);
+      // Change the local GUID to match the incoming record, then apply the
+      // incoming record.
+      this.changeItemID(localDupeID, remoteRecord.id);
+      this._doUpdateRecord(remoteRecord);
+      return;
+    }
+
+    // We didn't find a dupe, either, so must be a new record (or possibly
+    // a non-deleted version of an item we have a tombstone for, which create()
+    // handles for us.)
+    this._log.trace("Add record", remoteRecord);
+    remoteRecord.entry.guid = remoteRecord.id;
+    this.storage.add(remoteRecord.entry, {sourceSync: true});
+  },
+
+  createRecord(id, collection) {
+    this._log.trace("Create record", id);
+    let record = new AutofillRecord(collection, id);
+    record.entry = this.storage.get(id, {
+      noComputedFields: true,
+    });
+    if (!record.entry) {
+      // We should consider getting a more authortative indication it's actually deleted.
+      this._log.debug(`Failed to get autofill record with id "${id}", assuming deleted`);
+      record.deleted = true;
+    } else {
+      // The GUID is already stored in record.id, so we nuke it from the entry
+      // itself to save a tiny bit of space. The profileStorage clones profiles,
+      // so nuking in-place is OK.
+      delete record.entry.guid;
+    }
+    return record;
+  },
+
+  _doUpdateRecord(record) {
+    this._log.trace("Update record", record);
+
+    // XXX - until we get reconcilliation logic, this is dangerous - it
+    // unconditionally updates, which may cause DATA LOSS
+    this.storage.update(record.id, record.entry);
+    this._log.debug("Updated local record", record);
+  },
+
+  // NOTE: Because we re-implement the incoming/reconcilliation logic we leave
+  // the |create|, |remove| and |update| methods undefined - the base
+  // implementation throws, which is what we want to happen so we can identify
+  // any places they are "accidentally" called.
+};
+
+function FormAutofillTracker(name, engine) {
+  Tracker.call(this, name, engine);
+}
+
+FormAutofillTracker.prototype = {
+  __proto__: Tracker.prototype,
+  observe: function observe(subject, topic, data) {
+    Tracker.prototype.observe.call(this, subject, topic, data);
+    if (topic != "formautofill-storage-changed") {
+      return;
+    }
+    if (subject && subject.wrappedJSObject && subject.wrappedJSObject.sourceSync) {
+      return;
+    }
+    switch (data) {
+      case "add":
+      case "update":
+      case "remove":
+        this.score += SCORE_INCREMENT_XLARGE;
+        break;
+      default:
+        this._log.debug("unrecognized autofill notification", data);
+        break;
+    }
+  },
+
+  // `_ignore` checks the change source for each observer notification, so we
+  // don't want to let the engine ignore all changes during a sync.
+  get ignoreAll() {
+    return false;
+  },
+
+  // Define an empty setter so that the engine doesn't throw a `TypeError`
+  // setting a read-only property.
+  set ignoreAll(value) {},
+
+  startTracking() {
+    Services.obs.addObserver(this, "formautofill-storage-changed");
+  },
+
+  stopTracking() {
+    Services.obs.removeObserver(this, "formautofill-storage-changed");
+  },
+
+  // We never want to persist changed IDs, as the changes are already stored
+  // in ProfileStorage
+  persistChangedIDs: false,
+
+  // Ensure we aren't accidentally using the base persistence.
+  get changedIDs() {
+    throw new Error("changedIDs isn't meaningful for this engine");
+  },
+
+  set changedIDs(obj) {
+    throw new Error("changedIDs isn't meaningful for this engine");
+  },
+
+  addChangedID(id, when) {
+    throw new Error("Don't add IDs to the autofill tracker");
+  },
+
+  removeChangedID(id) {
+    throw new Error("Don't remove IDs from the autofill tracker");
+  },
+
+  // This method is called at various times, so we override with a no-op
+  // instead of throwing.
+  clearChangedIDs() {},
+};
+
+// This uses the same conventions as BookmarkChangeset in
+// services/sync/modules/engines/bookmarks.js. Specifically,
+// - "synced" means the item has already been synced (or we have another reason
+//   to ignore it), and should be ignored in most methods.
+class AutofillChangeset extends Changeset {
+  constructor() {
+    super();
+  }
+
+  getModifiedTimestamp(id) {
+    throw new Error("Don't use timestamps to resolve autofill merge conflicts");
+  }
+
+  has(id) {
+    let change = this.changes[id];
+    if (change) {
+      return !change.synced;
+    }
+    return false;
+  }
+
+  delete(id) {
+    let change = this.changes[id];
+    if (change) {
+      // Mark the change as synced without removing it from the set. We do this
+      // so that we can update ProfileStorage in `trackRemainingChanges`.
+      change.synced = true;
+    }
+  }
+}
+
+function FormAutofillEngine(service, name) {
+  SyncEngine.call(this, name, service);
+}
+
+FormAutofillEngine.prototype = {
+  __proto__: SyncEngine.prototype,
+
+  // the priority for this engine is == addons, so will happen after bookmarks
+  // prefs and tabs, but before forms, history, etc.
+  syncPriority: 5,
+
+  applyIncomingTombstones: true,
+
+  // XXX - do we need to look at, eg, the extensions.formautofill.addresses.enabled
+  // preference and make sync a no-op when disabled? (But note that we don't
+  // want to simply disable the engine in that case - that will explicitly
+  // "decline" it, which probably isn't what the user expects if the feature
+  // is enabled on a different device.)
+
+  // We handle reconciliation in the store, not the engine.
+  _reconcile() {
+    return true;
+  },
+
+  emptyChangeset() {
+    return new AutofillChangeset();
+  },
+
+  _uploadOutgoing() {
+    this._modified.replace(this._store.storage.pullSyncChanges());
+    SyncEngine.prototype._uploadOutgoing.call(this);
+  },
+
+  // Typically, engines populate the changeset before downloading records.
+  // However, we handle conflict resolution in the store, so we can wait
+  // to pull changes until we're ready to upload.
+  pullAllChanges() {
+    return {};
+  },
+
+  pullNewChanges() {
+    return {};
+  },
+
+  trackRemainingChanges() {
+    this._store.storage.pushSyncChanges(this._modified.changes);
+  },
+
+  _deleteId(id) {
+    this._noteDeletedId(id);
+  },
+
+  _resetClient() {
+    this._store.storage.resetSync();
+  },
+};
+
+// The concrete engines
+// XXX - these are (obviously) only partial...
+function AddressesRecord(collection, id) {
+  AutofillRecord.call(this, collection, id);
+}
+
+AddressesRecord.prototype = {
+  __proto__: AutofillRecord.prototype,
+  _logName: "Sync.Record.Addresses",
+};
+
+function AddressesStore(name, engine) {
+  FormAutofillStore.call(this, name, engine);
+}
+
+AddressesStore.prototype = {
+  __proto__: FormAutofillStore.prototype,
+  _subStorageName: "addresses",
+};
+
+function AddressesEngine(service) {
+  FormAutofillEngine.call(this, service, "Addresses");
+}
+
+AddressesEngine.prototype = {
+  __proto__: FormAutofillEngine.prototype,
+  _trackerObj: FormAutofillTracker,
+  _storeObj: AddressesStore,
+  _recordObj: AddressesRecord,
+
+  get prefName() {
+    return "addresses";
+  },
+};
+
+function CreditCardsRecord(collection, id) {
+  AutofillRecord.call(this, collection, id);
+}
+
+CreditCardsRecord.prototype = {
+  __proto__: AutofillRecord.prototype,
+  _logName: "Sync.Record.CreditCards",
+};
+
+function CreditCardsStore(name, engine) {
+  FormAutofillStore.call(this, name, engine);
+}
+
+CreditCardsStore.prototype = {
+  __proto__: FormAutofillStore.prototype,
+  _subStorageName: "creditCards",
+};
+
+
+function CreditCardsEngine(service) {
+  FormAutofillEngine.call(this, service, "CreditCards");
+}
+
+CreditCardsEngine.prototype = {
+  __proto__: FormAutofillEngine.prototype,
+  _trackerObj: FormAutofillTracker,
+  _storeObj: CreditCardsStore,
+  _recordObj: CreditCardsRecord,
+
+  get prefName() {
+    return "creditcards";
+  },
+};
--- a/browser/extensions/formautofill/test/unit/head.js
+++ b/browser/extensions/formautofill/test/unit/head.js
@@ -3,17 +3,17 @@
  */
 
 /* exported getTempFile, loadFormAutofillContent, runHeuristicsTest, sinon,
  *          initProfileStorage
  */
 
 "use strict";
 
-const {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
+var {classes: Cc, interfaces: Ci, utils: Cu, results: Cr} = Components;
 
 Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 Cu.import("resource://gre/modules/NetUtil.jsm");
 Cu.import("resource://gre/modules/FormLikeFactory.jsm");
 Cu.import("resource://testing-common/MockDocument.jsm");
 Cu.import("resource://testing-common/TestUtils.jsm");
 
new file mode 100644
--- /dev/null
+++ b/browser/extensions/formautofill/test/unit/test_sync.js
@@ -0,0 +1,384 @@
+/**
+ * Tests sync functionality.
+ */
+
+/* import-globals-from ../../../../../services/sync/tests/unit/head_appinfo.js */
+/* import-globals-from ../../../../../services/common/tests/unit/head_helpers.js */
+/* import-globals-from ../../../../../services/sync/tests/unit/head_helpers.js */
+/* import-globals-from ../../../../../services/sync/tests/unit/head_http_server.js */
+
+"use strict";
+
+Cu.import("resource://services-sync/service.js");
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://testing-common/services/sync/utils.js");
+Cu.import("resource://formautofill/ProfileStorage.jsm");
+
+let {sanitizeStorageObject, AutofillRecord, AddressesEngine} =
+  Cu.import("resource://formautofill/FormAutofillSync.jsm", {});
+
+
+Services.prefs.setCharPref("extensions.formautofill.loglevel", "Trace");
+initTestLogging("Trace");
+
+const TEST_PROFILE_1 = {
+  "given-name": "Timothy",
+  "additional-name": "John",
+  "family-name": "Berners-Lee",
+  organization: "World Wide Web Consortium",
+  "street-address": "32 Vassar Street\nMIT Room 32-G524",
+  "address-level2": "Cambridge",
+  "address-level1": "MA",
+  "postal-code": "02139",
+  country: "US",
+  tel: "+1 617 253 5702",
+  email: "timbl@w3.org",
+};
+
+const TEST_PROFILE_2 = {
+  "street-address": "Some Address",
+  country: "US",
+};
+
+function expectLocalProfiles(expected) {
+  let profiles = profileStorage.addresses.getAll({
+    includePrivateFields: true,
+    includeDeleted: true,
+  });
+  expected.sort((a, b) => a.guid.localeCompare(b.guid));
+  profiles.sort((a, b) => a.guid.localeCompare(b.guid));
+  let doCheckObject = (a, b) => {
+    for (let key of Object.keys(a)) {
+      if (typeof a[key] == "object") {
+        doCheckObject(a[key], b[key]);
+      } else {
+        equal(a[key], b[key]);
+      }
+    }
+  };
+  try {
+    deepEqual(profiles.map(p => p.guid), expected.map(p => p.guid));
+    for (let i = 0; i < expected.length; i++) {
+      let thisExpected = expected[i];
+      let thisGot = profiles[i];
+      // always check "deleted".
+      equal(thisExpected.deleted, thisGot.deleted);
+      doCheckObject(thisExpected, thisGot);
+    }
+  } catch (ex) {
+    do_print("Comparing expected profiles:");
+    do_print(JSON.stringify(expected, undefined, 2));
+    do_print("against actual profiles:");
+    do_print(JSON.stringify(profiles, undefined, 2));
+    throw ex;
+  }
+}
+
+async function setup() {
+  await profileStorage.initialize();
+  // should always start with no profiles.
+  Assert.equal(profileStorage.addresses.getAll({includeDeleted: true}).length, 0);
+
+  Services.prefs.setCharPref("services.sync.log.logger.engine.addresses", "Trace");
+  let engine = new AddressesEngine(Service);
+  // Avoid accidental automatic sync due to our own changes
+  Service.scheduler.syncThreshold = 10000000;
+  let server = serverForUsers({"foo": "password"}, {
+    meta: {global: {engines: {addresses: {version: engine.version, syncID: engine.syncID}}}},
+    addresses: {},
+  });
+
+  Service.engineManager._engines.addresses = engine;
+  engine.enabled = true;
+
+  generateNewKeys(Service.collectionKeys);
+
+  await SyncTestingInfrastructure(server);
+
+  let collection = server.user("foo").collection("addresses");
+  Service.scheduler.syncThreshold = 10000000;
+
+  return {server, collection, engine};
+}
+
+async function cleanup(server) {
+  let promiseStartOver = promiseOneObserver("weave:service:start-over:finish");
+  Service.startOver();
+  await promiseStartOver;
+  await promiseStopServer(server);
+  profileStorage.addresses._nukeAllRecords();
+}
+
+add_task(async function test_log_sanitization() {
+  await profileStorage.initialize();
+  let sanitized = sanitizeStorageObject(TEST_PROFILE_1);
+  // all strings have been mangled.
+  for (let key of Object.keys(TEST_PROFILE_1)) {
+    let val = TEST_PROFILE_1[key];
+    if (typeof val == "string") {
+      notEqual(sanitized[key], val);
+    }
+  }
+  // And check that stringifying a sync record is sanitized.
+  let record = new AutofillRecord("collection", "some-id");
+  record.entry = TEST_PROFILE_1;
+  let serialized = record.toString();
+  // None of the string values should appear in the output.
+  for (let key of Object.keys(TEST_PROFILE_1)) {
+    let val = TEST_PROFILE_1[key];
+    if (typeof val == "string") {
+      ok(!serialized.includes(val), `"${val}" shouldn't be in: ${serialized}`);
+    }
+  }
+  profileStorage.addresses._nukeAllRecords();
+});
+
+add_task(async function test_outgoing() {
+  let {server, collection, engine} = await setup();
+  try {
+    equal(engine._tracker.score, 0);
+    let existingGUID = profileStorage.addresses.add(TEST_PROFILE_1);
+    // And a deleted item.
+    let deletedGUID = profileStorage.addresses._generateGUID();
+    profileStorage.addresses.add({guid: deletedGUID, deleted: true});
+
+    expectLocalProfiles([
+      {
+        guid: existingGUID,
+      },
+      {
+        guid: deletedGUID,
+        deleted: true,
+      },
+    ]);
+
+    // The tracker should have a score recorded for the 2 additions we had.
+    equal(engine._tracker.score, SCORE_INCREMENT_XLARGE * 2);
+
+    engine.sync();
+
+    Assert.equal(collection.count(), 2);
+    Assert.ok(collection.wbo(existingGUID));
+    Assert.ok(collection.wbo(deletedGUID));
+
+    expectLocalProfiles([
+      {
+        guid: existingGUID,
+        _sync: {changeCounter: 0},
+      },
+      {
+        guid: deletedGUID,
+        _sync: {changeCounter: 0},
+        deleted: true,
+      },
+    ]);
+  } finally {
+    await cleanup(server);
+  }
+});
+
+add_task(async function test_incoming_new() {
+  let {server, engine} = await setup();
+  try {
+    let profileID = Utils.makeGUID();
+    let deletedID = Utils.makeGUID();
+
+    server.insertWBO("foo", "addresses", new ServerWBO(profileID, encryptPayload({
+      id: profileID,
+      entry: TEST_PROFILE_1,
+    }), Date.now() / 1000));
+    server.insertWBO("foo", "addresses", new ServerWBO(deletedID, encryptPayload({
+      id: deletedID,
+      deleted: true,
+    }), Date.now() / 1000));
+
+    // The tracker should start with no score.
+    equal(engine._tracker.score, 0);
+
+    engine.sync();
+
+    expectLocalProfiles([
+      {
+        guid: profileID,
+        _sync: {changeCounter: 0},
+      }, {
+        guid: deletedID,
+        _sync: {changeCounter: 0},
+        deleted: true,
+      },
+    ]);
+
+    // The sync applied new records - ensure our tracker knew it came from
+    // sync and didn't bump the score.
+    equal(engine._tracker.score, 0);
+  } finally {
+    await cleanup(server);
+  }
+});
+
+add_task(async function test_tombstones() {
+  let {server, collection, engine} = await setup();
+  try {
+    let existingGUID = profileStorage.addresses.add(TEST_PROFILE_1);
+
+    engine.sync();
+
+    Assert.equal(collection.count(), 1);
+    let payload = collection.payloads()[0];
+    equal(payload.id, existingGUID);
+    equal(payload.deleted, undefined);
+
+    profileStorage.addresses.remove(existingGUID);
+    engine.sync();
+
+    // should still exist, but now be a tombstone.
+    Assert.equal(collection.count(), 1);
+    payload = collection.payloads()[0];
+    equal(payload.id, existingGUID);
+    equal(payload.deleted, true);
+  } finally {
+    await cleanup(server);
+  }
+});
+
+// Unlike most sync engines, we want "both modified" to inspect the records,
+// and if materially different, create a duplicate.
+add_task(async function test_reconcile_both_modified_identical() {
+  let {server, engine} = await setup();
+  try {
+    // create a record locally.
+    let guid = profileStorage.addresses.add(TEST_PROFILE_1);
+
+    // and an identical record on the server.
+    server.insertWBO("foo", "addresses", new ServerWBO(guid, encryptPayload({
+      id: guid,
+      entry: TEST_PROFILE_1,
+    }), Date.now() / 1000));
+
+    engine.sync();
+
+    expectLocalProfiles([{guid}]);
+  } finally {
+    await cleanup(server);
+  }
+});
+
+add_task(async function test_dedupe_identical() {
+  let {server, engine} = await setup();
+  try {
+    // create a record locally.
+    let localGuid = profileStorage.addresses.add(TEST_PROFILE_1);
+
+    // and an identical record on the server but different GUID.
+    let remoteGuid = Utils.makeGUID();
+    server.insertWBO("foo", "addresses", new ServerWBO(remoteGuid, encryptPayload({
+      id: remoteGuid,
+      entry: TEST_PROFILE_1,
+    }), Date.now() / 1000));
+
+    engine.sync();
+
+    // Should have 1 item locally with GUID changed to the remote one, and a
+    // tombstone for the now deleted item.
+    expectLocalProfiles([
+      {
+        guid: localGuid,
+        deleted: true,
+      },
+      {
+        guid: remoteGuid,
+      },
+    ]);
+    // XXX - check tombstone on the server.
+  } finally {
+    await cleanup(server);
+  }
+});
+
+add_task(async function test_dedupe_multiple_candidates() {
+  let {server, engine} = await setup();
+  try {
+    // It's possible to have duplicate local profiles, with the same fields but
+    // different GUIDs. After a node reassignment, or after disconnecting and
+    // reconnecting to Sync, we might dedupe a local record A to a remote record
+    // B, if we see B before we download and apply A. Since A and B are dupes,
+    // that's OK. We'll write a tombstone for A when we dedupe A to B, and
+    // overwrite that tombstone when we see A.
+
+    let aRecord = {
+      "given-name": "Mark",
+      "family-name": "Hammond",
+      "organization": "Mozilla",
+      "country": "AU",
+      "tel": "123456",
+    };
+    // We don't pass `sourceSync` so that the records are marked as NEW.
+    let aGuid = profileStorage.addresses.add(aRecord);
+
+    let bRecord = Cu.cloneInto(aRecord, {});
+    let bGuid = profileStorage.addresses.add(bRecord);
+
+    // Insert B before A.
+    server.insertWBO("foo", "addresses", new ServerWBO(bGuid, encryptPayload({
+      id: bGuid,
+      entry: bRecord,
+    }), Date.now() / 1000));
+    server.insertWBO("foo", "addresses", new ServerWBO(aGuid, encryptPayload({
+      id: aGuid,
+      entry: aRecord,
+    }), Date.now() / 1000));
+
+    engine.sync();
+
+    expectLocalProfiles([
+      {
+        "guid": aGuid,
+        "given-name": "Mark",
+        "family-name": "Hammond",
+        "organization": "Mozilla",
+        "country": "AU",
+        "tel": "123456",
+      },
+      {
+        "guid": bGuid,
+        "given-name": "Mark",
+        "family-name": "Hammond",
+        "organization": "Mozilla",
+        "country": "AU",
+        "tel": "123456",
+      },
+    ]);
+
+    // TODO: Check Sync fields, verify they're both SYNCING.
+  } finally {
+    await cleanup(server);
+  }
+});
+
+// Unlike most sync engines, we want "both modified" to inspect the records,
+// and if materially different, create a duplicate.
+add_task(async function test_reconcile_both_modified_conflict() {
+  let {server, engine} = await setup();
+  try {
+    // create a record locally.
+    let guid = profileStorage.addresses.add(TEST_PROFILE_1);
+
+    // clone the profile and adjust something for storing on the server.
+    let serverCopy = Object.assign({}, TEST_PROFILE_1);
+    serverCopy["street-address"] = "I moved!";
+
+    server.insertWBO("foo", "addresses", new ServerWBO(guid, encryptPayload({
+      id: guid,
+      entry: serverCopy,
+    }), Date.now() / 1000));
+
+    engine.sync();
+
+    // TODO: check semantics - (ie, remote copy should have same ID, local
+    // copy should have had the GUID changed and be marked as changed.)
+    // XXX - call expectLocalProfiles - but this doesn't work yet.
+    // XXX - FIXME!
+  } finally {
+    await cleanup(server);
+  }
+});
--- a/browser/extensions/formautofill/test/unit/xpcshell.ini
+++ b/browser/extensions/formautofill/test/unit/xpcshell.ini
@@ -31,8 +31,10 @@ support-files =
 [test_markAsAutofillField.js]
 [test_nameUtils.js]
 [test_onFormSubmitted.js]
 [test_profileAutocompleteResult.js]
 [test_savedFieldNames.js]
 [test_storage_tombstones.js]
 [test_storage_syncfields.js]
 [test_transformFields.js]
+[test_sync.js]
+head = head.js ../../../../../services/sync/tests/unit/head_appinfo.js ../../../../../services/common/tests/unit/head_helpers.js ../../../../../services/sync/tests/unit/head_helpers.js ../../../../../services/sync/tests/unit/head_http_server.js
--- a/services/sync/modules/engines.js
+++ b/services/sync/modules/engines.js
@@ -813,16 +813,19 @@ SyncEngine.prototype = {
 
   // How many records to pull at one time when specifying IDs. This is to avoid
   // URI length limitations.
   guidFetchBatchSize: DEFAULT_GUID_FETCH_BATCH_SIZE,
 
   // How many records to process in a single batch.
   applyIncomingBatchSize: DEFAULT_STORE_BATCH_SIZE,
 
+  // should incoming tombstones be created locally?
+  applyIncomingTombstones: false,
+
   get storageURL() {
     return this.service.storageURL;
   },
 
   get engineURL() {
     return this.storageURL + this.name;
   },
 
@@ -1441,17 +1444,17 @@ SyncEngine.prototype = {
                     remoteAge);
 
     // We handle deletions first so subsequent logic doesn't have to check
     // deleted flags.
     if (item.deleted) {
       // If the item doesn't exist locally, there is nothing for us to do. We
       // can't check for duplicates because the incoming record has no data
       // which can be used for duplicate detection.
-      if (!existsLocally) {
+      if (!existsLocally && !this.applyIncomingTombstones) {
         this._log.trace("Ignoring incoming item because it was deleted and " +
                         "the item does not exist locally.");
         return false;
       }
 
       // We decide whether to process the deletion by comparing the record
       // ages. If the item is not modified locally, the remote side wins and
       // the deletion is processed. If it is modified locally, we take the
--- a/services/sync/modules/record.js
+++ b/services/sync/modules/record.js
@@ -178,18 +178,22 @@ CryptoWrapper.prototype = {
 
     // Verify that the encrypted id matches the requested record's id.
     if (this.cleartext.id != this.id)
       throw "Record id mismatch: " + this.cleartext.id + " != " + this.id;
 
     return this.cleartext;
   },
 
+  cleartextToString() {
+    return JSON.stringify(this.cleartext);
+  },
+
   toString: function toString() {
-    let payload = this.deleted ? "DELETED" : JSON.stringify(this.cleartext);
+    let payload = this.deleted ? "DELETED" : this.cleartextToString();
 
     return "{ " +
       "id: " + this.id + "  " +
       "index: " + this.sortindex + "  " +
       "modified: " + this.modified + "  " +
       "ttl: " + this.ttl + "  " +
       "payload: " + payload + "  " +
       "collection: " + (this.collection || "undefined") +
--- a/services/sync/modules/service.js
+++ b/services/sync/modules/service.js
@@ -31,25 +31,41 @@ Cu.import("resource://services-sync/reco
 Cu.import("resource://services-sync/resource.js");
 Cu.import("resource://services-sync/rest.js");
 Cu.import("resource://services-sync/stages/enginesync.js");
 Cu.import("resource://services-sync/stages/declined.js");
 Cu.import("resource://services-sync/status.js");
 Cu.import("resource://services-sync/telemetry.js");
 Cu.import("resource://services-sync/util.js");
 
-const ENGINE_MODULES = {
-  Addons: {module: "addons.js", symbol: "AddonsEngine"},
-  Bookmarks: {module: "bookmarks.js", symbol: "BookmarksEngine"},
-  Form: {module: "forms.js", symbol: "FormEngine"},
-  History: {module: "history.js", symbol: "HistoryEngine"},
-  Password: {module: "passwords.js", symbol: "PasswordEngine"},
-  Prefs: {module: "prefs.js", symbol: "PrefsEngine"},
-  Tab: {module: "tabs.js", symbol: "TabEngine"},
-  ExtensionStorage: {module: "extension-storage.js", symbol: "ExtensionStorageEngine"},
+function getEngineModules() {
+  let result = {
+    Addons: {module: "addons.js", symbol: "AddonsEngine"},
+    Bookmarks: {module: "bookmarks.js", symbol: "BookmarksEngine"},
+    Form: {module: "forms.js", symbol: "FormEngine"},
+    History: {module: "history.js", symbol: "HistoryEngine"},
+    Password: {module: "passwords.js", symbol: "PasswordEngine"},
+    Prefs: {module: "prefs.js", symbol: "PrefsEngine"},
+    Tab: {module: "tabs.js", symbol: "TabEngine"},
+    ExtensionStorage: {module: "extension-storage.js", symbol: "ExtensionStorageEngine"},
+  }
+
+  if (Svc.Prefs.get("engine.addresses.available", false)) {
+    result["Addresses"] = {
+      module: "resource://formautofill/FormAutofillSync.jsm",
+      symbol: "AddressesEngine",
+    };
+  }
+  if (Svc.Prefs.get("engine.creditcards.available", false)) {
+    result["CreditCards"] = {
+      module: "resource://formautofill/FormAutofillSync.jsm",
+      symbol: "CreditCardsEngine",
+    };
+  }
+  return result;
 };
 
 const STORAGE_INFO_TYPES = [INFO_COLLECTIONS,
                             INFO_COLLECTION_USAGE,
                             INFO_COLLECTION_COUNTS,
                             INFO_QUOTA];
 
 // A unique identifier for this browser session. Used for logging so
@@ -350,41 +366,43 @@ Sync11Service.prototype = {
   },
 
   /**
    * Register the built-in engines for certain applications
    */
   _registerEngines: function _registerEngines() {
     this.engineManager = new EngineManager(this);
 
+    let engineModules = getEngineModules();
+
     let engines = [];
     // We allow a pref, which has no default value, to limit the engines
     // which are registered. We expect only tests will use this.
     let pref = Svc.Prefs.get("registerEngines");
     if (pref) {
       engines = pref.split(",");
     } else {
       // default is all engines.
-      engines = Object.keys(ENGINE_MODULES);
+      engines = Object.keys(engineModules);
     }
 
     let declined = [];
     pref = Svc.Prefs.get("declinedEngines");
     if (pref) {
       declined = pref.split(",");
     }
 
     this.clientsEngine = new ClientEngine(this);
 
     for (let name of engines) {
-      if (!(name in ENGINE_MODULES)) {
+      if (!(name in engineModules)) {
         this._log.info("Do not know about engine: " + name);
         continue;
       }
-      let {module, symbol} = ENGINE_MODULES[name];
+      let {module, symbol} = engineModules[name];
       if (!module.includes(":")) {
         module = "resource://services-sync/engines/" + module;
       }
       let ns = {};
       try {
         Cu.import(module, ns);
         if (!(symbol in ns)) {
           this._log.warn("Could not find exported engine instance: " + symbol);
--- a/services/sync/modules/telemetry.js
+++ b/services/sync/modules/telemetry.js
@@ -55,17 +55,18 @@ const TOPICS = [
 ];
 
 const PING_FORMAT_VERSION = 1;
 
 const EMPTY_UID = "0".repeat(32);
 
 // The set of engines we record telemetry for - any other engines are ignored.
 const ENGINES = new Set(["addons", "bookmarks", "clients", "forms", "history",
-                         "passwords", "prefs", "tabs", "extension-storage"]);
+                         "passwords", "prefs", "tabs", "extension-storage",
+                         "addresses", "creditcards"]);
 
 // A regex we can use to replace the profile dir in error messages. We use a
 // regexp so we can simply replace all case-insensitive occurences.
 // This escaping function is from:
 // https://developer.mozilla.org/en/docs/Web/JavaScript/Guide/Regular_Expressions
 const reProfileDir = new RegExp(
         OS.Constants.Path.profileDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"),
         "gi");
--- a/services/sync/services-sync.js
+++ b/services/sync/services-sync.js
@@ -12,24 +12,35 @@ pref("services.sync.scheduler.idleInterv
 pref("services.sync.scheduler.activeInterval", 600);   // 10 minutes
 pref("services.sync.scheduler.immediateInterval", 90);    // 1.5 minutes
 pref("services.sync.scheduler.idleTime", 300);   // 5 minutes
 
 pref("services.sync.scheduler.fxa.singleDeviceInterval", 3600); // 1 hour
 
 pref("services.sync.errorhandler.networkFailureReportTimeout", 1209600); // 2 weeks
 
+// Note that new engines are typically added with a default of disabled, so
+// when an existing sync user get the Firefox upgrade that supports the engine
+// is starts as disabled until the user has explicitly opted in.
+// The sync "create account" process typically *will* offer these engines, so
+// they may be flipped to enabled at that time.
 pref("services.sync.engine.addons", true);
+pref("services.sync.engine.addresses", false);
 pref("services.sync.engine.bookmarks", true);
+pref("services.sync.engine.creditcards", false);
 pref("services.sync.engine.history", true);
 pref("services.sync.engine.passwords", true);
 pref("services.sync.engine.prefs", true);
 pref("services.sync.engine.tabs", true);
 pref("services.sync.engine.tabs.filteredUrls", "^(about:.*|resource:.*|chrome:.*|wyciwyg:.*|file:.*|blob:.*)$");
 
+// The addresses and CC engines might not actually be available at all.
+pref("services.sync.engine.addresses.available", false);
+pref("services.sync.engine.creditcards.available", false);
+
 // If true, add-on sync ignores changes to the user-enabled flag. This
 // allows people to have the same set of add-ons installed across all
 // profiles while maintaining different enabled states.
 pref("services.sync.addons.ignoreUserEnabledChanges", false);
 
 // Comma-delimited list of hostnames to trust for add-on install.
 pref("services.sync.addons.trustedSourceHostnames", "addons.mozilla.org");
 
@@ -53,16 +64,18 @@ pref("services.sync.log.logger.network.r
 pref("services.sync.log.logger.engine.bookmarks", "Debug");
 pref("services.sync.log.logger.engine.clients", "Debug");
 pref("services.sync.log.logger.engine.forms", "Debug");
 pref("services.sync.log.logger.engine.history", "Debug");
 pref("services.sync.log.logger.engine.passwords", "Debug");
 pref("services.sync.log.logger.engine.prefs", "Debug");
 pref("services.sync.log.logger.engine.tabs", "Debug");
 pref("services.sync.log.logger.engine.addons", "Debug");
+pref("services.sync.log.logger.engine.addresses", "Debug");
+pref("services.sync.log.logger.engine.creditcards", "Debug");
 pref("services.sync.log.logger.engine.extension-storage", "Debug");
 pref("services.sync.log.logger.engine.apps", "Debug");
 pref("services.sync.log.logger.identity", "Debug");
 pref("services.sync.log.cryptoDebug", false);
 
 pref("services.sync.fxa.termsURL", "https://accounts.firefox.com/legal/terms");
 pref("services.sync.fxa.privacyURL", "https://accounts.firefox.com/legal/privacy");
 
--- a/tools/lint/eslint/modules.json
+++ b/tools/lint/eslint/modules.json
@@ -72,16 +72,17 @@
   "fakeservices.js": ["FakeCryptoService", "FakeFilesystemService", "FakeGUIDService", "fakeSHA256HMAC"],
   "file_expandosharing.jsm": ["checkFromJSM"],
   "file_stringencoding.jsm": ["checkFromJSM"],
   "file_url.jsm": ["checkFromJSM"],
   "file_worker_url.jsm": ["checkFromJSM"],
   "Finder.jsm": ["Finder", "GetClipboardSearchString"],
   "forms.js": ["FormEngine", "FormRec", "FormValidator"],
   "forms.jsm": ["FormData"],
+  "FormAutofillSync.jsm": ["AddressesEngine", "CreditCardsEngine"],
   "frame.js": ["Collector", "Runner", "events", "runTestFile", "log", "timers", "persisted", "shutdownApplication"],
   "FrameScriptManager.jsm": ["getNewLoaderID"],
   "fxa_utils.js": ["initializeIdentityWithTokenServerResponse"],
   "fxaccounts.jsm": ["Authentication"],
   "FxAccounts.jsm": ["fxAccounts", "FxAccounts"],
   "FxAccountsCommon.js": ["log", "logPII", "FXACCOUNTS_PERMISSION", "DATA_FORMAT_VERSION", "DEFAULT_STORAGE_FILENAME", "ASSERTION_LIFETIME", "ASSERTION_USE_PERIOD", "CERT_LIFETIME", "KEY_LIFETIME", "POLL_SESSION", "ONLOGIN_NOTIFICATION", "ONVERIFIED_NOTIFICATION", "ONLOGOUT_NOTIFICATION", "ON_FXA_UPDATE_NOTIFICATION", "ON_DEVICE_CONNECTED_NOTIFICATION", "ON_DEVICE_DISCONNECTED_NOTIFICATION", "ON_PROFILE_UPDATED_NOTIFICATION", "ON_PASSWORD_CHANGED_NOTIFICATION", "ON_PASSWORD_RESET_NOTIFICATION", "ON_ACCOUNT_DESTROYED_NOTIFICATION", "ON_COLLECTION_CHANGED_NOTIFICATION", "FXA_PUSH_SCOPE_ACCOUNT_UPDATE", "ON_PROFILE_CHANGE_NOTIFICATION", "ON_ACCOUNT_STATE_CHANGE_NOTIFICATION", "UI_REQUEST_SIGN_IN_FLOW", "UI_REQUEST_REFRESH_AUTH", "FX_OAUTH_CLIENT_ID", "WEBCHANNEL_ID", "ERRNO_ACCOUNT_ALREADY_EXISTS", "ERRNO_ACCOUNT_DOES_NOT_EXIST", "ERRNO_INCORRECT_PASSWORD", "ERRNO_UNVERIFIED_ACCOUNT", "ERRNO_INVALID_VERIFICATION_CODE", "ERRNO_NOT_VALID_JSON_BODY", "ERRNO_INVALID_BODY_PARAMETERS", "ERRNO_MISSING_BODY_PARAMETERS", "ERRNO_INVALID_REQUEST_SIGNATURE", "ERRNO_INVALID_AUTH_TOKEN", "ERRNO_INVALID_AUTH_TIMESTAMP", "ERRNO_MISSING_CONTENT_LENGTH", "ERRNO_REQUEST_BODY_TOO_LARGE", "ERRNO_TOO_MANY_CLIENT_REQUESTS", "ERRNO_INVALID_AUTH_NONCE", "ERRNO_ENDPOINT_NO_LONGER_SUPPORTED", "ERRNO_INCORRECT_LOGIN_METHOD", "ERRNO_INCORRECT_KEY_RETRIEVAL_METHOD", "ERRNO_INCORRECT_API_VERSION", "ERRNO_INCORRECT_EMAIL_CASE", "ERRNO_ACCOUNT_LOCKED", "ERRNO_ACCOUNT_UNLOCKED", "ERRNO_UNKNOWN_DEVICE", "ERRNO_DEVICE_SESSION_CONFLICT", "ERRNO_SERVICE_TEMP_UNAVAILABLE", "ERRNO_PARSE", "ERRNO_NETWORK", "ERRNO_UNKNOWN_ERROR", "OAUTH_SERVER_ERRNO_OFFSET", "ERRNO_UNKNOWN_CLIENT_ID", "ERRNO_INCORRECT_CLIENT_SECRET", "ERRNO_INCORRECT_REDIRECT_URI", "ERRNO_INVALID_FXA_ASSERTION", "ERRNO_UNKNOWN_CODE", "ERRNO_INCORRECT_CODE", "ERRNO_EXPIRED_CODE", "ERRNO_OAUTH_INVALID_TOKEN", "ERRNO_INVALID_REQUEST_PARAM", "ERRNO_INVALID_RESPONSE_TYPE", "ERRNO_UNAUTHORIZED", "ERRNO_FORBIDDEN", "ERRNO_INVALID_CONTENT_TYPE", "ERROR_ACCOUNT_ALREADY_EXISTS", "ERROR_ACCOUNT_DOES_NOT_EXIST", "ERROR_ACCOUNT_LOCKED", "ERROR_ACCOUNT_UNLOCKED", "ERROR_ALREADY_SIGNED_IN_USER", "ERROR_DEVICE_SESSION_CONFLICT", "ERROR_ENDPOINT_NO_LONGER_SUPPORTED", "ERROR_INCORRECT_API_VERSION", "ERROR_INCORRECT_EMAIL_CASE", "ERROR_INCORRECT_KEY_RETRIEVAL_METHOD", "ERROR_INCORRECT_LOGIN_METHOD", "ERROR_INVALID_EMAIL", "ERROR_INVALID_AUDIENCE", "ERROR_INVALID_AUTH_TOKEN", "ERROR_INVALID_AUTH_TIMESTAMP", "ERROR_INVALID_AUTH_NONCE", "ERROR_INVALID_BODY_PARAMETERS", "ERROR_INVALID_PASSWORD", "ERROR_INVALID_VERIFICATION_CODE", "ERROR_INVALID_REFRESH_AUTH_VALUE", "ERROR_INVALID_REQUEST_SIGNATURE", "ERROR_INTERNAL_INVALID_USER", "ERROR_MISSING_BODY_PARAMETERS", "ERROR_MISSING_CONTENT_LENGTH", "ERROR_NO_TOKEN_SESSION", "ERROR_NO_SILENT_REFRESH_AUTH", "ERROR_NOT_VALID_JSON_BODY", "ERROR_OFFLINE", "ERROR_PERMISSION_DENIED", "ERROR_REQUEST_BODY_TOO_LARGE", "ERROR_SERVER_ERROR", "ERROR_SYNC_DISABLED", "ERROR_TOO_MANY_CLIENT_REQUESTS", "ERROR_SERVICE_TEMP_UNAVAILABLE", "ERROR_UI_ERROR", "ERROR_UI_REQUEST", "ERROR_PARSE", "ERROR_NETWORK", "ERROR_UNKNOWN", "ERROR_UNKNOWN_DEVICE", "ERROR_UNVERIFIED_ACCOUNT", "ERROR_UNKNOWN_CLIENT_ID", "ERROR_INCORRECT_CLIENT_SECRET", "ERROR_INCORRECT_REDIRECT_URI", "ERROR_INVALID_FXA_ASSERTION", "ERROR_UNKNOWN_CODE", "ERROR_INCORRECT_CODE", "ERROR_EXPIRED_CODE", "ERROR_OAUTH_INVALID_TOKEN", "ERROR_INVALID_REQUEST_PARAM", "ERROR_INVALID_RESPONSE_TYPE", "ERROR_UNAUTHORIZED", "ERROR_FORBIDDEN", "ERROR_INVALID_CONTENT_TYPE", "ERROR_NO_ACCOUNT", "ERROR_AUTH_ERROR", "ERROR_INVALID_PARAMETER", "ERROR_CODE_METHOD_NOT_ALLOWED", "ERROR_MSG_METHOD_NOT_ALLOWED", "FXA_PWDMGR_PLAINTEXT_FIELDS", "FXA_PWDMGR_SECURE_FIELDS", "FXA_PWDMGR_MEMORY_FIELDS", "FXA_PWDMGR_REAUTH_WHITELIST", "FXA_PWDMGR_HOST", "FXA_PWDMGR_REALM", "SERVER_ERRNO_TO_ERROR", "ERROR_TO_GENERAL_ERROR_CLASS"],
   "FxAccountsOAuthGrantClient.jsm": ["FxAccountsOAuthGrantClient", "FxAccountsOAuthGrantClientError"],
   "FxAccountsProfileClient.jsm": ["FxAccountsProfileClient", "FxAccountsProfileClientError"],