Bug 625684 - Merge fx-sync to mozilla-central. a=blockers
authorPhilipp von Weitershausen <philipp@weitershausen.de>
Fri, 14 Jan 2011 14:44:53 -0800
changeset 60628 9091ba1594f883eaea9c5d8c3c79dc5d9fc5de85
parent 60623 9ab122e89d6257518b340bc37874987f16952acf (current diff)
parent 60627 f5d5f9b2defa39a415d3fbc6490e6ed8d6f6e374 (diff)
child 60629 d3675348f88b5f5a6d9a6ae782a6679cb865f986
push id1
push userroot
push dateTue, 26 Apr 2011 22:38:44 +0000
treeherdermozilla-beta@bfdb6e623a36 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersblockers
bugs625684
milestone2.0b10pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 625684 - Merge fx-sync to mozilla-central. a=blockers
services/sync/modules/base_records/crypto.js
services/sync/modules/engines/bookmarks.js
services/sync/tests/unit/head_http_server.js
services/sync/tests/unit/test_bookmark_store.js
services/sync/tests/unit/test_service_verifyLogin.js
--- a/services/sync/modules/base_records/crypto.js
+++ b/services/sync/modules/base_records/crypto.js
@@ -124,16 +124,17 @@ CryptoWrapper.prototype = {
 
     return this.cleartext;
   },
 
   toString: function CryptoWrap_toString() "{ " + [
       "id: " + this.id,
       "index: " + this.sortindex,
       "modified: " + this.modified,
+      "ttl: " + this.ttl,
       "payload: " + (this.deleted ? "DELETED" : JSON.stringify(this.cleartext)),
       "collection: " + (this.collection || "undefined")
     ].join("\n  ") + " }",
 
   // The custom setter below masks the parent's getter, so explicitly call it :(
   get id() WBORecord.prototype.__lookupGetter__("id").call(this),
 
   // Keep both plaintext and encrypted versions of the id to verify integrity
--- a/services/sync/modules/base_records/wbo.js
+++ b/services/sync/modules/base_records/wbo.js
@@ -93,23 +93,26 @@ WBORecord.prototype = {
     } catch(ex) {}
   },
 
   toJSON: function toJSON() {
     // Copy fields from data to be stringified, making sure payload is a string
     let obj = {};
     for (let [key, val] in Iterator(this.data))
       obj[key] = key == "payload" ? JSON.stringify(val) : val;
+    if (this.ttl)
+      obj.ttl = this.ttl;
     return obj;
   },
 
   toString: function WBORec_toString() "{ " + [
       "id: " + this.id,
       "index: " + this.sortindex,
       "modified: " + this.modified,
+      "ttl: " + this.ttl,
       "payload: " + JSON.stringify(this.payload)
     ].join("\n  ") + " }",
 };
 
 Utils.deferGetSet(WBORecord, "data", ["id", "modified", "sortindex", "payload"]);
 
 Utils.lazy(this, 'Records', RecordManager);
 
--- a/services/sync/modules/engines/bookmarks.js
+++ b/services/sync/modules/engines/bookmarks.js
@@ -320,16 +320,22 @@ BookmarksStore.prototype = {
   },
 
 
   itemExists: function BStore_itemExists(id) {
     return this.idForGUID(id) > 0;
   },
 
   applyIncoming: function BStore_applyIncoming(record) {
+    // Don't bother with pre and post-processing for deletions.
+    if (record.deleted) {
+      Store.prototype.applyIncoming.apply(this, arguments);
+      return;
+    }
+
     // For special folders we're only interested in child ordering.
     if ((record.id in kSpecialIds) && record.children) {
       this._log.debug("Processing special node: " + record.id);
       // Reorder children later
       this._childrenToOrder[record.id] = record.children;
       return;
     }
 
@@ -362,46 +368,46 @@ BookmarksStore.prototype = {
           }
         }
         break;
       }
     }
 
     // Figure out the local id of the parent GUID if available
     let parentGUID = record.parentid;
-    record._orphan = false;
-    if (parentGUID != null) {
-      let parentId = this.idForGUID(parentGUID);
+    if (!parentGUID) {
+      throw "Record " + record.id + " has invalid parentid: " + parentGUID;
+    }
 
-      // Default to unfiled if we don't have the parent yet
-      if (parentId <= 0) {
-        this._log.trace("Reparenting to unfiled until parent is synced");
-        record._orphan = true;
-        parentId = kSpecialIds.unfiled;
-      }
-
+    let parentId = this.idForGUID(parentGUID);
+    if (parentId > 0) {
       // Save the parent id for modifying the bookmark later
       record._parent = parentId;
+      record._orphan = false;
+    } else {
+      this._log.trace("Record " + record.id +
+                      " is an orphan: could not find parent " + parentGUID);
+      record._orphan = true;
     }
 
     // Do the normal processing of incoming records
     Store.prototype.applyIncoming.apply(this, arguments);
 
     // Do some post-processing if we have an item
     let itemId = this.idForGUID(record.id);
     if (itemId > 0) {
       // Move any children that are looking for this folder as a parent
       if (record.type == "folder") {
         this._reparentOrphans(itemId);
         // Reorder children later
         if (record.children)
           this._childrenToOrder[record.id] = record.children;
       }
 
-      // Create an annotation to remember that it needs a parent
+      // Create an annotation to remember that it needs reparenting.
       if (record._orphan)
         Utils.anno(itemId, PARENT_ANNO, parentGUID);
     }
   },
 
   /**
    * Find all ids of items that have a given value for an annotation
    */
@@ -416,34 +422,56 @@ BookmarksStore.prototype = {
   _reparentOrphans: function _reparentOrphans(parentId) {
     // Find orphans and reunite with this folder parent
     let parentGUID = this.GUIDForId(parentId);
     let orphans = this._findAnnoItems(PARENT_ANNO, parentGUID);
 
     this._log.debug("Reparenting orphans " + orphans + " to " + parentId);
     orphans.forEach(function(orphan) {
       // Move the orphan to the parent and drop the missing parent annotation
-      Svc.Bookmark.moveItem(orphan, parentId, Svc.Bookmark.DEFAULT_INDEX);
-      Svc.Annos.removeItemAnnotation(orphan, PARENT_ANNO);
+      if (this._reparentItem(orphan, parentId)) {
+        Svc.Annos.removeItemAnnotation(orphan, PARENT_ANNO);
+      }
     }, this);
   },
 
+  _reparentItem: function _reparentItem(itemId, parentId) {
+    this._log.trace("Attempting to move item " + itemId + " to new parent " +
+                    parentId);
+    try {
+      if (parentId > 0) {
+        Svc.Bookmark.moveItem(itemId, parentId, Svc.Bookmark.DEFAULT_INDEX);
+        return true;
+      }
+    } catch(ex) {
+      this._log.debug("Failed to reparent item. " + Utils.exceptionStr(ex));
+    }
+    return false;
+  },
+
   create: function BStore_create(record) {
+    // Default to unfiled if we don't have the parent yet
+    if (!record._parent) {
+      record._parent = kSpecialIds.unfiled;
+    }
+
     let newId;
     switch (record.type) {
     case "bookmark":
     case "query":
     case "microsummary": {
       let uri = Utils.makeURI(record.bmkUri);
       newId = this._bms.insertBookmark(record._parent, uri,
                                        Svc.Bookmark.DEFAULT_INDEX, record.title);
       this._log.debug(["created bookmark", newId, "under", record._parent,
                        "as", record.title, record.bmkUri].join(" "));
 
-      this._tagURI(uri, record.tags);
+      if (Utils.isArray(record.tags)) {
+        this._tagURI(uri, record.tags);
+      }
       this._bms.setKeywordForBookmark(newId, record.keyword);
       if (record.description)
         Utils.anno(newId, "bookmarkProperties/description", record.description);
 
       if (record.loadInSidebar)
         Utils.anno(newId, "bookmarkProperties/loadInSidebar", true);
 
       if (record.type == "microsummary") {
@@ -535,36 +563,40 @@ BookmarksStore.prototype = {
     if (itemId <= 0) {
       this._log.debug("Skipping update for unknown item: " + record.id);
       return;
     }
 
     this._log.trace("Updating " + record.id + " (" + itemId + ")");
 
     // Move the bookmark to a new parent or new position if necessary
-    if (Svc.Bookmark.getFolderIdForItem(itemId) != record._parent) {
-      this._log.trace("Moving item to a new parent.");
-      Svc.Bookmark.moveItem(itemId, record._parent, Svc.Bookmark.DEFAULT_INDEX);
+    if (record._parent > 0 &&
+        Svc.Bookmark.getFolderIdForItem(itemId) != record._parent) {
+      this._reparentItem(itemId, record._parent);
     }
 
     for (let [key, val] in Iterator(record.cleartext)) {
       switch (key) {
       case "title":
+        val = val || "";
         this._bms.setItemTitle(itemId, val);
         break;
       case "bmkUri":
         this._bms.changeBookmarkURI(itemId, Utils.makeURI(val));
         break;
       case "tags":
-        this._tagURI(this._bms.getBookmarkURI(itemId), val);
+        if (Utils.isArray(val)) {
+          this._tagURI(this._bms.getBookmarkURI(itemId), val);
+        }
         break;
       case "keyword":
         this._bms.setKeywordForBookmark(itemId, val);
         break;
       case "description":
+        val = val || "";
         Utils.anno(itemId, "bookmarkProperties/description", val);
         break;
       case "loadInSidebar":
         if (val)
           Utils.anno(itemId, "bookmarkProperties/loadInSidebar", true);
         else
           Svc.Annos.removeItemAnnotation(itemId, "bookmarkProperties/loadInSidebar");
         break;
@@ -639,17 +671,17 @@ BookmarksStore.prototype = {
     }
     return this._ts.getTagsForURI(uri, {});
   },
 
   _getDescription: function BStore__getDescription(id) {
     try {
       return Utils.anno(id, "bookmarkProperties/description");
     } catch (e) {
-      return undefined;
+      return null;
     }
   },
 
   _isLoadInSidebar: function BStore__isLoadInSidebar(id) {
     return Svc.Annos.itemHasAnnotation(id, "bookmarkProperties/loadInSidebar");
   },
 
   _getStaticTitle: function BStore__getStaticTitle(id) {
--- a/services/sync/modules/engines/clients.js
+++ b/services/sync/modules/engines/clients.js
@@ -42,32 +42,41 @@ const Cu = Components.utils;
 
 Cu.import("resource://services-sync/constants.js");
 Cu.import("resource://services-sync/engines.js");
 Cu.import("resource://services-sync/ext/StringBundle.js");
 Cu.import("resource://services-sync/stores.js");
 Cu.import("resource://services-sync/type_records/clients.js");
 Cu.import("resource://services-sync/util.js");
 
+const CLIENTS_TTL_REFRESH = 604800; // 7 days
+
 Utils.lazy(this, "Clients", ClientEngine);
 
 function ClientEngine() {
   SyncEngine.call(this, "Clients");
 
   // Reset the client on every startup so that we fetch recent clients
   this._resetClient();
 }
 ClientEngine.prototype = {
   __proto__: SyncEngine.prototype,
   _storeObj: ClientStore,
   _recordObj: ClientsRec,
 
   // Always sync client data as it controls other sync behavior
   get enabled() true,
 
+  get lastRecordUpload() {
+    return Svc.Prefs.get(this.name + ".lastRecordUpload", 0);
+  },
+  set lastRecordUpload(value) {
+    Svc.Prefs.set(this.name + ".lastRecordUpload", Math.floor(value));
+  },
+
   // Aggregate some stats on the composition of clients on this account
   get stats() {
     let stats = {
       hasMobile: this.localType == "mobile",
       names: [this.localName],
       numClients: 1,
     };
 
@@ -145,16 +154,25 @@ ClientEngine.prototype = {
   set localType(value) Svc.Prefs.set("client.type", value),
 
   isMobile: function isMobile(id) {
     if (this._store._remoteClients[id])
       return this._store._remoteClients[id].type == "mobile";
     return false;
   },
 
+  _syncStartup: function _syncStartup() {
+    // Reupload new client record periodically.
+    if (Date.now() / 1000 - this.lastRecordUpload > CLIENTS_TTL_REFRESH) {
+      this._tracker.addChangedID(this.localID);
+      this.lastRecordUpload = Date.now() / 1000;
+    }
+    SyncEngine.prototype._syncStartup.call(this);
+  },
+
   // Always process incoming items because they might have commands
   _reconcile: function _reconcile() {
     return true;
   },
 
   // Treat reset the same as wiping for locally cached clients
   _resetClient: function _resetClient() this._wipeClient(),
 
--- a/services/sync/modules/type_records/clients.js
+++ b/services/sync/modules/type_records/clients.js
@@ -39,17 +39,20 @@ const EXPORTED_SYMBOLS = ["ClientsRec"];
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cr = Components.results;
 const Cu = Components.utils;
 
 Cu.import("resource://services-sync/base_records/crypto.js");
 Cu.import("resource://services-sync/util.js");
 
+const CLIENTS_TTL = 1814400; // 21 days
+
 function ClientsRec(collection, id) {
   CryptoWrapper.call(this, collection, id);
 }
 ClientsRec.prototype = {
   __proto__: CryptoWrapper.prototype,
   _logName: "Record.Clients",
+  ttl: CLIENTS_TTL
 };
 
 Utils.deferGetSet(ClientsRec, "cleartext", ["name", "type", "commands"]);
--- a/services/sync/modules/type_records/forms.js
+++ b/services/sync/modules/type_records/forms.js
@@ -39,17 +39,20 @@ const EXPORTED_SYMBOLS = ['FormRec'];
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cr = Components.results;
 const Cu = Components.utils;
 
 Cu.import("resource://services-sync/base_records/crypto.js");
 Cu.import("resource://services-sync/util.js");
 
+const FORMS_TTL = 5184000; // 60 days
+
 function FormRec(collection, id) {
   CryptoWrapper.call(this, collection, id);
 }
 FormRec.prototype = {
   __proto__: CryptoWrapper.prototype,
   _logName: "Record.Form",
+  ttl: FORMS_TTL
 };
 
 Utils.deferGetSet(FormRec, "cleartext", ["name", "value"]);
--- a/services/sync/modules/type_records/history.js
+++ b/services/sync/modules/type_records/history.js
@@ -39,17 +39,20 @@ const EXPORTED_SYMBOLS = ['HistoryRec'];
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cr = Components.results;
 const Cu = Components.utils;
 
 Cu.import("resource://services-sync/base_records/crypto.js");
 Cu.import("resource://services-sync/util.js");
 
+const HISTORY_TTL = 5184000; // 60 days
+
 function HistoryRec(collection, id) {
   CryptoWrapper.call(this, collection, id);
 }
 HistoryRec.prototype = {
   __proto__: CryptoWrapper.prototype,
   _logName: "Record.History",
+  ttl: HISTORY_TTL
 };
 
 Utils.deferGetSet(HistoryRec, "cleartext", ["histUri", "title", "visits"]);
--- a/services/sync/modules/type_records/tabs.js
+++ b/services/sync/modules/type_records/tabs.js
@@ -39,17 +39,20 @@ const EXPORTED_SYMBOLS = ['TabSetRecord'
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cr = Components.results;
 const Cu = Components.utils;
 
 Cu.import("resource://services-sync/base_records/crypto.js");
 Cu.import("resource://services-sync/util.js");
 
+const TABS_TTL = 604800; // 7 days
+
 function TabSetRecord(collection, id) {
   CryptoWrapper.call(this, collection, id);
 }
 TabSetRecord.prototype = {
   __proto__: CryptoWrapper.prototype,
   _logName: "Record.Tabs",
+  ttl: TABS_TTL
 };
 
 Utils.deferGetSet(TabSetRecord, "cleartext", ["clientName", "tabs"]);
--- a/services/sync/tests/unit/head_http_server.js
+++ b/services/sync/tests/unit/head_http_server.js
@@ -2,16 +2,23 @@ function httpd_setup (handlers) {
   let server = new nsHttpServer();
   for (let path in handlers) {
     server.registerPathHandler(path, handlers[path]);
   }
   server.start(8080);
   return server;
 }
 
+function httpd_handler(statusCode, status, body) {
+  return function(request, response) {
+    response.setStatusLine(request.httpVersion, statusCode, status);
+    response.bodyOutputStream.write(body, body.length);
+  };
+}
+
 function httpd_basic_auth_handler(body, metadata, response) {
   // no btoa() in xpcshell.  it's guest:guest
   if (metadata.hasHeader("Authorization") &&
       metadata.getHeader("Authorization") == "Basic Z3Vlc3Q6Z3Vlc3Q=") {
     response.setStatusLine(metadata.httpVersion, 200, "OK, authorized");
     response.setHeader("WWW-Authenticate", 'Basic realm="secret"', false);
   } else {
     body = "This path exists and is protected - failed";
@@ -118,18 +125,19 @@ ServerWBO.prototype = {
  * you need to register their handlers with the server separately!
  */
 function ServerCollection(wbos) {
   this.wbos = wbos || {};
 }
 ServerCollection.prototype = {
 
   _inResultSet: function(wbo, options) {
-    return ((!options.ids || (options.ids.indexOf(wbo.id) != -1))
-            && (!options.newer || (wbo.modified > options.newer)));
+    return wbo.payload
+           && (!options.ids || (options.ids.indexOf(wbo.id) != -1))
+           && (!options.newer || (wbo.modified > options.newer));
   },
 
   get: function(options) {
     let result;
     if (options.full) {
       let data = [wbo.get() for ([id, wbo] in Iterator(this.wbos))
                             if (this._inResultSet(wbo, options))];
       if (options.limit) {
--- a/services/sync/tests/unit/test_bookmark_store.js
+++ b/services/sync/tests/unit/test_bookmark_store.js
@@ -1,19 +1,22 @@
 Cu.import("resource://services-sync/engines.js");
 Cu.import("resource://services-sync/engines/bookmarks.js");
 Cu.import("resource://services-sync/type_records/bookmark.js");
 Cu.import("resource://services-sync/util.js");
 
+const PARENT_ANNO = "sync/parent";
+
 Engines.register(BookmarksEngine);
 let engine = Engines.get("bookmarks");
 let store = engine._store;
 let fxuri = Utils.makeURI("http://getfirefox.com/");
 let tburi = Utils.makeURI("http://getthunderbird.com/");
 
+
 function test_bookmark_create() {
   try {
     _("Ensure the record isn't present yet.");
     let ids = Svc.Bookmark.getBookmarkIdsForURI(fxuri, {});
     do_check_eq(ids.length, 0);
 
     _("Let's create a new record.");
     let fxrecord = new Bookmark("bookmarks", "get-firefox1");
@@ -26,32 +29,107 @@ function test_bookmark_create() {
     fxrecord.parentName    = "Bookmarks Toolbar";
     fxrecord.parentid      = "toolbar";
     store.applyIncoming(fxrecord);
 
     _("Verify it has been created correctly.");
     let id = store.idForGUID(fxrecord.id);
     do_check_eq(store.GUIDForId(id), fxrecord.id);
     do_check_eq(Svc.Bookmark.getItemType(id), Svc.Bookmark.TYPE_BOOKMARK);
+    do_check_true(Svc.Bookmark.getBookmarkURI(id).equals(fxuri));
     do_check_eq(Svc.Bookmark.getItemTitle(id), fxrecord.title);
+    do_check_eq(Utils.anno(id, "bookmarkProperties/description"),
+                fxrecord.description);
     do_check_eq(Svc.Bookmark.getFolderIdForItem(id),
                 Svc.Bookmark.toolbarFolder);
     do_check_eq(Svc.Bookmark.getKeywordForBookmark(id), fxrecord.keyword);
 
     _("Have the store create a new record object. Verify that it has the same data.");
     let newrecord = store.createRecord(fxrecord.id);
     do_check_true(newrecord instanceof Bookmark);
     for each (let property in ["type", "bmkUri", "description", "title",
-                               "keyword", "parentName", "parentid"])
+                               "keyword", "parentName", "parentid"]) {
       do_check_eq(newrecord[property], fxrecord[property]);
+    }
     do_check_true(Utils.deepEquals(newrecord.tags.sort(),
                                    fxrecord.tags.sort()));
 
     _("The calculated sort index is based on frecency data.");
     do_check_true(newrecord.sortindex >= 150);
+
+    _("Create a record with some values missing.");
+    let tbrecord = new Bookmark("bookmarks", "thunderbird1");
+    tbrecord.bmkUri        = tburi.spec;
+    tbrecord.parentName    = "Bookmarks Toolbar";
+    tbrecord.parentid      = "toolbar";
+    store.applyIncoming(tbrecord);
+
+    _("Verify it has been created correctly.");
+    id = store.idForGUID(tbrecord.id);
+    do_check_eq(store.GUIDForId(id), tbrecord.id);
+    do_check_eq(Svc.Bookmark.getItemType(id), Svc.Bookmark.TYPE_BOOKMARK);
+    do_check_true(Svc.Bookmark.getBookmarkURI(id).equals(tburi));
+    do_check_eq(Svc.Bookmark.getItemTitle(id), null);
+    let error;
+    try {
+      Utils.anno(id, "bookmarkProperties/description");
+    } catch(ex) {
+      error = ex;
+    }
+    do_check_eq(error.result, Cr.NS_ERROR_NOT_AVAILABLE);
+    do_check_eq(Svc.Bookmark.getFolderIdForItem(id),
+                Svc.Bookmark.toolbarFolder);
+    do_check_eq(Svc.Bookmark.getKeywordForBookmark(id), null);
+  } finally {
+    _("Clean up.");
+    store.wipe();
+  }
+}
+
+function test_bookmark_update() {
+  try {
+    _("Create a bookmark whose values we'll change.");
+    let bmk1_id = Svc.Bookmark.insertBookmark(
+      Svc.Bookmark.toolbarFolder, fxuri, Svc.Bookmark.DEFAULT_INDEX,
+      "Get Firefox!");
+    Utils.anno(bmk1_id, "bookmarkProperties/description", "Firefox is awesome.");
+    Svc.Bookmark.setKeywordForBookmark(bmk1_id, "firefox");
+    let bmk1_guid = store.GUIDForId(bmk1_id);
+
+    _("Update the record with some null values.");
+    let record = store.createRecord(bmk1_guid);
+    record.title = null;
+    record.description = null;
+    record.keyword = null;
+    record.tags = null;
+    store.applyIncoming(record);
+
+    _("Verify that the values have been cleared.");
+    do_check_eq(Utils.anno(bmk1_id, "bookmarkProperties/description"), "");
+    do_check_eq(Svc.Bookmark.getItemTitle(bmk1_id), "");
+    do_check_eq(Svc.Bookmark.getKeywordForBookmark(bmk1_id), null);
+  } finally {
+    _("Clean up.");
+    store.wipe();
+  }
+}
+
+function test_bookmark_createRecord() {
+  try {
+    _("Create a bookmark without a description or title.");
+    let bmk1_id = Svc.Bookmark.insertBookmark(
+      Svc.Bookmark.toolbarFolder, fxuri, Svc.Bookmark.DEFAULT_INDEX, null);
+    let bmk1_guid = store.GUIDForId(bmk1_id);
+
+    _("Verify that the record is created accordingly.");
+    let record = store.createRecord(bmk1_guid);
+    do_check_eq(record.title, null);
+    do_check_eq(record.description, null);
+    do_check_eq(record.keyword, null);
+
   } finally {
     _("Clean up.");
     store.wipe();
   }
 }
 
 function test_folder_create() {
   try {
@@ -111,16 +189,47 @@ function test_folder_createRecord() {
     do_check_eq(record.children[1], bmk2_guid);
 
   } finally {
     _("Clean up.");
     store.wipe();
   }
 }
 
+function test_deleted() {
+  try {
+    _("Create a bookmark that will be deleted.");
+    let bmk1_id = Svc.Bookmark.insertBookmark(
+      Svc.Bookmark.toolbarFolder, fxuri, Svc.Bookmark.DEFAULT_INDEX,
+      "Get Firefox!");
+    let bmk1_guid = store.GUIDForId(bmk1_id);
+
+    _("Delete the bookmark through the store.");
+    let record = new PlacesItem("bookmarks", bmk1_guid);
+    record.deleted = true;
+    store.applyIncoming(record);
+
+    _("Ensure it has been deleted.");
+    let error;
+    try {
+      Svc.Bookmark.getBookmarkURI(bmk1_id);
+    } catch(ex) {
+      error = ex;
+    }
+    do_check_eq(error.result, Cr.NS_ERROR_ILLEGAL_VALUE);
+
+    let newrec = store.createRecord(bmk1_guid);
+    do_check_eq(newrec.deleted, true);
+
+  } finally {
+    _("Clean up.");
+    store.wipe();
+  }
+}
+
 function test_move_folder() {
   try {
     _("Create two folders and a bookmark in one of them.");
     let folder1_id = Svc.Bookmark.createFolder(
       Svc.Bookmark.toolbarFolder, "Folder1", 0);
     let folder1_guid = store.GUIDForId(folder1_id);
     let folder2_id = Svc.Bookmark.createFolder(
       Svc.Bookmark.toolbarFolder, "Folder2", 0);
@@ -128,17 +237,16 @@ function test_move_folder() {
     let bmk_id = Svc.Bookmark.insertBookmark(
       folder1_id, fxuri, Svc.Bookmark.DEFAULT_INDEX, "Get Firefox!");
     let bmk_guid = store.GUIDForId(bmk_id);
 
     _("Get a record, reparent it and apply it to the store.");
     let record = store.createRecord(bmk_guid);
     do_check_eq(record.parentid, folder1_guid);
     record.parentid = folder2_guid;
-    record.description = ""; //TODO for some reason we need this
     store.applyIncoming(record);
 
     _("Verify the new parent.");
     let new_folder_id = Svc.Bookmark.getFolderIdForItem(bmk_id);
     do_check_eq(store.GUIDForId(new_folder_id), folder2_guid);
   } finally {
     _("Clean up.");
     store.wipe();
@@ -183,15 +291,74 @@ function test_move_order() {
 
   } finally {
     Svc.Obs.notify("weave:engine:stop-tracking");
     _("Clean up.");
     store.wipe();
   }
 }
 
+function test_orphan() {
+  try {
+
+    _("Add a new bookmark locally.");
+    let bmk1_id = Svc.Bookmark.insertBookmark(
+      Svc.Bookmark.toolbarFolder, fxuri, Svc.Bookmark.DEFAULT_INDEX,
+      "Get Firefox!");
+    let bmk1_guid = store.GUIDForId(bmk1_id);
+    do_check_eq(Svc.Bookmark.getFolderIdForItem(bmk1_id), Svc.Bookmark.toolbarFolder);
+    let error;
+    try {
+      Utils.anno(bmk1_id, PARENT_ANNO);
+    } catch(ex) {
+      error = ex;
+    }
+    do_check_eq(error.result, Cr.NS_ERROR_NOT_AVAILABLE);
+
+    _("Apply a server record that is the same but refers to non-existent folder.");
+    let record = store.createRecord(bmk1_guid);
+    record.parentid = "non-existent";
+    store.applyIncoming(record);
+
+    _("Verify that bookmark has been flagged as orphan, has not moved.");
+    do_check_eq(Svc.Bookmark.getFolderIdForItem(bmk1_id), Svc.Bookmark.toolbarFolder);
+    do_check_eq(Utils.anno(bmk1_id, PARENT_ANNO), "non-existent");
+
+  } finally {
+    _("Clean up.");
+    store.wipe();
+  }
+}
+
+function test_reparentOrphans() {
+  try {
+    let folder1_id = Svc.Bookmark.createFolder(
+      Svc.Bookmark.toolbarFolder, "Folder1", 0);
+    let folder1_guid = store.GUIDForId(folder1_id);
+
+    _("Create a bogus orphan record and write the record back to the store to trigger _reparentOrphans.");
+    Utils.anno(folder1_id, PARENT_ANNO, folder1_guid);
+    let record = store.createRecord(folder1_guid);
+    record.title = "New title for Folder 1";
+    store._childrenToOrder = {};
+    store.applyIncoming(record);
+
+    _("Verify that is has been marked as an orphan even though it couldn't be moved into itself.");
+    do_check_eq(Utils.anno(folder1_id, PARENT_ANNO), folder1_guid);
+
+  } finally {
+    _("Clean up.");
+    store.wipe();
+  }
+}
+
 function run_test() {
   test_bookmark_create();
+  test_bookmark_createRecord();
+  test_bookmark_update();
   test_folder_create();
   test_folder_createRecord();
+  test_deleted();
   test_move_folder();
   test_move_order();
+  test_orphan();
+  test_reparentOrphans();
 }
new file mode 100644
--- /dev/null
+++ b/services/sync/tests/unit/test_clients_engine.js
@@ -0,0 +1,80 @@
+Cu.import("resource://services-sync/constants.js");
+Cu.import("resource://services-sync/base_records/crypto.js");
+Cu.import("resource://services-sync/base_records/wbo.js");
+Cu.import("resource://services-sync/identity.js");
+Cu.import("resource://services-sync/util.js");
+Cu.import("resource://services-sync/engines/clients.js");
+
+const MORE_THAN_CLIENTS_TTL_REFRESH = 691200; // 8 days
+const LESS_THAN_CLIENTS_TTL_REFRESH = 86400; // 1 day
+
+function test_properties() {
+  try {
+    _("Test lastRecordUpload property");
+    do_check_eq(Svc.Prefs.get("clients.lastRecordUpload"), undefined);
+    do_check_eq(Clients.lastRecordUpload, 0);
+
+    let now = Date.now();
+    Clients.lastRecordUpload = now / 1000;
+    do_check_eq(Clients.lastRecordUpload, Math.floor(now / 1000));
+  } finally {
+    Svc.Prefs.resetBranch("");
+  }
+}
+
+function test_sync() {
+  _("Ensure that Clients engine uploads a new client record once a week.");
+  Svc.Prefs.set("clusterURL", "http://localhost:8080/");
+  Svc.Prefs.set("username", "foo");
+  new SyncTestingInfrastructure();
+
+  CollectionKeys.generateNewKeys();
+
+  let global = new ServerWBO('global',
+                             {engines: {clients: {version: Clients.version,
+                                                  syncID: Clients.syncID}}});
+  let coll = new ServerCollection();
+  let clientwbo = coll.wbos[Clients.localID] = new ServerWBO(Clients.localID);
+  let server = httpd_setup({
+      "/1.0/foo/storage/meta/global": global.handler(),
+      "/1.0/foo/storage/clients": coll.handler()
+  });
+  do_test_pending();
+
+  try {
+
+    _("First sync, client record is uploaded");
+    do_check_eq(clientwbo.payload, undefined);
+    do_check_eq(Clients.lastRecordUpload, 0);
+    Clients.sync();
+    do_check_true(!!clientwbo.payload);
+    do_check_true(Clients.lastRecordUpload > 0);
+
+    _("Let's time travel more than a week back, new record should've been uploaded.");
+    Clients.lastRecordUpload -= MORE_THAN_CLIENTS_TTL_REFRESH;
+    let lastweek = Clients.lastRecordUpload;
+    clientwbo.payload = undefined;
+    Clients.sync();
+    do_check_true(!!clientwbo.payload);
+    do_check_true(Clients.lastRecordUpload > lastweek);
+
+    _("Time travel one day back, no record uploaded.");
+    Clients.lastRecordUpload -= LESS_THAN_CLIENTS_TTL_REFRESH;
+    let yesterday = Clients.lastRecordUpload;
+    clientwbo.payload = undefined;
+    Clients.sync();
+    do_check_eq(clientwbo.payload, undefined);
+    do_check_eq(Clients.lastRecordUpload, yesterday);
+
+  } finally {
+    server.stop(do_test_finished);
+    Svc.Prefs.resetBranch("");
+    Records.clearCache();
+  }
+}
+
+
+function run_test() {
+  test_properties();
+  test_sync();
+}
--- a/services/sync/tests/unit/test_records_wbo.js
+++ b/services/sync/tests/unit/test_records_wbo.js
@@ -1,80 +1,82 @@
-try {
-  Cu.import("resource://services-sync/auth.js");
-  Cu.import("resource://services-sync/base_records/wbo.js");
-  Cu.import("resource://services-sync/base_records/collection.js");
-  Cu.import("resource://services-sync/identity.js");
-  Cu.import("resource://services-sync/log4moz.js");
-  Cu.import("resource://services-sync/resource.js");
-  Cu.import("resource://services-sync/util.js");
-} catch (e) { do_throw(e); }
+Cu.import("resource://services-sync/auth.js");
+Cu.import("resource://services-sync/base_records/wbo.js");
+Cu.import("resource://services-sync/base_records/collection.js");
+Cu.import("resource://services-sync/identity.js");
+Cu.import("resource://services-sync/resource.js");
+Cu.import("resource://services-sync/util.js");
+
 
-function record_handler(metadata, response) {
-  let obj = {id: "asdf-1234-asdf-1234",
-             modified: 2454725.98283,
-             payload: JSON.stringify({cheese: "roquefort"})};
-  return httpd_basic_auth_handler(JSON.stringify(obj), metadata, response);
+function test_toJSON() {
+  _("Create a record, for now without a TTL.");
+  let wbo = new WBORecord("coll", "a_record");
+  wbo.modified = 12345;
+  wbo.sortindex = 42;
+  wbo.payload = {};
+
+  _("Verify that the JSON representation contains the WBO properties, but not TTL.");
+  let json = JSON.parse(JSON.stringify(wbo));
+  do_check_eq(json.modified, 12345);
+  do_check_eq(json.sortindex, 42);
+  do_check_eq(json.payload, "{}");
+  do_check_false("ttl" in json);
+
+  _("Set a TTL, make sure it's present in the JSON representation.");
+  wbo.ttl = 30*60;
+  json = JSON.parse(JSON.stringify(wbo));
+  do_check_eq(json.ttl, 30*60);
 }
 
-function record_handler2(metadata, response) {
-  let obj = {id: "record2",
-             modified: 2454725.98284,
-             payload: JSON.stringify({cheese: "gruyere"})};
-  return httpd_basic_auth_handler(JSON.stringify(obj), metadata, response);
-}
 
-function coll_handler(metadata, response) {
-  let obj = [{id: "record2",
-              modified: 2454725.98284,
-              payload: JSON.stringify({cheese: "gruyere"})}];
-  return httpd_basic_auth_handler(JSON.stringify(obj), metadata, response);
-}
+function test_fetch() {
+  let record = {id: "asdf-1234-asdf-1234",
+                modified: 2454725.98283,
+                payload: JSON.stringify({cheese: "roquefort"})};
+  let record2 = {id: "record2",
+                 modified: 2454725.98284,
+                 payload: JSON.stringify({cheese: "gruyere"})};
+  let coll = [{id: "record2",
+               modified: 2454725.98284,
+               payload: JSON.stringify({cheese: "gruyere"})}];
 
-function run_test() {
-  let server;
+  _("Setting up server.");
+  let server = httpd_setup({
+    "/record":  httpd_handler(200, "OK", JSON.stringify(record)),
+    "/record2": httpd_handler(200, "OK", JSON.stringify(record2)),
+    "/coll":    httpd_handler(200, "OK", JSON.stringify(coll))
+  });
   do_test_pending();
 
   try {
-    let log = Log4Moz.repository.getLogger('Test');
-    Log4Moz.repository.rootLogger.addAppender(new Log4Moz.DumpAppender());
-
-    log.info("Setting up server and authenticator");
-
-    server = httpd_setup({"/record": record_handler,
-                          "/record2": record_handler2,
-                          "/coll": coll_handler});
-
-    let auth = new BasicAuthenticator(new Identity("secret", "guest", "guest"));
-    Auth.defaultAuthenticator = auth;
-
-    log.info("Getting a WBO record");
-
-    let res = new Resource("http://localhost:8080/record");
-    let resp = res.get();
-
+    _("Fetching a WBO record");
     let rec = new WBORecord("coll", "record");
-    rec.deserialize(res.data);
+    rec.fetch("http://localhost:8080/record");
     do_check_eq(rec.id, "asdf-1234-asdf-1234"); // NOT "record"!
 
     do_check_eq(rec.modified, 2454725.98283);
     do_check_eq(typeof(rec.payload), "object");
     do_check_eq(rec.payload.cheese, "roquefort");
-    do_check_eq(resp.status, 200);
 
-    log.info("Getting a WBO record using the record manager");
-
+    _("Fetching a WBO record using the record manager");
     let rec2 = Records.get("http://localhost:8080/record2");
     do_check_eq(rec2.id, "record2");
     do_check_eq(rec2.modified, 2454725.98284);
     do_check_eq(typeof(rec2.payload), "object");
     do_check_eq(rec2.payload.cheese, "gruyere");
     do_check_eq(Records.response.status, 200);
 
     // Testing collection extraction.
-    log.info("Extracting collection.");
+    _("Extracting collection.");
     let rec3 = new WBORecord("tabs", "foo");   // Create through constructor.
     do_check_eq(rec3.collection, "tabs");
-    log.info("Done!");
+
+  } finally {
+    server.stop(do_test_finished);
   }
-  catch (e) { do_throw(e); }
-  finally { server.stop(do_test_finished); }
 }
+
+function run_test() {
+  initTestLogging("Trace");
+
+  test_toJSON();
+  test_fetch();
+}
--- a/services/sync/tests/unit/test_service_checkAccount.js
+++ b/services/sync/tests/unit/test_service_checkAccount.js
@@ -1,27 +1,20 @@
 Cu.import("resource://services-sync/service.js");
 Cu.import("resource://services-sync/util.js");
 
-function send(statusCode, status, body) {
-  return function(request, response) {
-    response.setStatusLine(request.httpVersion, statusCode, status);
-    response.bodyOutputStream.write(body, body.length);
-  };
-}
-
 function run_test() {
   do_test_pending();
   let server = httpd_setup({
-    "/user/1.0/johndoe": send(200, "OK", "1"),
-    "/user/1.0/janedoe": send(200, "OK", "0"),
+    "/user/1.0/johndoe": httpd_handler(200, "OK", "1"),
+    "/user/1.0/janedoe": httpd_handler(200, "OK", "0"),
     // john@doe.com
-    "/user/1.0/7wohs32cngzuqt466q3ge7indszva4of": send(200, "OK", "0"),
+    "/user/1.0/7wohs32cngzuqt466q3ge7indszva4of": httpd_handler(200, "OK", "0"),
     // jane@doe.com
-    "/user/1.0/vuuf3eqgloxpxmzph27f5a6ve7gzlrms": send(200, "OK", "1")
+    "/user/1.0/vuuf3eqgloxpxmzph27f5a6ve7gzlrms": httpd_handler(200, "OK", "1")
   });
   try {
     Service.serverURL = "http://localhost:8080/";
 
     _("A 404 will be recorded as 'generic-server-error'");
     do_check_eq(Service.checkUsername("jimdoe"), "generic-server-error");
 
     _("Account that's available.");
--- a/services/sync/tests/unit/test_service_cluster.js
+++ b/services/sync/tests/unit/test_service_cluster.js
@@ -6,41 +6,34 @@ function do_check_throws(func) {
   try {
     func();
   } catch (ex) {
     raised = true;
   }
   do_check_true(raised);
 }
 
-function send(statusCode, status, body) {
-  return function(request, response) {
-    response.setStatusLine(request.httpVersion, statusCode, status);
-    response.bodyOutputStream.write(body, body.length);
-  };
-}
-
 function test_findCluster() {
   _("Test Service._findCluster()");
   let server;
   try {
     Service.serverURL = "http://localhost:8080/";
     Service.username = "johndoe";
 
     _("_findCluster() throws on network errors (e.g. connection refused).");
     do_check_throws(function() {
       Service._findCluster();
     });
 
     server = httpd_setup({
-      "/user/1.0/johndoe/node/weave": send(200, "OK", "http://weave.user.node/"),
-      "/user/1.0/jimdoe/node/weave": send(200, "OK", "null"),
-      "/user/1.0/janedoe/node/weave": send(404, "Not Found", "Not Found"),
-      "/user/1.0/juliadoe/node/weave": send(400, "Bad Request", "Bad Request"),
-      "/user/1.0/joedoe/node/weave": send(500, "Server Error", "Server Error")
+      "/user/1.0/johndoe/node/weave": httpd_handler(200, "OK", "http://weave.user.node/"),
+      "/user/1.0/jimdoe/node/weave": httpd_handler(200, "OK", "null"),
+      "/user/1.0/janedoe/node/weave": httpd_handler(404, "Not Found", "Not Found"),
+      "/user/1.0/juliadoe/node/weave": httpd_handler(400, "Bad Request", "Bad Request"),
+      "/user/1.0/joedoe/node/weave": httpd_handler(500, "Server Error", "Server Error")
     });
 
     _("_findCluster() returns the user's cluster node");
     let cluster = Service._findCluster();
     do_check_eq(cluster, "http://weave.user.node/");
 
     _("A 'null' response is converted to null.");
     Service.username = "jimdoe";
@@ -71,18 +64,18 @@ function test_findCluster() {
     }
   }
 }
 
 
 function test_setCluster() {
   _("Test Service._setCluster()");
   let server = httpd_setup({
-    "/user/1.0/johndoe/node/weave": send(200, "OK", "http://weave.user.node/"),
-    "/user/1.0/jimdoe/node/weave": send(200, "OK", "null")
+    "/user/1.0/johndoe/node/weave": httpd_handler(200, "OK", "http://weave.user.node/"),
+    "/user/1.0/jimdoe/node/weave": httpd_handler(200, "OK", "null")
   });
   try {
     Service.serverURL = "http://localhost:8080/";
     Service.username = "johndoe";
 
     _("Check initial state.");
     do_check_eq(Service.clusterURL, "");
 
@@ -103,18 +96,18 @@ function test_setCluster() {
     Svc.Prefs.resetBranch("");
     server.stop(runNextTest);
   }
 }
 
 function test_updateCluster() {
   _("Test Service._updateCluster()");
   let server = httpd_setup({
-    "/user/1.0/johndoe/node/weave": send(200, "OK", "http://weave.user.node/"),
-    "/user/1.0/janedoe/node/weave": send(200, "OK", "http://weave.cluster.url/")
+    "/user/1.0/johndoe/node/weave": httpd_handler(200, "OK", "http://weave.user.node/"),
+    "/user/1.0/janedoe/node/weave": httpd_handler(200, "OK", "http://weave.cluster.url/")
   });
   try {
     Service.serverURL = "http://localhost:8080/";
     Service.username = "johndoe";
 
     _("Check initial state.");
     do_check_eq(Service.clusterURL, "");
     do_check_eq(Svc.Prefs.get("lastClusterUpdate"), null);
--- a/services/sync/tests/unit/test_service_quota.js
+++ b/services/sync/tests/unit/test_service_quota.js
@@ -1,30 +1,23 @@
 Cu.import("resource://services-sync/service.js");
 Cu.import("resource://services-sync/util.js");
 
-function send(body) {
-  return function(request, response) {
-    response.setStatusLine(request.httpVersion, 200, "OK");
-    response.bodyOutputStream.write(body, body.length);
-  };
-}
-
 function run_test() {
   let collection_usage = {steam:  65.11328,
                           petrol: 82.488281,
                           diesel: 2.25488281};
   let quota = [2169.65136, 8192];
 
   do_test_pending();
   let server = httpd_setup({
-    "/1.0/johndoe/info/collection_usage": send(JSON.stringify(collection_usage)),
-    "/1.0/johndoe/info/quota":            send(JSON.stringify(quota)),
-    "/1.0/janedoe/info/collection_usage": send("gargabe"),
-    "/1.0/janedoe/info/quota":            send("more garbage")
+    "/1.0/johndoe/info/collection_usage": httpd_handler(200, "OK", JSON.stringify(collection_usage)),
+    "/1.0/johndoe/info/quota":            httpd_handler(200, "OK", JSON.stringify(quota)),
+    "/1.0/janedoe/info/collection_usage": httpd_handler(200, "OK", "gargabe"),
+    "/1.0/janedoe/info/quota":            httpd_handler(200, "OK", "more garbage")
   });
 
   try {
     Weave.Service.clusterURL = "http://localhost:8080/";
     Weave.Service.username = "johndoe";
 
     _("Test getCollectionUsage().");
     let res = Weave.Service.getCollectionUsage();
--- a/services/sync/tests/unit/test_service_verifyLogin.js
+++ b/services/sync/tests/unit/test_service_verifyLogin.js
@@ -13,23 +13,16 @@ function login_handler(request, response
     response.setStatusLine(request.httpVersion, 200, "OK");
   } else {
     body = "Unauthorized";
     response.setStatusLine(request.httpVersion, 401, "Unauthorized");
   }
   response.bodyOutputStream.write(body, body.length);
 }
 
-function send(statusCode, status, body) {
-  return function(request, response) {
-    response.setStatusLine(request.httpVersion, statusCode, status);
-    response.bodyOutputStream.write(body, body.length);
-  };
-}
-
 function service_unavailable(request, response) {
   let body = "Service Unavailable";
   response.setStatusLine(request.httpVersion, 503, "Service Unavailable");
   response.setHeader("Retry-After", "42");
   response.bodyOutputStream.write(body, body.length);
 }
 
 function run_test() {
@@ -40,17 +33,17 @@ function run_test() {
   Weave.Svc.Login.removeAllLogins();
   
   do_test_pending();
   let server = httpd_setup({
     "/api/1.0/johndoe/info/collections": login_handler,
     "/api/1.0/janedoe/info/collections": service_unavailable,
     "/api/1.0/johndoe/storage/meta/global": new ServerWBO().handler(),
     "/api/1.0/johndoe/storage/crypto/keys": new ServerWBO().handler(),
-    "/user/1.0/johndoe/node/weave": send(200, "OK", "http://localhost:8080/api/")
+    "/user/1.0/johndoe/node/weave": httpd_handler(200, "OK", "http://localhost:8080/api/")
   });
 
   try {
     Service.serverURL = "http://localhost:8080/";
 
     _("Force the initial state.");
     Status.service = STATUS_OK;
     do_check_eq(Status.service, STATUS_OK);