status commit for the massive refactoring as described at the status meeting,
authorAndrew Sutherland <asutherland@asutherland.org>
Tue, 14 Oct 2008 23:57:28 -0700
changeset 974 06fabe695ede97f7960935c4515e2e867a3eefec
parent 970 2c5e2d260836f0789d3f10bb6a297694086305b4
child 975 d8efe76d7bf37411412cb0d654e377d36f67253b
push idunknown
push userunknown
push dateunknown
status commit for the massive refactoring as described at the status meeting, roughly circa the status meeting. https://wiki.mozilla.org/Thunderbird/StatusMeetings/2008-10-14#asuth
components/glautocomp.js
modules/collection.js
modules/datamodel.js
modules/datastore.js
modules/explattr.js
modules/fundattr.js
modules/gloda.js
modules/index_ab.js
modules/indexer.js
modules/noun_freetag.js
modules/noun_tag.js
modules/query.js
modules/utils.js
--- a/components/glautocomp.js
+++ b/components/glautocomp.js
@@ -172,46 +172,42 @@ nsAutoCompleteGlodaResult.prototype = {
     return gravURL;
   },
   removeValueAt: function() {},
 
   _stop: function() {
   },
 };
 
+const MAX_POPULAR_CONTACTS = 200;
+
 /**
  * Complete contacts/identities based on name/email.  Instant phase is based on
  *  a suffix-tree built of popular contacts/identities.  Delayed phase relies
  *  on a LIKE search of all known contacts.
  */
 function ContactIdentityCompleter() {
   // get all the contacts
   let contactQuery = Gloda.newQuery(Gloda.NOUN_CONTACT);
-  this.contactCollection = contactQuery.popularityRange(10, null).getAllSync();
+  contactQuery.orderBy("-popularity").limit(MAX_POPULAR_CONTACTS);
+  this.contactCollection = contactQuery.getAllSync();
 
   // cheat and explicitly add our own contact...
-  this.contactCollection._onItemsAdded([Gloda.myContact]);
-
-  // assuming we found some contacts...
-  if (this.contactCollection.items.length) {
-    // get all the identities...
-    let identityQuery = Gloda.newQuery(Gloda.NOUN_IDENTITY);
-    // ...that belong to one of the above contacts.
-    identityQuery.contact.apply(identityQuery, this.contactCollection.items);
-    this.identityCollection = identityQuery.getAllSync();
-  }
-  else {
-    // create an empty explicit collection
-    this.identityCollection = Gloda.explicitCollection(Gloda.NOUN_IDENTITY, []);
-  }
+  if (!(Gloda.myContact.id in this.contactCollection._idMap))
+    this.contactCollection._onItemsAdded([Gloda.myContact]);
+    
+  // the set of identities owned by the contacts is automatically loaded as part
+  //  of the contact loading...
+  this.identityCollection =
+    this.contactCollection.subCollections[Gloda.NOUN_IDENTITY];
 
   let contactNames = [(c.name.replace(" ", "").toLowerCase() || "x") for each
-                      ([ic, c] in Iterator(this.contactCollection.items))];
+                      ([, c] in Iterator(this.contactCollection.items))];
   let identityMails = [i.value.toLowerCase() for each
-                       ([ii, i] in Iterator(this.identityCollection.items))];
+                       ([, i] in Iterator(this.identityCollection.items))];
 
   this.suffixTree = new MultiSuffixTree(contactNames.concat(identityMails),
     this.contactCollection.items.concat(this.identityCollection.items));
 }
 ContactIdentityCompleter.prototype = {
   _popularitySorter: function(a, b){ return b.popularity - a.popularity; },
   complete: function ContactIdentityCompleter_complete(aResult, aString) {
     if (aString.length < 3)
--- a/modules/collection.js
+++ b/modules/collection.js
@@ -386,32 +386,29 @@ function GlodaCollection(aNounMeta, aIte
   if (aNounMeta === undefined)
     return;
 
   this._nounMeta = aNounMeta;
   // should we also maintain a unique value mapping...
   if (this._nounMeta.usesUniqueValue)
     this._uniqueValueMap = {};
 
-  this.items = aItems || [];
+  this.items = [];
   this._idMap = {};
-  if (this._uniqueValueMap) {
-    for each (let [iItem, item] in Iterator(this.items)) {
-      this._idMap[item.id] = item;
-      this._uniqueValueMap[item.uniqueValue] = item;
-    }
-  }
-  else {
-    for each (let [iItem, item] in Iterator(this.items)) {
-      this._idMap[item.id] = item;
-    }
-  }
+  
+  // force the listener to null for our call to _onItemsAdded; no events for
+  //  the initial load-out.
+  this._listener = null;
+  this._onItemsAdded(items);
   
   this.query = aQuery || null;
   this._listener = aListener || null;
+  
+  this.referencesByNounID = {};
+  this.subCollections = {};
 }
 
 GlodaCollection.prototype = {
   get listener() { return this._listener; },
   set listener(aListener) { this._listener = aListener; },
   
   /**
    * Clear the contents of this collection.  This only makes sense for explicit
--- a/modules/datamodel.js
+++ b/modules/datamodel.js
@@ -49,27 +49,27 @@ const LOG = Log4Moz.Service.getLogger("g
 
 Cu.import("resource://gloda/modules/utils.js");
 
 /**
  * @class Represents a gloda attribute definition.
  */
 function GlodaAttributeDef(aDatastore, aID, aCompoundName, aProvider, aAttrType,
                            aPluginName, aAttrName, aSubjectTypes,
-                           aObjectType, aObjectNounMeta) {
+                           aObjectType, aObjectNounDef) {
   this._datastore = aDatastore;
   this._id = aID;
   this._compoundName = aCompoundName;
   this._provider = aProvider;
   this._attrType = aAttrType;
   this._pluginName = aPluginName;
   this._attrName = aAttrName;
   this._subjectTypes = aSubjectTypes;
   this._objectType = aObjectType;
-  this._objectNounMeta = aObjectNounMeta;
+  this._objectNounDef = aObjectNounDef;
 
   this.boundName = null;
   this._singular = null;
 
   this._special = 0; // not special
   this._specialColumnName = null;
 
   /** Map parameter values to the underlying database id. */
@@ -77,17 +77,17 @@ function GlodaAttributeDef(aDatastore, a
 }
 
 GlodaAttributeDef.prototype = {
   get id() { return this._id; },
   get provider() { return this._provider; },
   get attributeName() { return this._attrName; },
 
   get objectNoun() { return this._objectType; },
-  get objectNounMeta() { return this._objectNounMeta; },
+  get objectNounDef() { return this._objectNounDef; },
 
   get isBound() { return this.boundName !== null; },
   get singular() { return this._singular; },
 
   get special() { return this._special; },
   get specialColumnName() { return this._specialColumnName; },
   
   get parameterBindings() { return this._parameterBindings; },
@@ -110,49 +110,49 @@ GlodaAttributeDef.prototype = {
     let id = this._datastore._createAttributeDef(this._attrType,
                  this._pluginName, this._attrName, aValue);
     this._parameterBindings[aValue] = id;
     this._datastore.reportBinding(id, this, aValue);
     return id;
   },
 
   /**
-   * Given an instance of an object with this attribute, return the value
-   *  of the attribute.  This handles bound and un-bound attributes.  For
-   *  singular attributes, the value is null or the value; for non-singular
-   *  attributes the value is a list.
+   * Given a list of values (if non-singular) or a single value (if singular),
+   *  return a list (regardless of plurality) of database-ready [attribute id,
+   *  value] tuples.  This is intended to be used to directly convert the value
+   *  of a property on an object that corresponds to a bound attribute.
    */
-  getValueFromInstance: function gloda_attr_getValueFromInstance(aObj) {
-    // if it's bound, we can just use the binding and trigger his caching
-    // if it's special, the attribute actually exists, but just with explicit
-    //  code backing it.
-    if (this.boundName !== null || this._special) {
-      return aObj[this.boundName];
-    }
-    let instances = aObj.getAttributeInstances(this);
-    let nounMeta = this._objectNounMeta;
+  convertValuesToDBAttributes:
+      function gloda_attr_convertValuesToDBAttributes(aInstanceValues) {
+    let nounDef = this._objectNounDef;
+    
     if (this._singular) {
-      if (instances.length > 0)
-        return nounMeta.fromParamAndValue(instances[0][1], instances[0][2]);
-      else
-        return null;
+      if (nounDef.usesParameter) {
+        let [param, dbValue] = nounDef.toParamAndValue(aInstanceValues);
+        return [[this.bindParameter(param), dbValue]];
+      }
+      else {
+        return [[this._id, nounDef.toParamAndValue(aInstanceValues)[1]]];
+      }
     }
     else {
-      let values;
-      if (instances.length > 0) {
-        values = [];
-        for (let iInst = 0; iInst < instances.length; iInst++) {
-          values.push(nounMeta.fromParamAndValue(instances[iInst][1],
-                                                 instances[iInst][2]));
+      let dbAttributes = [];
+      if (nounDef.usesParameter) {
+        for each (let [iValue, instanceValue] in Iterator(aInstanceValues)) {
+          let [param, dbValue] = nounDef.toParamAndValue(aInstanceValues);
+          dbAttributes.push([this.bindParameter(param), dbValue]);
         }
       }
       else {
-        values = instances; // empty is empty
+        for each (let [iValue, instanceValue] in Iterator(aInstanceValues)) {
+          dbAttributes.push([this._id,
+                             nounDef.toParamAndValue(instanceValue)[1]);
+        }
       }
-      return values;
+      return dbAttributes;
     }
   },
 
   toString: function() {
     return this._compoundName;
   }
 };
 
@@ -272,56 +272,57 @@ function MixIn(aConstructor, aMixIn) {
  */
 function GlodaConversation(aDatastore, aID, aSubject, aOldestMessageDate,
                            aNewestMessageDate) {
   this._datastore = aDatastore;
   this._id = aID;
   this._subject = aSubject;
   this._oldestMessageDate = aOldestMessageDate;
   this._newestMessageDate = aNewestMessageDate;
-
-  this._messages = null;
 }
 
 GlodaConversation.prototype = {
   NOUN_ID: 101,
   get id() { return this._id; },
   get subject() { return this._subject; },
   get oldestMessageDate() { return this._oldestMessageDate; },
   get newestMessageDate() { return this._newestMessageDate; },
 
-  /**
-   * @TODO Return the collection of messages belonging to this conversation.
-   * (And weakly store a reference to the collection.  Once the user is rid of
-   *  it, we really don't care.)
-   */
-  get messages() {
-    if (this._messages == null) {
-      this._messages = this._datastore.getMessagesByConversationID(this._id,
-                                                                   false);
-    }
-    return this._messages;
-  },
-
   toString: function gloda_conversation_toString() {
     return this._subject;
   },
 };
 
-function GlodaFolder(aDatastore, aID, aURI, aPrettyName) {
+function GlodaFolder(aDatastore, aID, aURI, aDirtyStatus, aPrettyName) {
   this._datastore = aDatastore;
   this._id = aID;
   this._uri = aURI;
+  this._dirtyStatus = aDirtyStatus;
   this._prettyName = aPrettyName;
 }
 
 GlodaFolder.prototype = {
- NOUN_ID: 100,
- get id() { return this._id; },
- get uri() { return this._uri; },
+  NOUN_ID: 100,
+  /** The folder is believed to be up-to-date */
+  kFolderClean: 0,
+  /** The folder has some un-indexed or dirty messages */
+  kFolderDirty: 1,
+  /** The folder needs to be entirely re-indexed, regardless of the flags on
+   * the messages in the folder. This state will be downgraded to dirty */
+  kFolderFilthy: 2,
+  get id() { return this._id; },
+  get uri() { return this._uri; },
+  get dirtyStatus { return this._dirtyStatus; },
+  set dirtyStatus (aNewStatus) {
+    if (aNewStatus != this._dirtyStatus) {
+      this._dirtyStatus = aNewStatus;
+      this._datastore.updateFolderDirtyStatus(this);
+    }
+  },
+  get name { return this._prettyName; },
   toString: function gloda_folder_toString() {
     return this._prettyName;
   }
 }
 
 /**
  * @class A message representation.
  */
@@ -329,58 +330,62 @@ function GlodaMessage(aDatastore, aID, a
                       aConversationID, aConversation, aDate,
                       aHeaderMessageID, aDeleted) {
   this._datastore = aDatastore;
   this._id = aID;
   this._folderID = aFolderID;
   this._messageKey = aMessageKey;
   this._conversationID = aConversationID;
   this._conversation = aConversation;
-  this.date = aDate;
+  this._date = aDate;
   this._headerMessageID = aHeaderMessageID;
 
   // only set _deleted if we're deleted, otherwise the undefined does our
   //  speaking for us.
   if (aDeleted)
     this._deleted = aDeleted;
 }
 
 GlodaMessage.prototype = {
   NOUN_ID: 102,
   get id() { return this._id; },
   get folderID() { return this._folderID; },
   get messageKey() { return this._messageKey; },
   get conversationID() { return this._conversationID; },
   // conversation is special
   get headerMessageID() { return this._headerMessageID; },
+  
+  get date() { return this._date; },
+  set date(aNewDate) { this._date = aNewDate; },
 
   get folderURI() {
     if (this._folderID != null)
-      return this._datastore._mapFolderID(this._folderID);
+      return this._datastore._mapFolderID(this._folderID).uri;
     else
       return null;
   },
   get conversation() {
     if (this._conversation == null) {
       this._conversation = this._datastore.getConversationByID(
         this._conversationID);
     }
     return this._conversation;
   },
 
-  set messageKey(aMessageKey) { this._messageKey = aMessageKey; },
-  set folderURI(aFolderURI) {
-    this._folderID = this._datastore._mapFolderURI(aFolderURI);
-  },
-
   toString: function gloda_message_toString() {
     // uh, this is a tough one...
     return "Message " + this._id;
   },
 
+  _clone: function gloda_message_clone() {
+    return new GlodaMessage(this._datastore, this._id, this._folderId,
+      this._messageKey, this._conversationID, this._conversation, this._date,
+      this._headerMessageID, this._deleted);
+  },
+
   _ghost: function gloda_message_ghost() {
     this._folderID = null;
     this._messageKey = null;
   },
 
   _nuke: function gloda_message_nuke() {
     this._id = null;
     this._folderID = null;
@@ -399,17 +404,17 @@ GlodaMessage.prototype = {
    * This method no longer caches the result, so it's up to you.
    */
   get folderMessage() {
     if (this._folderID === null || this._messageKey === null)
       return null;
     let rdfService = Cc['@mozilla.org/rdf/rdf-service;1'].
                      getService(Ci.nsIRDFService);
     let folder = rdfService.GetResource(
-                   this._datastore._mapFolderID(this._folderID));
+                   this._datastore._mapFolderID(this._folderID).uri);
     if (folder instanceof Ci.nsIMsgFolder) {
       let folderMessage = folder.GetMessageHeader(this._messageKey);
       if (folderMessage !== null) {
         // verify the message-id header matches what we expect...
         if (folderMessage.messageId != this._headerMessageID) {
           LOG.info("Message with message key does not match expected " +
                    "header! (" + this._headerMessageID + " expected, got " +
                    folderMessage.messageId + ")");
@@ -454,17 +459,18 @@ function GlodaContact(aDatastore, aID, a
 }
 
 GlodaContact.prototype = {
   NOUN_ID: 103,
 
   get id() { return this._id; },
   get directoryUUID() { return this._directoryUUID; },
   get contactUUID() { return this._contactUUID; },
-  get name() { return this._name },
+  get name() { return this._name; },
+  set name(aName) { this._name = aName; },
 
   get popularity() { return this._popularity; },
   set popularity(aPopularity) {
     this._popularity = aPopularity;
     this.dirty = true;
   },
 
   get frecency() { return this._frecency; },
@@ -480,17 +486,22 @@ GlodaContact.prototype = {
   },
 
   toString: function gloda_contact_toString() {
     return this._name;
   },
   
   get accessibleLabel() {
     return "Contact: " + this._name;
-  }
+  },
+
+  _clone: function gloda_contact_clone() {
+    return new GlodaContact(this._datastore, this._id, this._directoryUUID,
+      this._contactUUID, this._name, this._popularity, this._frecency);
+  },
 };
 MixIn(GlodaContact, GlodaHasAttributesMixIn);
 
 
 /**
  * @class A specific means of communication for a contact.
  */
 function GlodaIdentity(aDatastore, aID, aContactID, aContact, aKind, aValue,
@@ -524,34 +535,17 @@ GlodaIdentity.prototype = {
     return this._contact;
   },
 
   toString: function gloda_identity_toString() {
     return this._value;
   },
 
   get abCard() {
-    // search through all of our local address books looking for a match.
-    let enumerator = Components.classes["@mozilla.org/abmanager;1"]
-                               .getService(Ci.nsIAbManager)
-                               .directories;
-    let cardForEmailAddress;
-    let addrbook;
-    while (!cardForEmailAddress && enumerator.hasMoreElements())
-    {
-      addrbook = enumerator.getNext().QueryInterface(Ci.nsIAbDirectory);
-      try
-      {
-        cardForEmailAddress = addrbook.cardForEmailAddress(this._value);
-        if (cardForEmailAddress)
-          return cardForEmailAddress;
-      } catch (ex) {}
-    }
-
-    return null;
+    return GlodaUtils.getCardForEmail(this._value);
   },
   
   pictureURL: function(aSize) {
     let md5hash = GlodaUtils.md5HashString(this._value);
     let gravURL = "http://www.gravatar.com/avatar/" + md5hash +
                                 "?d=identicon&s=" + aSize + "&r=g";
     return gravURL;
   }
--- a/modules/datastore.js
+++ b/modules/datastore.js
@@ -48,21 +48,16 @@ const Cr = Components.results;
 const Cu = Components.utils;
 
 Cu.import("resource://gloda/modules/log4moz.js");
 
 Cu.import("resource://gloda/modules/datamodel.js");
 Cu.import("resource://gloda/modules/databind.js");
 Cu.import("resource://gloda/modules/collection.js");
 
-// XXX from Gloda.js.  duplicated here for dependency reasons.  bad!
-const kSpecialColumn = 1;
-const kSpecialString = 2;
-const kSpecialFulltext = 3;
-
 /**
  * @class This callback handles processing the asynchronous query results of
  *  GlodaDatastore.getMessagesByMessageID.  Because that method is only
  *  called as part of the indexing process, we are guaranteed that there will
  *  be no real caching ramifications.  Accordingly, we can also defer our cache
  *  processing (via GlodaCollectionManager) until the query completes.
  *
  * @param aMsgIDToIndex Map from message-id to the desired
@@ -108,35 +103,65 @@ MessagesByMessageIdCallback.prototype = 
     this.statement = null;
 
     this.callback.apply(this.callbackThis, args);
 
     GlodaDatastore._asyncCompleted();
   }
 };
 
+function PostCommitHandler(aCallbacks) {
+  this.callbacks = aCallbacks;
+}
+
+PostCommitHandler.prototype = {
+  handleResult: function gloda_ds_pch_handleResult(aResultSet) {
+  },
+  
+  handleError: function gloda_ds_pch_handleError(aError) {
+  },
+  
+  handleCompletion: function gloda_ds_pch_handleCompletion(aReason) {
+    if (aReason == Ci.mozIStorageStatementCallback.REASON_FINISHED) {
+      for each (let [iCallback, callback] in Iterator(this.callbacks)) {
+        try {
+          callback();
+        }
+        catch (ex) {
+          dump("PostCommitHandler callback (" + ex.fileName + ":" +
+               ex.lineNumber + ") threw: " + ex);
+        }
+      }
+    }
+    GlodaDatastore._asyncCompleted();
+  }
+};
+
 /**
  * @class Handles the results from a GlodaDatastore.queryFromQuery call.
  * @constructor
  */
 function QueryFromQueryCallback(aStatement, aNounMeta, aCollection) {
   this.statement = aStatement;
   this.nounMeta = aNounMeta;
   this.collection = aCollection;
+  
+  this.referencesByNounID = {};
 
   GlodaDatastore._pendingAsyncStatements++;
 }
 
 QueryFromQueryCallback.prototype = {
   handleResult: function gloda_ds_qfq_handleResult(aResultSet) {
     let newItems = [];
     let row;
     let nounMeta = this.nounMeta;
     while (row = aResultSet.getNextRow()) {
       let item = nounMeta.objFromRow.call(nounMeta.datastore, row);
+      GlodaDatastore.loadNounItem(item, this.referencesByNounID);
       newItems.push(item);
     }
     // have the collection manager attempt to replace the instances we just
     //  created with pre-existing instances.  there is some waste here...
     // XXX consider having collection manager take row objects with the
     //  knowledge of what index is the 'id' index and knowing what objFromRow
     //  method to call if it needs to realize the row.
     // queries have the potential to easily exceed the size of our cache, and
@@ -257,27 +282,40 @@ QueryFromQueryCallback.prototype = {
  *  states of attributes to accomplish this, but that is not desirable.)  This
  *  needs to be addressed, and may be best addressed at layers above
  *  datastore.js.
  * @namespace
  */
 var GlodaDatastore = {
   _log: null,
 
+  /* see Gloda's documentation for these constants */
+  kSpecialColumn: 1,
+  kSpecialString: 2,
+  kSpecialFulltext: 3,
+  
+  kMagicAttrIDs: -1,
+  
+  kConstraintEquals: 0,
+  kConstraintIn: 1,
+  kConstraintRanges: 2,
+
   /* ******************* SCHEMA ******************* */
 
-  _schemaVersion: 9,
+  _schemaVersion: 10,
   _schema: {
     tables: {
 
       // ----- Messages
       folderLocations: {
         columns: [
           "id INTEGER PRIMARY KEY",
           "folderURI TEXT NOT NULL",
+          "dirtyStatus INTEGER NOT NULL",
+          "name TEXT NOT NULL",
         ],
 
         triggers: {
           delete: "DELETE from messages WHERE folderID = OLD.id",
         },
       },
 
       conversations: {
@@ -318,16 +356,17 @@ var GlodaDatastore = {
           "messageKey INTEGER",
           "conversationID INTEGER NOT NULL REFERENCES conversations(id)",
           "date INTEGER",
           // we used to have the parentID, but because of the very real
           //  possibility of multiple copies of a message with a given
           //  message-id, the parentID concept is unreliable.
           "headerMessageID TEXT",
           "deleted INTEGER NOT NULL default 0",
+          "jsonAttributes TEXT",
         ],
 
         indices: {
           messageLocation: ['folderID', 'messageKey'],
           headerMessageID: ['headerMessageID'],
           conversationID: ['conversationID'],
           date: ['date'],
           deleted: ['deleted'],
@@ -366,22 +405,16 @@ var GlodaDatastore = {
           "attributeID INTEGER NOT NULL REFERENCES attributeDefinitions(id)",
           "value NUMERIC",
         ],
 
         indices: {
           attribQuery: [
             "attributeID", "value",
             /* covering: */ "conversationID", "messageID"],
-          messageAttribFetch: [
-            "messageID",
-            /* covering required: */ "attributeID", "value"],
-          conversationAttribFetch: [
-            "conversationID",
-            /* covering: */ "messageID", "attributeID", "value"],
         },
       },
 
       // ----- Contacts / Identities
 
       /**
        * Corresponds to a human being and roughly to an address book entry.
        *  Constrast with an identity, which is a specific e-mail address, IRC
@@ -390,17 +423,18 @@ var GlodaDatastore = {
        */
       contacts: {
         columns: [
           "id INTEGER PRIMARY KEY",
           "directoryUUID TEXT",
           "contactUUID TEXT",
           "popularity INTEGER",
           "frecency INTEGER",
-          "name TEXT"
+          "name TEXT",
+          "jsonAttributes TEXT",
         ],
         indices: {
           popularity: ["popularity"],
           frecency: ["frecency"],
         },
       },
 
       contactAttributes: {
@@ -408,19 +442,16 @@ var GlodaDatastore = {
           "contactID INTEGER NOT NULL REFERENCES contacts(id)",
           "attributeID INTEGER NOT NULL REFERENCES attributeDefinitions(id)",
           "value NUMERIC"
         ],
         indices: {
           contactAttribQuery: [
             "attributeID", "value",
             /* covering: */ "contactID"],
-          contactAttribFetch: [
-            "contactID",
-            /* covering */ "attributeID", "value"]
         }
       },
 
       /**
        * Identities correspond to specific e-mail addresses, IRC nicks, etc.
        */
       identities: {
         columns: [
@@ -460,18 +491,20 @@ var GlodaDatastore = {
    */
   asyncConnection: null,
 
   /**
    * Initialize logging, create the database if it doesn't exist, "upgrade" it
    *  if it does and it's not up-to-date, fill our authoritative folder uri/id
    *  mapping.
    */
-  _init: function gloda_ds_init() {
+  _init: function gloda_ds_init(aNsJSON) {
     this._log = Log4Moz.Service.getLogger("gloda.datastore");
+    
+    this._json = aNsJSON;
 
     // Get the path to our global database
     var dirService = Cc["@mozilla.org/file/directory_service;1"].
                      getService(Ci.nsIProperties);
     var dbFile = dirService.get("ProfD", Ci.nsIFile);
     dbFile.append("global-messages-db.sqlite");
 
     // Get the storage (sqlite) service
@@ -668,16 +701,32 @@ var GlodaDatastore = {
       aDBFile.remove(false);
       this._log.warn("Global database has been purged due to schema change.");
       return this._createDB(aDBService, aDBFile);
     }
     // version 9 just adds the contactAttributes table
     if (aCurVersion < 9) {
       this._createTableSchema(aDBConnection, "contactAttributes");
     }
+    // version 10:
+    // we have so many changes here, not to mention semantic changes, that
+    //  purging is the right answer.
+    // - adds dirtyStatus, name to folderLocations
+    // - removes messageAttribFetch index from messageAttributes
+    // - removes conversationAttribFetch index from messageAttributes
+    // - removes contactAttribFetch index from contactAttributes
+    // - adds jsonAttributes column to messages table
+    // - adds jsonAttributes column to contacts table
+    if (aCurVersion < 10) {
+      aDBConnection.close();
+      aDBFile.remove(false);
+      this._log.warn("Global database has been purged due to schema change.");
+      return this._createDB(aDBService, aDBFile);
+    }
+    
     aDBConnection.schemaVersion = aNewVersion;
     
     return aDBConnection;
   },
 
   _outstandingAsyncStatements: [],
 
   _createAsyncStatement: function gloda_ds_createAsyncStatement(aSQLString,
@@ -812,46 +861,58 @@ var GlodaDatastore = {
   },
 
   get _rollbackTransactionStatement() {
     let statement = this._createAsyncStatement("ROLLBACK");
     this.__defineGetter__("_rollbackTransactionStatement", function() statement);
     return this._rollbackTransactionStatement;
   },
 
+  _pendingPostCommitCallbacks: null,
+  /**
+   * Register a callback to be invoked when the current transaction's commit
+   *  completes.
+   */
+  runPostCommit: function gloda_ds_runPostCommit(aCallback) {
+    this._pendingPostCommitCallbacks.push(aCallback);
+  },
+
   /**
    * Begin a potentially nested transaction; only the outermost transaction gets
    *  to be an actual transaction, and the failure of any nested transaction
    *  results in a rollback of the entire outer transaction.  If you really
    *  need an atomic transaction
    */
   _beginTransaction: function gloda_ds_beginTransaction() {
     if (this._transactionDepth == 0) {
+      this._pendingPostCommitCallbacks = [];
       this._beginTransactionStatement.executeAsync(this.trackAsync());
       this._transactionGood = true;
     }
     this._transactionDepth++;
   },
   /**
    * Commit a potentially nested transaction; if we are the outer-most
    *  transaction and no sub-transaction issues a rollback
    *  (via _rollbackTransaction) then we commit, otherwise we rollback.
    */
   _commitTransaction: function gloda_ds_commitTransaction() {
     this._transactionDepth--;
     if (this._transactionDepth == 0) {
       try {
         if (this._transactionGood)
-          this._commitTransactionStatement.executeAsync(this.trackAsync());
+          this._commitTransactionStatement.executeAsync(
+            new PostCommitHandler(this._pendingPostCommitCallbacks));
         else
           this._rollbackTransaction.executeAsync(this.trackAsync());
       }
       catch (ex) {
         this._log.error("Commit problem: " + ex);
       }
+      this._pendingPostCommitCallbacks = [];
     }
   },
   /**
    * Abort the commit of the potentially nested transaction.  If we are not the
    *  outermost transaction, we set a flag that tells the outermost transaction
    *  that it must roll back.
    */
   _rollbackTransaction: function gloda_ds_rollbackTransaction() {
@@ -1019,104 +1080,137 @@ var GlodaDatastore = {
   },
 
   /* ********** Folders ********** */
   /** next folder (row) id to issue, populated by _getAllFolderMappings. */
   _nextFolderId: 1,
 
   get _insertFolderLocationStatement() {
     let statement = this._createAsyncStatement(
-      "INSERT INTO folderLocations (id, folderURI) VALUES (?1, ?2)");
+      "INSERT INTO folderLocations (id, folderURI, dirtyStatus, name) VALUES \
+        (?1, ?2, ?3, ?4)");
     this.__defineGetter__("_insertFolderLocationStatement",
       function() statement);
     return this._insertFolderLocationStatement;
   },
 
   /**
    * Authoritative map from folder URI to folder ID.  (Authoritative in the
    *  sense that this map exactly represents the state of the underlying
    *  database.  If it does not, it's a bug in updating the database.)
    */
-  _folderURIs: {},
+  _folderByURI: {},
   /** Authoritative map from folder ID to folder URI */
-  _folderIDs: {},
+  _folderByID: {},
 
-  /** Intialize our _folderURIs/_folderIDs mappings, called by _init(). */
+  /** Intialize our _folderByURI/_folderByID mappings, called by _init(). */
   _getAllFolderMappings: function gloda_ds_getAllFolderMappings() {
     let stmt = this._createSyncStatement(
-      "SELECT id, folderURI FROM folderLocations", true);
+      "SELECT id, folderURI, dirtyStatus, name FROM folderLocations", true);
 
     while (stmt.executeStep()) {  // no chance of this SQLITE_BUSY on this call
       let folderID = stmt.getInt64(0);
       let folderURI = stmt.getString(1);
-      this._folderURIs[folderURI] = folderID;
-      this._folderIDs[folderID] = folderURI;
+      let dirtyStatus = stmt.getInt32(2);
+      let folderName = stmt.getString(3);
+      
+      let folder = new GlodaFolder(this, folderID, folderURI, dirtyStatus,
+                                   folderName);
+      
+      this._folderByURI[folderURI] = folder;
+      this._folderByID[folderID] = folder;
 
-      if (folderID + 1 > this._nextFolderId)
+      if (folderID >= this._nextFolderId)
         this._nextFolderId = folderID + 1;
     }
     stmt.finalize();
   },
 
-  _folderURIKnown: function gloda_ds_folderURIKnown(aFolderURI) {
-    return aFolderURI in this._folderURIs;
+  _folderKnown: function gloda_ds_folderKnown(aFolder) {
+    let folderURI = aFolder.URI;
+    return folderURI in this._folderByURI;
   },
 
   /**
    * Map a folder URI to a folder ID, creating the mapping if it does not yet
    *  exist.
    */
-  _mapFolderURI: function gloda_ds_mapFolderURI(aFolderURI) {
-    if (aFolderURI in this._folderURIs) {
-      return this._folderURIs[aFolderURI];
+  _mapFolder: function gloda_ds_mapFolderURI(aFolder) {
+    let folderURI = aFolder.URI;
+    if (folderURI in this._folderByURI) {
+      return this._folderByURI[folderURI];
     }
 
     let folderID = this._nextFolderId++;
-    this._insertFolderLocationStatement.bindInt64Parameter(0, folderID)
-    this._insertFolderLocationStatement.bindStringParameter(1, aFolderURI);
+    
+    let folder = new GlodaFolder(this, folderID, folderURI,
+      GlodaFolder.prototype.kFolderFilthy, aFolder.prettyName);
+    
+    this._insertFolderLocationStatement.bindInt64Parameter(0, folder.id)
+    this._insertFolderLocationStatement.bindStringParameter(1, folder.uri);
+    this._insertFolderLocationStatement.bindInt64Parameter(2,
+                                                           folder.dirtyStatus);
+    this._insertFolderLocationStatement.bindStringParameter(3, folder.name);
     this._insertFolderLocationStatement.executeAsync(this.trackAsync());
 
-    this._folderURIs[aFolderURI] = folderID;
-    this._folderIDs[folderID] = aFolderURI;
-    this._log.info("mapping URI " + aFolderURI + " to " + folderID);
-    return folderID;
+    this._folderByURI[aFolderURI] = folder;
+    this._folderByID[folderID] = folder;
+    return folder;
   },
 
   _mapFolderID: function gloda_ds_mapFolderID(aFolderID) {
     if (aFolderID === null)
       return null;
-    if (aFolderID in this._folderIDs)
-      return this._folderIDs[aFolderID];
+    if (aFolderID in this._folderByID)
+      return this._folderByID[aFolderID];
     throw "Got impossible folder ID: " + aFolderID;
   },
 
+  get _updateFolderDirtyStatusStatement() {
+    let statement = this._createAsyncStatement(
+      "UPDATE folderLocations SET dirtyStatus = ?1 \
+              WHERE id = ?2");
+    this.__defineGetter__("_updateFolderDirtyStatusStatement",
+      function() statement);
+    return this._updateFolderDirtyStatusStatement;
+  },
+
+  updateFolderDirtyStatus: function gloda_ds_updateFolderDirtyStatus(aFolder) {
+    let ufds = this._updateFolderDirtyStatusStatement;
+    ufds.bindInt64Parameter(1, folder.id);
+    ufds.bindInt64Parameter(0, folder.dirtyStatus);
+    ufds.executeAsync(this.trackAsync());
+  },
+
   get _updateFolderLocationStatement() {
     let statement = this._createAsyncStatement(
       "UPDATE folderLocations SET folderURI = ?1 \
-              WHERE folderURI = ?2");
+              WHERE id = ?2");
     this.__defineGetter__("_updateFolderLocationStatement",
       function() statement);
     return this._updateFolderLocationStatement;
   },
 
   /**
    * Non-recursive asynchronous folder renaming based on the URI.
    *
    * @TODO provide a mechanism for recursive folder renames or have a higher
    *     layer deal with it and remove this note.
    */
-  renameFolder: function gloda_ds_renameFolder(aOldURI, aNewURI) {
-    let folderID = this._mapFolderURI(aOldURI); // ensure the URI is mapped...
-    this._folderURIs[aNewURI] = folderID;
-    this._folderIDs[folderID] = aNewURI;
-    this._log.info("renaming folder URI " + aOldURI + " to " + aNewURI);
-    this._updateFolderLocationStatement.bindStringParameter(1, aOldURI);
+  renameFolder: function gloda_ds_renameFolder(aOldFolder, aNewURI) {
+    let folder = this._mapFolder(aOldFolder); // ensure the folder is mapped
+    let oldURI = folder.uri; 
+    this._folderByURI[aNewURI] = folder;
+    folder._uri = aNewURI;
+    this._log.info("renaming folder URI " + oldURI + " to " + aNewURI);
+    this._updateFolderLocationStatement.bindStringParameter(1, folder.id);
     this._updateFolderLocationStatement.bindStringParameter(0, aNewURI);
     this._updateFolderLocationStatement.executeAsync(this.trackAsync());
-    delete this._folderURIs[aOldURI];
+    
+    delete this._folderByURI[oldURI];
   },
 
   get _deleteFolderByIDStatement() {
     let statement = this._createAsyncStatement(
       "DELETE FROM folderLocations WHERE id = ?1");
     this.__defineGetter__("_deleteFolderByIDStatement",
       function() statement);
     return this._deleteFolderByIDStatement;
@@ -1275,18 +1369,18 @@ var GlodaDatastore = {
       this._nextMessageId = stmt.getInt64(0) + 1;
     }
     stmt.finalize();
   },
 
   get _insertMessageStatement() {
     let statement = this._createAsyncStatement(
       "INSERT INTO messages (id, folderID, messageKey, conversationID, date, \
-                             headerMessageID) \
-              VALUES (?1, ?2, ?3, ?4, ?5, ?6)");
+                             headerMessageID, jsonAttributes) \
+              VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)");
     this.__defineGetter__("_insertMessageStatement", function() statement);
     return this._insertMessageStatement;
   },
 
   get _insertMessageTextStatement() {
     let statement = this._createAsyncStatement(
       "INSERT INTO messagesText (docid, subject, body, attachmentNames) \
               VALUES (?1, ?2, ?3, ?4)");
@@ -1299,59 +1393,78 @@ var GlodaDatastore = {
    *  of the process of creating a message (the attributes still need to be
    *  completed), it's on the caller's head to call GlodaCollectionManager's
    *  itemAdded method once the message is fully created.
    *
    * This method uses the async connection, any downstream logic that depends on
    *  this message actually existing in the database must be done using an
    *  async query.
    */
-  createMessage: function gloda_ds_createMessage(aFolderURI, aMessageKey,
+  createMessage: function gloda_ds_createMessage(aFolder, aMessageKey,
                               aConversationID, aDatePRTime, aHeaderMessageID,
                               aSubject, aBody, aAttachmentNames) {
     let folderID;
-    if (aFolderURI != null) {
-      folderID = this._mapFolderURI(aFolderURI);
+    if (aFolder != null) {
+      folderID = this._mapFolder(aFolder).id;
     }
     else {
       folderID = null;
     }
 
     let messageID = this._nextMessageId++;
 
+    let message = new GlodaMessage(this, messageID, folderID,
+                            aMessageKey, aConversationID, null,
+                            aDatePRTime ? new Date(aDatePRTime / 1000) : null,
+                            aHeaderMessageID);
+
+    this._log.debug("CreateMessage: " + folderID + ", " + aMessageKey + ", " +
+                    aConversationID + ", " + aDatePRTime + ", " +
+                    aHeaderMessageID);
+
+    // We would love to notify the collection manager about the message at this
+    //  point (at least if it's not a ghost), but we can't yet.  We need to wait
+    //  until the attributes have been indexed, which means it's out of our
+    //  hands.  (Gloda.processMessage does it.)
+
+    return message;
+  },
+  
+  insertMessage: function gloda_ds_insertMessage(aMessage) {
+
     let ims = this._insertMessageStatement;
-    ims.bindInt64Parameter(0, messageID);
-    if (folderID === null)
+    ims.bindInt64Parameter(0, aMessage.id);
+    if (aMessage.folderID === null)
       ims.bindNullParameter(1);
     else
-      ims.bindInt64Parameter(1, folderID);
-    if (aMessageKey === null)
+      ims.bindInt64Parameter(1, aMessage.folderID);
+    if (aMessage.messageKey === null)
       ims.bindNullParameter(2);
     else
-      ims.bindInt64Parameter(2, aMessageKey);
-    ims.bindInt64Parameter(3, aConversationID);
-    if (aDatePRTime === null)
+      ims.bindInt64Parameter(2, aMessage.messageKey);
+    ims.bindInt64Parameter(3, aMessage.conversationID);
+    if (aMessage.date === null)
       ims.bindNullParameter(4);
     else
-      ims.bindInt64Parameter(4, aDatePRTime);
-    ims.bindStringParameter(5, aHeaderMessageID);
+      ims.bindInt64Parameter(4, aMessage.date * 1000);
+    ims.bindStringParameter(5, aMessage.headerMessageID);
+    if (aMessage._jsonText)
+      ims.bindStringParameter(6, aMessage._jsonText);
+    else
+      ims.bindNullParameter(6);
 
     try {
        ims.executeAsync(this.trackAsync());
     }
     catch(ex) {
        throw("error executing statement... " +
              this.asyncConnection.lastError + ": " +
              this.asyncConnection.lastErrorString + " - " + ex);
     }
 
-    this._log.debug("CreateMessage: " + folderID + ", " + aMessageKey + ", " +
-                    aConversationID + ", " + aDatePRTime + ", " +
-                    aHeaderMessageID);
-
     // we only create the full-text row if the body is non-null.
     // so, even though body might be null, we still want to create the
     //  full-text search row
     if (aBody) {
       let imts = this._insertMessageTextStatement;
       imts.bindInt64Parameter(0, messageID);
       imts.bindStringParameter(1, aSubject);
       imts.bindStringParameter(2, aBody);
@@ -1365,64 +1478,58 @@ var GlodaDatastore = {
       }
       catch(ex) {
          throw("error executing fulltext statement... " +
                this.asyncConnection.lastError + ": " +
                this.asyncConnection.lastErrorString + " - " + ex);
       }
     }
 
-    let message = new GlodaMessage(this, messageID, folderID,
-                            aMessageKey, aConversationID, null,
-                            aDatePRTime ? new Date(aDatePRTime / 1000) : null,
-                            aHeaderMessageID);
-
-    // We would love to notify the collection manager about the message at this
-    //  point (at least if it's not a ghost), but we can't yet.  We need to wait
-    //  until the attributes have been indexed, which means it's out of our
-    //  hands.  (Gloda.processMessage does it.)
-
-    return message;
   },
 
   get _updateMessageStatement() {
     let statement = this._createAsyncStatement(
       "UPDATE messages SET folderID = ?1, \
                            messageKey = ?2, \
                            conversationID = ?3, \
                            date = ?4, \
-                           headerMessageID = ?5 \
-              WHERE id = ?6");
+                           headerMessageID = ?5,
+                           jsonAttributes = ?6 \
+              WHERE id = ?7");
     this.__defineGetter__("_updateMessageStatement", function() statement);
     return this._updateMessageStatement;
   },
 
   /**
    * Update the database row associated with the message.  If aBody is supplied,
    *  the associated full-text row is created; it is assumed that it did not
    *  previously exist.
    */
   updateMessage: function gloda_ds_updateMessage(aMessage, aSubject, aBody,
                                                  aAttachmentNames) {
     let ums = this._updateMessageStatement;
-    ums.bindInt64Parameter(5, aMessage.id);
+    ums.bindInt64Parameter(6, aMessage.id);
     if (aMessage.folderID === null)
       ums.bindNullParameter(0);
     else
       ums.bindInt64Parameter(0, aMessage.folderID);
     if (aMessage.messageKey === null)
       ums.bindNullParameter(1);
     else
       ums.bindInt64Parameter(1, aMessage.messageKey);
     ums.bindInt64Parameter(2, aMessage.conversationID);
     if (aMessage.date === null)
       ums.bindNullParameter(3);
     else
       ums.bindInt64Parameter(3, aMessage.date * 1000);
     ums.bindStringParameter(4, aMessage.headerMessageID);
+    if (aMessage._jsonText)
+      ims.bindStringParameter(5, aMessage._jsonText);
+    else
+      ims.bindNullParameter(5);
 
     ums.executeAsync(this.trackAsync());
 
     if (aBody) {
       let imts = this._insertMessageTextStatement;
       imts.bindInt64Parameter(0, aMessage.id);
       imts.bindStringParameter(1, aSubject);
       imts.bindStringParameter(2, aBody);
@@ -1449,19 +1556,19 @@ var GlodaDatastore = {
   },
 
   /**
    * Given a list of gloda message ids, and a list of their new message keys in
    *  the given new folder location, asynchronously update the message's
    *  database locations.  Also, update the in-memory representations.
    */
   updateMessageLocations: function gloda_ds_updateMessageLocations(aMessageIds,
-      aNewMessageKeys, aDestFolderURI) {
+      aNewMessageKeys, aDestFolder) {
     let statement = this._updateMessageLocationStatement;
-    let destFolderID = this._mapFolderURI(aDestFolderURI);
+    let destFolderID = this._mapFolder(aDestFolder).id;
 
     let modifiedItems = [];
 
     for (let iMsg = 0; iMsg < aMessageIds.length; iMsg++) {
       let id = aMessageIds[iMsg]
       statement.bindInt64Parameter(0, destFolderID);
       statement.bindInt64Parameter(1, aNewMessageKeys[iMsg]);
       statement.bindInt64Parameter(2, id);
@@ -1493,20 +1600,20 @@ var GlodaDatastore = {
   },
 
   /**
    * Asynchronously mutate message folder id/message keys for the given
    *  messages, indicating that we are moving them to the target folder, but
    *  don't yet know their target message keys.
    */
   updateMessageFoldersByKeyPurging:
-      function gloda_ds_updateMessageFoldersByKeyPurging(aSrcFolderURI,
-        aMessageKeys, aDestFolderURI) {
-    let srcFolderID = this._mapFolderURI(aSrcFolderURI);
-    let destFolderID = this._mapFolderURI(aDestFolderURI);
+      function gloda_ds_updateMessageFoldersByKeyPurging(aSrcFolder,
+        aMessageKeys, aDestFolder) {
+    let srcFolderID = this._mapFolder(aSrcFolder).id;
+    let destFolderID = this._mapFolder(aDestFolder).id;
 
     let sqlStr = "UPDATE messages SET folderID = ?1, \
                                       messageKey = ?2 \
                    WHERE folderID = ?3 \
                      AND messageKey IN (" + aMessageKeys.join(", ") + ")";
     let statement = this._createAsyncStatement(sqlStr, true);
     statement.bindInt64Parameter(2, srcFolderID);
     statement.bindInt64Parameter(0, destFolderID);
@@ -1583,20 +1690,20 @@ var GlodaDatastore = {
 
   /**
    * Synchronously retrieve the message that we believe to correspond to the
    *  given message key in the given folder.
    * @return null on failure to locate the message, the message on success.
    *
    * @XXX on failure, attempt to resolve the problem through re-indexing, etc.
    */
-  getMessageFromLocation: function gloda_ds_getMessageFromLocation(aFolderURI,
+  getMessageFromLocation: function gloda_ds_getMessageFromLocation(aFolder,
                                                                  aMessageKey) {
     this._selectMessageByLocationStatement.bindInt64Parameter(0,
-      this._mapFolderURI(aFolderURI));
+      this._mapFolder(aFolder).id);
     this._selectMessageByLocationStatement.bindInt64Parameter(1, aMessageKey);
 
     let message = null;
     if (this._syncStep(this._selectMessageByLocationStatement))
       message = this._messageFromRow(this._selectMessageByLocationStatement);
     this._selectMessageByLocationStatement.reset();
 
     if (message === null)
@@ -1803,61 +1910,85 @@ var GlodaDatastore = {
       "INSERT INTO messageAttributes (conversationID, messageID, attributeID, \
                              value) \
               VALUES (?1, ?2, ?3, ?4)");
     this.__defineGetter__("_insertMessageAttributeStatement",
       function() statement);
     return this._insertMessageAttributeStatement;
   },
 
+  get _deleteMessageAttributeStatement() {
+    let statement = this._createAsyncStatement(
+      "DELETE FROM messageAttributes WHERE attributeID = ?1 AND value = ?2 \
+         AND conversationID = ?3 AND messageID = ?4");
+    this.__defineGetter__("_deleteMessageAttributeStatement",
+      function() statement);
+    return this._deleteMessageAttributeStatement;
+  },
+
   /**
-   * Insert a bunch of attributes relating to a GlodaMessage.  This is performed
+   * Insert and remove attributes relating to a GlodaMessage.  This is performed
    *  inside a pseudo-transaction (we create one if we aren't in one, using
    *  our _beginTransaction wrapper, but if we are in one, no additional
    *  meaningful semantics are added).
    * No attempt is made to verify uniqueness of inserted attributes, either
    *  against the current database or within the provided list of attributes.
    *  The caller is responsible for ensuring that unwanted duplicates are
    *  avoided.
-   * Currently, it is expected that this method will be used following a call to
-   *  clearMessageAttributes to wipe out the existing attributes in the
-   *  database.  We will probably try and move to a delta-mechanism in the
-   *  future, avoiding needless database churn for small changes in state.
    *
    * @param aMessage The GlodaMessage the attributes belong to.  This is used
    *     to provide the message id and conversation id.
-   * @param aAttributes A list of attribute tuples, where each tuple contains
-   *     an attribute ID and a value.  Lest you forget, an attribute ID
+   * @param aAddDBAttributes A list of attribute tuples to add, where each tuple
+   *     contains an attribute ID and a value.  Lest you forget, an attribute ID
    *     corresponds to a row in the attribute definition table.  The attribute
    *     definition table stores the 'parameter' for the attribute, if any.
    *     (Which is to say, our frequent Attribute-Parameter-Value triple has
    *     the Attribute-Parameter part distilled to a single attribute id.)
+   * @param aRemoveDBAttributes A list of attribute tuples to remove.
    */
-  insertMessageAttributes: function gloda_ds_insertMessageAttributes(aMessage,
-                                        aAttributes) {
+  adjustMessageAttributes: function gloda_ds_adjustMessageAttributes(aMessage,
+                                        aAddDBAttributes, aRemoveDBAttributes) {
     let imas = this._insertMessageAttributeStatement;
+    let dmas = this._deleteMessageAttributeStatement;
     this._beginTransaction();
     try {
-      for (let iAttribute = 0; iAttribute < aAttributes.length; iAttribute++) {
-        let attribValueTuple = aAttributes[iAttribute];
+      for (let iAttrib = 0; iAttrib < aAddDBAttributes.length; iAttrib++) {
+        let attribValueTuple = aAddDBAttributes[iAttrib];
 
         imas.bindInt64Parameter(0, aMessage.conversationID);
         imas.bindInt64Parameter(1, aMessage.id);
         imas.bindInt64Parameter(2, attribValueTuple[0]);
         // use 0 instead of null, otherwise the db gets upset.  (and we don't
         //  really care anyways.)
         if (attribValueTuple[1] == null)
           imas.bindInt64Parameter(3, 0);
         else if (Math.floor(attribValueTuple[1]) == attribValueTuple[1])
           imas.bindInt64Parameter(3, attribValueTuple[1]);
         else
           imas.bindDoubleParameter(3, attribValueTuple[1]);
         imas.executeAsync(this.trackAsync());
       }
 
+      for (let iAttrib = 0; iAttrib < aRemoveDBAttributes.length; iAttrib++) {
+        let attribValueTuple = aRemoveDBAttributes[iAttrib];
+
+        dmas.bindInt64Parameter(0, attribValueTuple[0]);
+        // use 0 instead of null, otherwise the db gets upset.  (and we don't
+        //  really care anyways.)
+        if (attribValueTuple[1] == null)
+          dmas.bindInt64Parameter(1, 0);
+        else if (Math.floor(attribValueTuple[1]) == attribValueTuple[1])
+          dmas.bindInt64Parameter(1, attribValueTuple[1]);
+        else
+          dmas.bindDoubleParameter(1, attribValueTuple[1]);
+        dmas.bindInt64Parameter(2, aMessage.conversationID);
+        dmas.bindInt64Parameter(3, aMessage.id);
+        dmas.executeAsync(this.trackAsync());
+      }
+
       this._commitTransaction();
     }
     catch (ex) {
       this._rollbackTransaction();
       throw ex;
     }
   },
 
@@ -1927,46 +2058,68 @@ var GlodaDatastore = {
   _stringSQLQuoter: function(aString) {
     return "'" + aString.replace("'", "''", "g") + "'";
   },
   _numberQuoter: function(aNum) {
     return aNum;
   },
 
   /* ===== Generic Attribute Support ===== */
-  insertAttributes: function gloda_ds_insertAttributes(aItem,
-                                        aAttributes) {
+  adjustAttributes: function gloda_ds_adjustAttributes(aItem, aAddDBAttributes,
+      aRemoveDBAttributes) {
     let nounMeta = aItem.NOUN_META;
     let dbMeta = nounMeta._dbMeta;
     if (dbMeta.insertAttrStatement === undefined) {
       dbMeta.insertAttrStatement = this._createAsyncStatement(
         "INSERT INTO " + nounMeta.attrTableName +
         " (" + nounMeta.attrIDColumnName + ", attributeID, value) " +
         " VALUES (?1, ?2, ?3)");
+      // we always create this at the same time (right here), no need to check
+      dbMeta.deleteAttrStatement = this._createAsyncStatement(
+        "DELETE FROM " + nounMeta.attrTableName + " WHERE " +
+        " attributeID = ?1 AND value = ?2 AND " +
+        nounMeta.attrIDColumnName + " = ?3");
     }
 
     let ias = dbMeta.insertAttrStatement;
+    let das = dbMeta.deleteAttrStatement;
     this._beginTransaction();
     try {
-      for (let iAttribute = 0; iAttribute < aAttributes.length; iAttribute++) {
-        let attribValueTuple = aAttributes[iAttribute];
+      for (let iAttr = 0; iAttr < aAddDBAttributes.length; iAttr++) {
+        let attribValueTuple = aAddDBAttributes[iAttr];
 
         ias.bindInt64Parameter(0, aItem.id);
         ias.bindInt64Parameter(1, attribValueTuple[0]);
         // use 0 instead of null, otherwise the db gets upset.  (and we don't
         //  really care anyways.)
         if (attribValueTuple[1] == null)
           ias.bindInt64Parameter(2, 0);
         else if (Math.floor(attribValueTuple[1]) == attribValueTuple[1])
           ias.bindInt64Parameter(2, attribValueTuple[1]);
         else
           ias.bindDoubleParameter(2, attribValueTuple[1]);
         ias.executeAsync(this.trackAsync());
       }
 
+      for (let iAttr = 0; iAttr < aRemoveDBAttributes.length; iAttr++) {
+        let attribValueTuple = aRemoveDBAttributes[iAttr];
+
+        das.bindInt64Parameter(0, attribValueTuple[0]);
+        // use 0 instead of null, otherwise the db gets upset.  (and we don't
+        //  really care anyways.)
+        if (attribValueTuple[1] == null)
+          das.bindInt64Parameter(1, 0);
+        else if (Math.floor(attribValueTuple[1]) == attribValueTuple[1])
+          das.bindInt64Parameter(1, attribValueTuple[1]);
+        else
+          das.bindDoubleParameter(1, attribValueTuple[1]);
+        das.bindInt64Parameter(2, aItem.id);
+        das.executeAsync(this.trackAsync());
+      }
+
       this._commitTransaction();
     }
     catch (ex) {
       this._rollbackTransaction();
       throw ex;
     }
   },
 
@@ -2000,17 +2153,18 @@ var GlodaDatastore = {
    * Perform a database query given a GlodaQueryClass instance that specifies
    *  a set of constraints relating to the noun type associated with the query.
    *  A GlodaCollection is returned containing the results of the look-up.
    *  By default the collection is "live", and will mutate (generating events to
    *  its listener) as the state of the database changes.
    * This functionality is made user/extension visible by the Query's
    *  getCollection (asynchronous) and getAllSync (synchronous).
    */
-  queryFromQuery: function gloda_ds_queryFromQuery(aQuery, aListener, bSynchronous) {
+  queryFromQuery: function gloda_ds_queryFromQuery(aQuery, aListener,
+      bSynchronous, aListenerData) {
     // when changing this method, be sure that GlodaQuery's testMatch function
     //  likewise has its changes made.
     let nounMeta = aQuery._nounMeta;
 
     let whereClauses = [];
     let unionQueries = [aQuery].concat(aQuery._unions);
     let boundArgs = [];
 
@@ -2145,16 +2299,32 @@ var GlodaDatastore = {
 
       if (selects.length)
         whereClauses.push("id IN (" + selects.join(" INTERSECT ") + " )");
     }
 
     let sqlString = "SELECT * FROM " + nounMeta.tableName;
     if (whereClauses.length)
       sqlString += " WHERE " + whereClauses.join(" OR ");
+    
+    if (aQuery._order.length) {
+      let orderClauses = [];
+      for (let [, colName] in Iterator(aQuery._order)) {
+         if (colName[0] == "-")
+           orderClauses.push(colName.substring(1) + " DESC");
+         else
+           orderClauses.push(colName + " ASC");
+      }
+      sqlString += " ORDER BY " + orderClauses.join(", ");
+    }
+    
+    if (aQuery._limit) {
+      sqlString += " LIMIT ?";
+      boundArgs.push(aQuery._limit); 
+    }
 
     this._log.debug("QUERY FROM QUERY: " + sqlString);
 
     let collection;
     if (bSynchronous) {
       let statement = this._createSyncStatement(sqlString, true);
       for (let [iBinding, bindingValue] in Iterator(boundArgs)) {
         this._bindVariant(statement, iBinding, bindingValue);
@@ -2168,148 +2338,185 @@ var GlodaDatastore = {
 
       // have the collection manager attempt to replace the instances we just
       //  created with pre-existing instances.  if the instance didn't exist,
       //  cache the newly observed ones.  We are trading off wastes here; we don't
       //  want to have to ask the collection manager about every row, and we don't
       //  want to invent some alternate row storage.
       GlodaCollectionManager.cacheLoadUnify(nounMeta.id, items);
       collection = new GlodaCollection(nounMeta, items, aQuery, aListener);
+      if (aListenerData !== undefined)
+        collection.data = aListenerData;
 
       GlodaCollectionManager.registerCollection(collection);
     }
     else { // async!
       let statement = this._createAsyncStatement(sqlString, true);
       for (let [iBinding, bindingValue] in Iterator(boundArgs)) {
         this._bindVariant(statement, iBinding, bindingValue);
       }
 
       collection = new GlodaCollection(nounMeta, [], aQuery, aListener);
+      if (aListenerData !== undefined)
+        collection.data = aListenerData;
       GlodaCollectionManager.registerCollection(collection);
 
       statement.executeAsync(new QueryFromQueryCallback(statement, nounMeta,
         collection));
       statement.finalize();
     }
     return collection;
   },
 
-  /**
-   * Deprecated, but still in existence for the benefit of expmess code that
-   *  needs to go away anyways and can take this with it.
-   */
-  queryMessagesAPV: function gloda_ds_queryMessagesAPV(aAPVs) {
-    let selects = [];
-
-    for (let iAPV = 0; iAPV < aAPVs.length; iAPV++) {
-      let APV = aAPVs[iAPV];
-
-      let attributeID;
-      if (APV[1] != null)
-        attributeID = APV[0].bindParameter(APV[1]);
-      else
-        attributeID = APV[0].id;
-      let select = "SELECT messageID FROM messageAttributes WHERE attributeID" +
-                   " = " + attributeID;
-      // straight value match?
-      if (APV.length == 3) {
-        if (APV[2] != null)
-          select += " AND value = " + APV[2];
+  loadNounItem: function gloda_ds_loadNounItem(aItem, aReferencesByNounID) {
+    let jsonDict = this._json.decode(aItem._jsonText);
+    delete aItem._jsonText;
+    
+    let attribIDToDef = this._attributeIDToDef;
+    
+    let deps = {};
+    let hasDeps = false;
+    
+    // Iterate over the attributes on the item
+    for each (let [attribId, jsonValue] in Iterator(jsonDict)) {
+      // find the attribute definition that corresponds to this key
+      let attrib = attribIDToDef[attribId][0];
+      // the attribute should only fail to exist if an extension was removed
+      if (attrib === undefined)
+        continue;
+      
+      let objectNounDef = attrib.objectNounDef;
+      
+      // if it has a tableName member, then it's a persistent object that needs
+      //  to be loaded, which also means we need to hold it in a collection
+      //  owned by our collection.
+      if (objectNounDef.tableName) {
+        let references = aReferencesByNounID[objectNounDef.id];
+        if (references === undefined)
+          references = aReferencesByNounID[objectNounDef.id] = {};
+          
+        if (attrib.singular)
+          references[jsonValue] = null;
+        else {
+          for each (let [, anID] in Iterator(jsonValue))
+            references[anID] = null;
+        }
+        
+        deps[attrib] = jsonValue;
+        hasDeps = true;
       }
-      else { // APV.length == 4, so range match
-        // BETWEEN is optimized to >= and <=, or we could just do that ourself.
-        //  (in other words, this shouldn't hurt our use of indices)
-        select += " AND value BETWEEN " + APV[2] + " AND " + APV[3];
+      /* if it has custom contribution logic, use it */
+      else if (objectNounDef.contributeObjDependencies) {
+        if (objectNounDef.contributeObjDependencies(jsonValue,
+                                                    aReferencesByNounID)) {
+          deps[attrib] = jsonValue;
+          hasDeps = true;
+        }
+        else // just propagate the value, it's some form of simple sentinel
+          aItem[attrib.boundName] = jsonValue;
       }
-      selects.push(select);
+      // otherwise, the value just needs to be de-persisted, or not
+      else if (objectNounDef.fromJSON) {
+        if (attrib.singular)
+          aItem[attrib.boundName] = objectNounDef.fromJSON(jsonValue);
+        else
+          aItem[attrib.boundName] = [objectNounDef.fromJSON(val) for each
+            ([, val] in Iterator(jsonValue)];
+      }
+      // it's fine as is
+      else
+        aItem[attrib.boundName] = jsonValue;
     }
-
-    let sqlString = "SELECT * FROM messages WHERE id IN (" +
-                    selects.join(" INTERSECT ") + " )";
-    let statement = this._createSyncStatement(sqlString, true);
-
-    let messages = [];
-    while (this._syncStep(statement)) {
-      messages.push(this._messageFromRow(statement));
-    }
-    statement.finalize();
-
-    if (messages.length)
-      GlodaCollectionManager.cacheLoadUnify(GlodaMessage.prototype.NOUN_ID,
-                                            messages);
-
-    return messages;
+    
+    if (hasDeps)
+      aItem._deps = deps;
+    return hasDeps;
   },
 
   /* ********** Contact ********** */
   _nextContactId: 1,
 
   _populateContactManagedId: function () {
     let stmt = this._createSyncStatement("SELECT MAX(id) FROM contacts", true);
     if (stmt.executeStep()) {  // no chance of this SQLITE_BUSY on this call
       this._nextContactId = stmt.getInt64(0) + 1;
     }
     stmt.finalize();
   },
 
   get _insertContactStatement() {
     let statement = this._createAsyncStatement(
       "INSERT INTO contacts (id, directoryUUID, contactUUID, name, popularity,\
-                             frecency) \
-              VALUES (?1, ?2, ?3, ?4, ?5, ?6)");
+                             frecency, jsonAttributes) \
+              VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)");
     this.__defineGetter__("_insertContactStatement", function() statement);
     return this._insertContactStatement;
   },
 
   createContact: function gloda_ds_createContact(aDirectoryUUID, aContactUUID,
       aName, aPopularity, aFrecency) {
     let contactID = this._nextContactId++;
-    let ics = this._insertContactStatement;
-    ics.bindInt64Parameter(0, contactID);
-    if (aDirectoryUUID == null)
-      ics.bindNullParameter(1);
-    else
-      ics.bindStringParameter(1, aDirectoryUUID);
-    if (aContactUUID == null)
-      ics.bindNullParameter(2);
-    else
-      ics.bindStringParameter(2, aContactUUID);
-    ics.bindStringParameter(3, aName);
-    ics.bindInt64Parameter(4, aPopularity);
-    ics.bindInt64Parameter(5, aFrecency);
-
-    ics.executeAsync(this.trackAsync());
 
     let contact = new GlodaContact(this, contactID,
                                    aDirectoryUUID, aContactUUID, aName,
                                    aPopularity, aFrecency);
     GlodaCollectionManager.itemsAdded(contact.NOUN_ID, [contact]);
     return contact;
   },
+  
+  insertContact: function gloda_ds_insertContact(aContact) {
+    let ics = this._insertContactStatement;
+    ics.bindInt64Parameter(0, aContact.id);
+    if (aContact.directoryUUID == null)
+      ics.bindNullParameter(1);
+    else
+      ics.bindStringParameter(1, aContact.directoryUUID);
+    if (aContact.contactUUID == null)
+      ics.bindNullParameter(2);
+    else
+      ics.bindStringParameter(2, aContact.contactUUID);
+    ics.bindStringParameter(3, aContact.name);
+    ics.bindInt64Parameter(4, aContact.popularity);
+    ics.bindInt64Parameter(5, aContact.frecency);
+    if (aContact._jsonText)
+      ims.bindStringParameter(6, aContact._jsonText);
+    else
+      ims.bindNullParameter(6);
+
+    ics.executeAsync(this.trackAsync());
+
+    GlodaCollectionManager.itemsAdded(contact.NOUN_ID, [contact]);
+    return contact;
+  },
 
   get _updateContactStatement() {
     let statement = this._createAsyncStatement(
       "UPDATE contacts SET directoryUUID = ?1, \
                            contactUUID = ?2, \
                            name = ?3, \
                            popularity = ?4, \
-                           frecency = ?5 \
-                       WHERE id = ?6");
+                           frecency = ?5,
+                           jsonAttributes = ?6 \
+                       WHERE id = ?7");
     this.__defineGetter__("_updateContactStatement", function() statement);
     return this._updateContactStatement;
   },
 
   updateContact: function gloda_ds_updateContact(aContact) {
     let ucs = this._updateContactStatement;
-    ucs.bindInt64Parameter(5, aContact.id);
+    ucs.bindInt64Parameter(6, aContact.id);
     ucs.bindStringParameter(0, aContact.directoryUUID);
     ucs.bindStringParameter(1, aContact.contactUUID);
     ucs.bindStringParameter(2, aContact.name);
     ucs.bindInt64Parameter(3, aContact.popularity);
     ucs.bindInt64Parameter(4, aContact.frecency);
+    if (aContact._jsonText)
+      ims.bindStringParameter(5, aContact._jsonText);
+    else
+      ims.bindNullParameter(5);
 
     ucs.executeAsync(this.trackAsync());
   },
 
   _contactFromRow: function gloda_ds_contactFromRow(aRow) {
     let directoryUUID, contactUUID;
     if (aRow.getTypeOfIndex(1) == Ci.mozIStorageValueArray.VALUE_TYPE_NULL)
       directoryUUID = null;
--- a/modules/explattr.js
+++ b/modules/explattr.js
@@ -103,24 +103,16 @@ var GlodaExplicitAttr = {
                         bindName: "tags",
                         singular: false,
                         subjectNouns: [Gloda.NOUN_MESSAGE],
                         objectNoun: Gloda.NOUN_TAG,
                         parameterNoun: null,
                         // Property change notifications that we care about:
                         propertyChanges: ["keywords"],
                         }); // not-tested
-    Gloda.defineNounAction(Gloda.NOUN_TAG, {
-      actionType: "filter", actionTarget: Gloda.NOUN_TAG,
-      shortName: "same tag",
-      makeConstraint: function(aAttrDef, aTagged) {
-        return [GlodaExplicitAttr._attrTag].concat(
-          TagNoun.toParamAndValue(aTagged, true));
-      },
-      });
 
     // Star
     this._attrStar = Gloda.defineAttribute({
                         provider: this,
                         extensionName: Gloda.BUILT_IN,
                         attributeType: Gloda.kAttrExplicit,
                         attributeName: "star",
                         bind: true,
@@ -141,35 +133,33 @@ var GlodaExplicitAttr = {
                         subjectNouns: [Gloda.NOUN_MESSAGE],
                         objectNoun: Gloda.NOUN_BOOLEAN,
                         parameterNoun: null,
                         }); // tested-by: test_attributes_explicit
     
   },
   
   process: function Gloda_explattr_process(aGlodaMessage, aMsgHdr, aMimeMsg) {
-    let attribs = [];
+    aGlodaMessage.flagged = aMsgHdr.isFlagged;
+    aGlodeMessage.read = aMsgHdr.isRead;
     
-    attribs.push([this._attrStar.id, aMsgHdr.isFlagged ? 1 : 0]);
-    attribs.push([this._attrRead.id, aMsgHdr.isRead ? 1 : 0]);
+    let tags = aGlodaMessage.tags = [];
     
     // -- Tag
     // build a map of the keywords
     let keywords = aMsgHdr.getStringProperty("keywords");
     let keywordList = keywords.split(' ');
     let keywordMap = {};
     for (let iKeyword = 0; iKeyword < keywordList.length; iKeyword++) {
       let keyword = keywordList[iKeyword];
       keywordMap[keyword] = true;
     }
 
-    let nowPRTime = Date.now() * 1000;
-
     let tagArray = this._msgTagService.getAllTags({});
     for (let iTag = 0; iTag < tagArray.length; iTag++) {
       let tag = tagArray[iTag];
       if (tag.key in keywordMap)
-        attribs.push([this._attrTag, tag.key, nowPRTime]);
+        tags.push(tag);
     }
     
-    return attribs;
+    yield Gloda.kWorkDone;
   },
 };
--- a/modules/fundattr.js
+++ b/modules/fundattr.js
@@ -164,38 +164,16 @@ var GlodaFundAttr = {
                         attributeType: Gloda.kAttrFundamental,
                         attributeName: "cc",
                         bind: true,
                         singular: false,
                         subjectNouns: [Gloda.NOUN_MESSAGE],
                         objectNoun: Gloda.NOUN_IDENTITY,
                         }); // not-tested
 
-    Gloda.defineNounAction(Gloda.NOUN_IDENTITY, {actionType: "filter",
-      actionTarget: Gloda.NOUN_MESSAGE,
-      shortName: "from",
-      makeConstraint: function(aAttrDef, aIdentity) {
-        return [GlodaFundAttr._attrFrom, null, aIdentity.id];
-      },
-      });
-    Gloda.defineNounAction(Gloda.NOUN_IDENTITY, {actionType: "filter",
-      actionTarget: Gloda.NOUN_MESSAGE,
-      shortName: "to",
-      makeConstraint: function(aAttrDef, aIdentity) {
-        return [GlodaFundAttr._attrTo, null, aIdentity.id];
-      },
-      });
-    Gloda.defineNounAction(Gloda.NOUN_IDENTITY, {actionType: "filter",
-      actionTarget: Gloda.NOUN_MESSAGE,
-      shortName: "cc",
-      makeConstraint: function(aAttrDef, aIdentity) {
-        return [GlodaFundAttr._attrCc, null, aIdentity.id];
-      },
-      });
-
     // Date.  now lives on the row.
     this._attrDate = Gloda.defineAttribute({
                         provider: this,
                         extensionName: Gloda.BUILT_IN,
                         attributeType: Gloda.kAttrFundamental,
                         attributeName: "date",
                         bind: false,
                         singular: true,
@@ -301,79 +279,111 @@ var GlodaFundAttr = {
    * Specializations:
    * - Mailing Lists.  Replies to a message on a mailing list frequently only
    *   have the list-serve as the 'to', so we try to generate a synthetic 'to'
    *   based on the author of the parent message when possible.  (The 'possible'
    *   part is that we may not have a copy of the parent message at the time of
    *   processing.)
    * - Newsgroups.  Same deal as mailing lists.
    */
-  process: function gloda_fundattr_process(aGlodaMessage, aMsgHdr, aMimeMsg,
-                                           aIsNew) {
-    let attribs = [];
+  process: function gloda_fundattr_process(aGlodaMessage, aRawReps,
+                                           aIsNew, aCallbackHandle) {
+    let aMsgHdr = aRawReps.header;
+    let aMimeMsg = aRawReps.mime;
+    
+    let attribs = aGlodaMessage.attributes;
+    let optimizations = aGlobaMessage.optimizationAttributes;
+    
     let involvedIdentities = {};
     
+    let involved = aGlodaMessage.involved;
+    if (involved === undefined)
+      involved = aGlodaMessage.involved = [];
+    let to = aGlodaMessage.to;
+    if (to === undefined)
+      to = aGlodaMessage.to = [];
+    let cc = aGlodaMessage.cc;
+    if (cc === undefined)
+      cc = aGlodaMessage.cc = [];
+    
+    // me specialization optimizations
+    let toMe = aGlodaMessage.toMe;
+    if (toMe === undefined)
+      toMe = aGlodaMessage.toMe = [];
+    let fromMeTo = aGlodaMessage.fromMeTo;
+    if (fromMeTo === undefined)
+      fromMeTo = aGlodaMessage.fromMeTo = [];
+    let ccMe = aGlodaMessage.ccMe;
+    if (ccMe === undefineD)
+      ccMe = aGlodaMEssage.ccMe = [];
+    let fromMeCc = aGlodaMessage.fromMeCc;
+    if (fromMeCc === undefined)
+      fromMeCc = aGlodaMessage.fromMeCc = [];
+    
     // -- From
     // Let's use replyTo if available.
     // er, since we are just dealing with mailing lists for now, forget the
     //  reply-to...
     // TODO: deal with default charset issues
     let author = null;
     /*
     try {
       author = aMsgHdr.getStringProperty("replyTo");
     }
     catch (ex) {
     }
     */
     if (author == null || author == "")
       author = aMsgHdr.author;
+    
+    let [authorIdentities, toIdentities, ccIdentities] =
+      yield aCallbackHandle.pushAndGo(
+        Gloda.getOrCreateMailIdentities(aCallbackHandle,
+                                        author, aMsgHdr.recipients,
+                                        aMsgHdr.ccList));
 
-    let authorIdentity = Gloda.getIdentityForFullMailAddress(author);
-    if (authorIdentity == null) {
+    if (authorIdentities.length == 0) {
       this._log.error("Message with subject '" + aMsgHdr.mime2DecodedSubject +
                       "' somehow lacks a valid author.  Bailing.");
       return attribs;
     }
-    attribs.push([this._attrFrom.id, authorIdentity.id]);
-    attribs.push([this._attrInvolves.id, authorIdentity.id]);
-    involvedIdentities[authorIdentity.id] = true;
+    aGlodaMessage.from = authorIdentities[0];
+    involved.push(authorIdentities[0]);
+    involvedIdentities[authorIdentities[0].id] = true;
     
     let myIdentities = Gloda.myIdentities; // needless optimization?
     let isFromMe = authorIdentity.id in myIdentities;
     
     // -- To, Cc
     // TODO: handle mailing list semantics (use my visterity logic as a first
     //  pass.)
-    let toIdentities = Gloda.getIdentitiesForFullMailAddresses(
-                           aMsgHdr.recipients);
     for (let iTo = 0; iTo < toIdentities.length; iTo++) {
       let toIdentity = toIdentities[iTo];
-      attribs.push([this._attrTo.id, toIdentity.id]);
+      to.push(toIdentity);
       if (!(toIdentity.id in involvedIdentities)) {
-        attribs.push([this._attrInvolves.id, toIdentity.id]);
+        involved.push(toIdentity);
         involvedIdentities[toIdentity.id] = true;
       }
       // optimization attribute to-me ('I' am the parameter)
       if (toIdentity.id in myIdentities) {
         attribs.push([this._attrCcMe.bindParameter(toIdentity.id),
                       authorIdentity.id]);
         if (aIsNew)
           authorIdentity.contact.popularity += this.POPULARITY_TO_ME;
       }
       // optimization attribute from-me-to ('I' am the parameter)
       if (isFromMe) {
+        fromMeTo.push(
         attribs.push([this._attrFromMeCc.bindParameter(authorIdentity.id),
                       toIdentity.id]);
         // also, popularity
         if (aIsNew)
           toIdentity.contact.popularity += this.POPULARITY_FROM_ME_TO;
       }
     }
-    let ccIdentities = Gloda.getIdentitiesForFullMailAddresses(aMsgHdr.ccList);
     for (let iCc = 0; iCc < ccIdentities.length; iCc++) {
       let ccIdentity = ccIdentities[iCc];
       attribs.push([this._attrCc.id, ccIdentity.id]);
       if (!(ccIdentity.id in involvedIdentities)) {
         attribs.push([this._attrInvolves.id, ccIdentity.id]);
         involvedIdentities[ccIdentity.id] = true;
       }
       // optimization attribute cc-me ('I' am the parameter)
@@ -396,11 +406,11 @@ var GlodaFundAttr = {
     // TODO: deal with mailing lists, including implicit-to.  this will require
     //  convincing the indexer to pass us in the previous message if it is
     //  available.  (which we'll simply pass to everyone... it can help body
     //  logic for quoting purposes, etc. too.)
     
     // -- Date
     attribs.push([this._attrDate.id, aMsgHdr.date]);
     
-    return attribs;
+    yield Gloda.kWorkDone;
   },
 };
--- a/modules/gloda.js
+++ b/modules/gloda.js
@@ -146,17 +146,18 @@ var Gloda = {
    *  current user (based on accounts).
    *
    * Additional nouns and the core attribute providers are initialized by the
    *  everybody.js module which ensures all of those dependencies are loaded
    *  (and initialized).
    */
   _init: function gloda_ns_init() {
     this._initLogging();
-    GlodaDatastore._init();
+    this._json = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
+    GlodaDatastore._init(this._jsn);
     this._initAttributes();
     this._initMyIdentities();
   },
 
   _log: null,
   /**
    * Initialize logging; the error console window gets Warning/Error, and stdout
    *  (via dump) gets everything.
@@ -178,89 +179,135 @@ var Gloda = {
     this._log.info("Logging Initialized");
   },
 
   kIndexerIdle: 0,
   kIndexerIndexing: 1,
   kIndexerMoving: 2,
   kIndexerRemoving: 3,
 
+  /** Synchronous activities performed, you can drive us more. */
+  kWorkSync: 0,
+  /**
+   * Asynchronous activity performed, you need to relinquish flow control and
+   *  trust us to call callbackDriver later.
+   */
+  kWorkAsync: 1,
+  /**
+   * We are all done with our task, close us and figure out something else to do.
+   */
+  kWorkDone: 2,
+  /**
+   * We are not done with our task, but we think it's a good idea to take a
+   *  breather.
+   */
+  kWorkPause: 3,
+  /**
+   * We are done with our task, and have a result that we are returning.  This
+   *  should only be used by your callback handler's doneWithResult method.
+   *  Ex: you are passed aCallbackHandle, and you do
+   *  "yield aCallbackHandle.doneWithResult(myResult);".
+   */
+  kWorkDoneWithResult: 4,
+
   /**
    * Lookup a gloda message from an nsIMsgDBHdr.
    *
    * @param aMsgHdr The header of the message you want the gloda message for.
    *
    * @return the gloda messages that corresponds to the provided nsIMsgDBHdr
    *    if one exists, null if one cannot be found.
    */
   getMessageForHeader: function gloda_ns_getMessageForHeader(aMsgHdr) {
-    return GlodaDatastore.getMessageFromLocation(aMsgHdr.folder.URI,
+    return GlodaDatastore.getMessageFromLocation(aMsgHdr.folder,
                                                  aMsgHdr.messageKey);
   },
   
   getFolderForFolder: function gloda_ns_getFolderForFolder(aMsgFolder) {
-    let uri = aMsgFolder.URI;
-    return new GlodaFolder(GlodaDatastore, GlodaDatastore._mapFolderURI(uri),
-                           uri, aMsgFolder.prettyName);
+    return GlodaDatastore._mapFolder(aMsgFolder);
   },
-
+  
   /**
    * Given one or more full mail addresses (ex: "Bob Smith" <bob@smith.com>),
    *  return a list of the identities that corresponds to each mail address,
    *  creating them as required.
    */
-  getIdentitiesForFullMailAddresses:
-      function gloda_ns_getIdentitiesForMailAddresses(aMailAddresses) {
-    let parsed = GlodaUtils.parseMailAddresses(aMailAddresses);
-
-    let identities = [];
-    for (let iAddress = 0; iAddress < parsed.count; iAddress++) {
-      let identity = GlodaDatastore.getIdentity("email",
-                                                parsed.addresses[iAddress]);
-
-      if (identity === null) {
-        let name = parsed.names[iAddress];
-        let mailAddr = parsed.addresses[iAddress];
-
-        // fall-back to the mail address if the name is empty
-        if ((name === null) || (name == ""))
-          name = mailAddr;
-
-        // we must create a contact
-        let contact = GlodaDatastore.createContact(null, null, name, 0, 0);
-
-        // we must create the identity.  use a blank description because there's
-        //  nothing to differentiate it from other identities, as this contact
-        //  only has one initially (us).
-        identity = GlodaDatastore.createIdentity(contact.id, contact, "email",
-                                                 mailAddr,
-                                                 "", false);
+  getOrCreateMailIdentities:
+      function gloda_ns_getOrCreateMailIdentities(aCallbackHandle) {
+    let addresses = {};
+    let resultLists = [];
+    
+    for (let iArg = 1; iArg < arguments.length; iArg++) {
+      let aMailAddresses = arguments[iArg];
+      let parsed = GlodaUtils.parseMailAddresses(aMailAddresses);
+      
+      let resultList = [];
+      resultLists.push(resultList);
+      
+      let identities = [];
+      for (let iAddress = 0; iAddress < parsed.count; iAddress++) {
+        let address = parsed.addresses[iAddress];
+        if (address in addresses)
+          addresses[address].push(resultList);
+        else
+          addresses[address] = [parsed.names[iAddress], resultList];
       }
-      identities.push(identity);
     }
 
-    return identities;
-  },
+    let query = this.newQuery(this.NOUN_IDENTITY);
+    query.kind("email");
+    query.value.apply(query.value, [address for (address in addresses)]);
+    let collection = query.getCollection(aCallbackHandle);
+    yield this.kWorkAsync;
+
+    // put the identities in the appropriate result lists
+    for each (let [, identity] in Iterator(collection.items)) {
+      let nameAndResultLists = addresses[identity.value];
+      // index 0 is the name, skip it
+      for (let iResList = 1; iResList < nameAndResultLists.length; iResList++)
+        nameAndResultLists[iResList].push(identity);
+      }
+      delete addresses[identity.value];
+    }
+    
+    // create the identities that did not exist yet
+    for each (let [address, nameAndResultLists] in Iterator(addresses)) {
+      let name = nameAndResultsLists[0]; 
 
-  /**
-   * Given a full mail address (ex: "Bob Smith" <bob@smith.com>), return the
-   *  identity that corresponds to that mail address, creating it if required.
-   *  (If you want the contact, it is easily retrieved via the 'contact'
-   *  attribute on the identity.)
-   */
-  getIdentityForFullMailAddress:
-      function gloda_ns_getIdentityForFullMailAddress(aMailAddress) {
-    let identities = this.getIdentitiesForFullMailAddresses(aMailAddress);
-    if (identities.length != 1) {
-      this._log.info("Expected exactly 1 address, got " + identities.length +
-                     " for address: " + aMailAddress);
-      return null;
+      // try and find an existing address book contact.
+      let card = GlodaUtils.getCardForEmail();
+      // XXX when we have the address book GUID stuff, we need to use that to
+      //  find existing contacts... (this will introduce a new query phase
+      //  where we batch all the GUIDs for an async query)
+      // XXX when the address book supports multiple e-mail addresses, we
+      //  should also just create identities for any that don't yet exist
+
+      let contact = GlodaDatastore.createContact(null, null, name, 0, 0);
+      // give the address book indexer a chance if we have a card.
+      // (it will fix-up the name based on the card as appropriate)
+      if (card)
+        yield aCallbackHandle.pushAndGo(
+          Gloda.grokNounItem(contact, card, true));
+      else // grokNounItem will issue the insert for us...
+        GlodaDatastore.insertContact(contact);
+
+      // we must create the identity.  use a blank description because there's
+      //  nothing to differentiate it from other identities, as this contact
+      //  only has one initially (us).
+      // XXX when we have multiple e-mails and there is a meaning associated
+      //  with each e-mail, try and use that to populate the description. 
+      let identity = GlodaDatastore.createIdentity(contact.id, contact,
+        "email", mailAddr, /* description */ "", /* relay? */ false);
+      
+      for (let iResList = 1; iResList < nameAndResultLists.length; iResList++)
+        nameAndResultLists[iResList].push(identity);
+      }
     }
 
-    return identities[0];
+    yield aCallbackHandle.doneWithResult(resultLists);
   },
 
   /**
    * Dictionary of the user's known identities; key is the identity id, value
    *  is the actual identity.  This is populated by _initMyIdentities based on
    *  the accounts defined.
    */
   myIdentities: {},
@@ -338,16 +385,17 @@ var Gloda = {
     if (existingIdentities.length) {
       // just use the first guy's contact
       myContact = existingIdentities[0].contact;
     }
     else {
       // create a new contact
       myContact = GlodaDatastore.createContact(null, null, fullName || "Me",
                                                0, 0);
+      GlodaDatastore.insertContact(myContact);
     }
 
     if (identitiesToCreate.length) {
       for (let iIdentity = 0; iIdentity < identitiesToCreate.length;
           iIdentity++) {
         let emailAddress = identitiesToCreate[iIdentity];
         // XXX this won't always be of type "email" as we add new account types
         // XXX the blank string could be trying to differentiate; we do have
@@ -406,29 +454,29 @@ var Gloda = {
    *  first class noun.)
    */
   kSpecialNotAtAll: 0,
   /**
    * This attribute is stored as a numeric column on the row for the noun.  The
    *  attribute definition should include this value as 'special' and the
    *  column name that stores the attribute as 'specialColumnName'.
    */
-  kSpecialColumn: 1,
+  kSpecialColumn: GlodaDatastore.kSpecialColumn,
   /**
    * This attribute is stored as a string column on the row for the noun.  It
    *  differs from kSpecialColumn in that it is a string and thus uses different
    *  query mechanisms.
    */
-  kSpecialString: 2,
+  kSpecialString: GlodaDatastore.kSpecialString,
   /**
    * This attribute is stored as a fulltext column on the fulltext table for
    *  the noun.  The attribute defintion should include this value as 'special'
    *  and the column name that stores the table as 'specialColumnName'.
    */
-  kSpecialFulltext: 3,
+  kSpecialFulltext: GlodaDatastore.kSpecialFulltext,
 
   /**
    * The extensionName used for the attributes defined by core gloda plugins
    *  such as fundattr.js and explattr.js.
    */
   BUILT_IN: "built-in",
 
 
@@ -542,29 +590,33 @@ var Gloda = {
    */
   _nounNameToNounID: {},
   /**
    * Maps noun IDs to noun meta dictionaries.  (Noun meta dictionaries being
    *  the dictionary provided to us at the time a noun was defined, plus some
    *  additional stuff we put in there.)
    */
   _nounIDToMeta: {},
+  
+  _managedToJSON: function gloda_ns_managedToJSON(aItem) {
+    return aItem.id;
+  }
 
   /**
    * Define a noun.  Takes a dictionary with the following keys/values:
    *
    * @param name The name of the noun.  This is not a display name (anything
    *     being displayed needs to be localized, after all), but simply the
    *     canonical name for debugging purposes and for people to pass to
    *     lookupNoun.  The suggested convention is lower-case-dash-delimited,
    *     with names being singular (since it's a single noun we are referring
    *     to.)
    * @param class The 'class' to which an instance of the noun will belong (aka
    *     will pass an instanceof test).
-   * @param firstClass Is this a 'first class noun'/can it be a subject, AKA can
+   * @param allowsArbitraryAttrs Is this a 'first class noun'/can it be a subject, AKA can
    *     this noun have attributes stored on it that relate it to other things?
    *     For example, a message is first-class; we store attributes of
    *     messages.  A date is not first-class now, nor is it likely to be; we
    *     will not store attributes about a date, although dates will be the
    *     objects of other subjects.  (For example: we might associate a date
    *     with a calendar event, but the date is an attribute of the calendar
    *     event and not vice versa.)
    * @param usesParameter A boolean indicating whether this noun requires use
@@ -591,30 +643,38 @@ var Gloda = {
     aNounMeta.id = aNounID;
     // if it has a table, you can query on it.  seems straight-forward.
     if (aNounMeta.tableName) {
       [aNounMeta.queryClass, aNounMeta.explicitQueryClass,
        aNounMeta.wildcardQueryClass] =
           GlodaQueryClassFactory(aNounMeta);
       aNounMeta._dbMeta = {};
       aNounMeta.class.prototype.NOUN_META = aNounMeta;
+      aNounMeta.toJSON = this._managedToJSON;
     }
     if (aNounMeta.cache) {
       let cacheCost = aNounMeta.cacheCost || 1024;
       let cacheBudget = aNounMeta.cacheBudget || 128 * 1024;
       let cacheSize = Math.floor(cacheBudget / cacheCost);
       if (cacheSize)
         GlodaCollectionManager.defineCache(aNounMeta, cacheSize);
     }
+    if (aNounMeta.allowsArbitraryAttrs) {
+      aNounMeta.attribsByBoundName = {};
+    }
     this._nounNameToNounID[aNounMeta.name] = aNounID;
     this._nounIDToMeta[aNounID] = aNounMeta;
     aNounMeta.actions = [];
     
     this._attrProviderOrderByNoun[aNounMeta.id] = [];
     this._attrProvidersByNoun[aNounMeta.id] = {};
+    
+    if (aNounMeta.tableName) {
+      
+    }
   },
 
   /**
    * Lookup a noun (ID) suitable for passing to defineAttribute's various
    *  noun arguments.  Throws an exception if the noun with the given name
    *  cannot be found; the assumption is that you can't live without the noun.
    */
   lookupNoun: function gloda_ns_lookupNoun(aNounName) {
@@ -701,86 +761,83 @@ var Gloda = {
    *  SQL table def and helper code from datastore.js (and this code) to their
    *  own noun_*.js files.  There are some trade-offs to be made, and I think
    *  we can deal with those once we start to integrate lightning/calendar and
    *  our noun space gets large and more heterogeneous.
    */
   _initAttributes: function gloda_ns_initAttributes() {
     this.defineNoun({
       name: "bool",
-      class: Boolean, firstClass: false,
+      class: Boolean, allowsArbitraryAttrs: false,
       fromParamAndValue: function(aParam, aVal) {
         if(aVal != 0) return true; else return false;
       },
       toParamAndValue: function(aBool) {
         return [null, aBool ? 1 : 0];
       }}, this.NOUN_BOOLEAN);
     this.defineNoun({
       name: "number",
-      class: Number, firstClass: false, continuous: true,
+      class: Number, allowsArbitraryAttrs: false, continuous: true,
       fromParamAndValue: function(aIgnoredParam, aNum) {
         return aNum;
       },
       toParamAndValue: function(aNum) {
         return [null, aNum];
       }}, this.NOUN_NUMBER);
     this.defineNoun({
       name: "string",
-      class: String, firstClass: false,
+      class: String, allowsArbitraryAttrs: false,
       fromParamAndValue: function(aIgnoredParam, aString) {
         return aString;
       },
       toParamAndValue: function(aString) {
         return [null, aString];
       }}, this.NOUN_STRING);
     this.defineNoun({
       name: "date",
-      class: Date, firstClass: false, continuous: true,
+      class: Date, allowsArbitraryAttrs: false, continuous: true,
       fromParamAndValue: function(aParam, aPRTime) {
         return new Date(aPRTime / 1000);
       },
       toParamAndValue: function(aDate) {
         return [null, aDate.valueOf() * 1000];
       }}, this.NOUN_DATE);
     this.defineNoun({
       name: "fulltext",
-      class: String, firstClass: false, continuous: false,
+      class: String, allowsArbitraryAttrs: false, continuous: false,
       // as noted on NOUN_FULLTEXT, we just pass the string around.  it never
       //  hits the database, so it's okay.
       fromParamAndValue: function(aParam, aString) {
         return aString;
       },
       toParamAndValue: function(aString) {
         return [null, aString];
       }}, this.NOUN_FULLTEXT);
 
     this.defineNoun({
       name: "folder",
       class: GlodaFolder,
-      firstClass: false,
+      allowsArbitraryAttrs: false,
       fromParamAndValue: function(aParam, aID) {
-        // XXX map into folder-space rather than uri-space
-        // (this does not pose an immediate problem because folders are
-        //  currently special attributes because they are cols on message rows.)
         return GlodaDatastore._mapFolderID(aID);
       },
-      toParamAndValue: function(aFolderOrURI) {
-        if (aFolderOrURI instanceof GlodaFolder)
+      toParamAndValue: function(aFolderOrGlodaFolder) {
+        if (aFolderOrGlodaFolder instanceof GlodaFolder)
           return [null, aFolderOrURI.id];
         else
-          return [null, GlodaDatastore._mapFolderURI(aFolderOrURI)];
+          return [null, GlodaDatastore._mapFolder(aFolderOrGlodaFolder).id];
       }}, this.NOUN_FOLDER);
     // TODO: use some form of (weak) caching layer... it is reasonably likely
     //  that there will be a high degree of correlation in many cases, and
     //  unless the UI is extremely clever and does its cleverness before
     //  examining the data, we will probably hit the correlation.
     this.defineNoun({
       name: "conversation",
       class: GlodaConversation,
-      firstClass: false,
+      allowsArbitraryAttrs: false,
       cache: true, cacheCost: 512,
       tableName: "conversations",
       attrTableName: "messageAttributes", attrIDColumnName: "conversationID",
       datastore: GlodaDatastore,
       objFromRow: GlodaDatastore._conversationFromRow,
       fromParamAndValue: function(aParam, aID) {
         return GlodaDatastore.getConversationByID(aID);
       },
@@ -788,52 +845,56 @@ var Gloda = {
         if (aConversation instanceof GlodaConversation)
           return [null, aConversation.id];
         else // assume they're just passing the id directly
           return [null, aConversation];
       }}, this.NOUN_CONVERSATION);
     this.defineNoun({
       name: "message",
       class: GlodaMessage,
-      firstClass: true,
+      allowsArbitraryAttrs: true,
       cache: true, cacheCost: 2048,
       tableName: "messages",
       attrTableName: "messageAttributes", attrIDColumnName: "messageID",
       datastore: GlodaDatastore, objFromRow: GlodaDatastore._messageFromRow,
+      dbAttribAdjuster: GlodaDatastore.adjustMessageAttributes,
+      objInsert: GlodaDatastore.insertMessage,
+      objUpdate: GlodaDatastore.updateMessage,
       fromParamAndValue: function(aParam, aID) {
         return GlodaDatastore.getMessageByID(aID);
       },
       toParamAndValue: function(aMessage) {
         if (aMessage instanceof GlodaMessage)
           return [null, aMessage.id];
         else // assume they're just passing the id directly
           return [null, aMessage];
       }}, this.NOUN_MESSAGE);
     this.defineNoun({
       name: "contact",
       class: GlodaContact,
-      firstClass: true,
+      allowsArbitraryAttrs: true,
       cache: true, cacheCost: 128,
       tableName: "contacts",
       attrTableName: "contactAttributes", attrIDColumnName: "contactID",
       datastore: GlodaDatastore, objFromRow: GlodaDatastore._contactFromRow,
+      objInsert: GlodaDatastore.insertContact,
       objUpdate: GlodaDatastore.updateContact,
       fromParamAndValue: function(aParam, aID) {
         return GlodaDatastore.getContactByID(aID);
       },
       toParamAndValue: function(aContact) {
         if (aContact instanceof GlodaContact)
           return [null, aContact.id];
         else // assume they're just passing the id directly
           return [null, aContact];
       }}, this.NOUN_CONTACT);
     this.defineNoun({
       name: "identity",
       class: GlodaIdentity,
-      firstClass: false,
+      allowsArbitraryAttrs: false,
       cache: true, cacheCost: 128,
       usesUniqueValue: true,
       tableName: "identities",
       datastore: GlodaDatastore, objFromRow: GlodaDatastore._identityFromRow,
       fromParamAndValue: function(aParam, aID) {
         return GlodaDatastore.getIdentityByID(aID);
       },
       toParamAndValue: function(aIdentity) {
@@ -845,45 +906,63 @@ var Gloda = {
 
     // parameterized identity is just two identities; we store the first one
     //  (whose value set must be very constrainted, like the 'me' identities)
     //  as the parameter, the second (which does not need to be constrained)
     //  as the value.
     this.defineNoun({
       name: "parameterized-identity",
       class: null,
-      firstClass: false,
+      allowsArbitraryAttrs: false,
+      computeDelta: function(aCurValues, aOldValues) {
+        let oldMap = {};
+        for each (let [, tupe] in Iterator(aOldValues)) {
+          let [originIdentity, 
+        }
+      },
+      contributeObjDependencies: function(aJsonValues, aReferencesByNounID) {
+        // nothing to do with a zero-length list
+        if (aJsonValues.length == 0)
+          return false;
+      
+        let references = aReferencesByNounID[this.NOUN_IDENTITY];
+        if (references === undefined)
+          references = aReferencesByNounID[this.NOUN_IDENTITY] = {};
+        
+        for each (let [, tupe] in Iterator(aJsonValues)) {
+          let [originIdentityID, targetIdentityID] = tupe;
+          references[originIdentityID] = null;
+          references[targetIdentityID] = null;
+        }
+        
+        return true;
+      },
+      resolveObjDependencies: function(aJsonValues, aReferencesByNounID) {
+        let references = aReferencesByNounID[this.NOUN_IDENTITY];
+        if (references === undefined)
+          references = aReferencesByNounID[this.NOUN_IDENTITY] = {};
+        
+        let results = [];
+        for each (let [, tupe] in Iterator(aJsonValues)) {
+          let [originIdentityID, targetIdentityID] = tupe;
+          results.push([references[originIdentityID],
+                        references[targetIdentityID]]);
+        }
+        
+        return results;
+      },
       fromParamAndValue: function(aParamIdentityID, aValueIdentityID) {
         return [GlodaDatastore.getIdentityByID(aParamIdentityID),
                 GlodaDatastore.getIdentityByID(aValueIdentityID)];
       },
       toParamAndValue: function(aIdentityTuple) {
-        if (typeof aIdentityTuple == "number")
-          return aIdentityTuple;
         return [aIdentityTuple[0].id, aIdentityTuple[1].id];
       }}, this.NOUN_PARAM_IDENTITY);
 
     GlodaDatastore.getAllAttributes();
-
-    /* boolean actions, these are parameterized by the attribute they operate
-       in the context of.  They are also (not coincidentally), ugly. */
-    Gloda.defineNounAction(Gloda.NOUN_BOOLEAN, {actionType: "filter",
-      actionTarget: Gloda.NOUN_MESSAGE,
-      shortName: "true",
-      makeConstraint: function(aAttrDef, aIdentity) {
-        return [aAttrDef, null, 1];
-      },
-      });
-    Gloda.defineNounAction(Gloda.NOUN_BOOLEAN, {actionType: "filter",
-      actionTarget: Gloda.NOUN_MESSAGE,
-      shortName: "false",
-      makeConstraint: function(aAttrDef, aIdentity) {
-        return [aAttrDef, null, 0];
-      },
-      });
   },
 
   /**
    * Create accessor functions to 'bind' an attribute to underlying normalized
    *  attribute storage, as well as creating the appropriate query object
    *  constraint helper functions.  This name is somewhat of a misnomer because
    *  special attributes are not 'bound' (because specific/non-generic per-class
    *  code provides the properties) but still depend on this method to
@@ -891,67 +970,21 @@ var Gloda = {
    *
    * @XXX potentially rename to not suggest binding is required.
    */
   _bindAttribute: function gloda_ns_bindAttr(aAttr, aSubjectType, aObjectType,
                                              aSingular, aDoBind, aBindName) {
     if (!(aSubjectType in this._nounIDToMeta))
       throw Error("Invalid subject type: " + aSubjectType);
 
-    let nounMeta = this._nounIDToMeta[aObjectType];
+    let objNounMeta = this._nounIDToMeta[aObjectType];
     let subjectNounMeta = this._nounIDToMeta[aSubjectType];
 
     // -- the on-object bindings
     if (aDoBind) {
-      let storageName = "__" + aBindName;
-      let getter;
-      // should we memoize the value as a getter per-instance?
-      if (aSingular) {
-        getter = function() {
-          let val = this[storageName];
-          if (val !== undefined)
-            return val;
-          let instances = this.getAttributeInstances(aAttr);
-          if (instances.length > 0)
-            val = nounMeta.fromParamAndValue(instances[0][1], instances[0][2]);
-          else
-            val = null;
-          //this[storageName] = val;
-          this.__defineGetter__(aBindName, function() val);
-          return val;
-        }
-      } else {
-        getter = function() {
-          let values = this[storageName];
-          if (values !== undefined)
-            return values;
-          let instances = this.getAttributeInstances(aAttr);
-          if (instances.length > 0) {
-            values = [];
-            for (let iInst = 0; iInst < instances.length; iInst++) {
-              values.push(nounMeta.fromParamAndValue(instances[iInst][1],
-                                                     instances[iInst][2]));
-            }
-          }
-          else {
-            values = instances; // empty is empty
-          }
-          //this[storageName] = values;
-          this.__defineGetter__(aBindName, function() values);
-          return values;
-        }
-      }
-
-      let subjectProto = subjectNounMeta.class.prototype;
-      subjectProto.__defineGetter__(aBindName, getter);
-      // no setters for now; manipulation comes later, and will require the attr
-      //  definer to provide the actual logic, since we need to affect reality,
-      //  not just the data-store.  we may also just punt that all off onto
-      //  STEEL...
-
       aAttr.boundName = aBindName;
     }
 
     // -- the query constraint helpers
     if (subjectNounMeta.queryClass !== undefined) {
       let constrainer = function() {
         // all the arguments provided end up being ORed together
         let our_ors = [];
@@ -1106,34 +1139,33 @@ var Gloda = {
 
         // update the provider maps...
         if (this._attrProviderOrderByNoun[subjectType]
                 .indexOf(aAttrDef.provider) == -1) {
           this._attrProviderOrderByNoun[subjectType].push(aAttrDef.provider);
           this._attrProvidersByNoun[subjectType][aAttrDef.provider] = [];
         }
         this._attrProvidersByNoun[subjectType][aAttrDef.provider].push(aAttrDef);
+        
+        let subjectNounDef = this._nounIDToMeta[subjectType];
+        subjectNounDef.attribsByBoundName[bindName] = attr;
+        
       }
 
       this._attrProviders[aAttrDef.provider.providerName].push(attr);
       return attr;
     }
 
     let objectNounMeta = this._nounIDToMeta[aAttrDef.objectNoun];
 
-    // Being here means the attribute def does not exist in the database.
-    // Of course, we only want to create something in the database if the
-    //  parameter is forever un-used (noun does not 'usesParameter')
     let attrID = null;
-    if (!objectNounMeta.usesParameter) {
-      attrID = GlodaDatastore._createAttributeDef(aAttrDef.attributeType,
-                                                  aAttrDef.extensionName,
-                                                  aAttrDef.attributeName,
-                                                  null);
-    }
+    attrID = GlodaDatastore._createAttributeDef(aAttrDef.attributeType,
+                                                aAttrDef.extensionName,
+                                                aAttrDef.attributeName,
+                                                null);
 
     attr = new GlodaAttributeDef(GlodaDatastore, attrID, compoundName,
                                  aAttrDef.provider, aAttrDef.attributeType,
                                  aAttrDef.extensionName, aAttrDef.attributeName,
                                  aAttrDef.subjectNouns, aAttrDef.objectNoun);
     // things here match the HATHATHAT clause above.  clearly, this should also
     //  be resolved more satisfactorily.
     attr._special = aAttrDef.special || this.kSpecialNotAtAll;
@@ -1144,18 +1176,17 @@ var Gloda = {
     for (let iSubject = 0; iSubject < aAttrDef.subjectNouns.length;
          iSubject++) {
       let subjectType = aAttrDef.subjectNouns[iSubject];
       this._bindAttribute(attr, subjectType, aAttrDef.objectNoun,
                           aAttrDef.singular, aAttrDef.bind, bindName);
     }
 
     this._attrProviders[aAttrDef.provider.providerName].push(attr);
-    if (!objectNounMeta.usesParameter)
-      GlodaDatastore._attributeIDToDef[attrID] = [attr, null];
+    GlodaDatastore._attributeIDToDef[attrID] = [attr, null];
     return attr;
   },
 
   /**
    * Retrieve the attribute provided by the given extension with the given
    *  attribute name.  The original idea was that plugins would effectively
    *  name-space attributes, helping avoid collisions.  Since we are leaning
    *  towards using binding heavily, this doesn't really help, as the collisions
@@ -1337,16 +1368,170 @@ var Gloda = {
     GlodaDatastore.insertMessageAttributes(aMessage, outAttribs);
     aMessage._replaceAttributes(memAttribs);
 
     if (aIsNew)
       GlodaCollectionManager.itemsAdded(aMessage.NOUN_ID, [aMessage]);
     else
       GlodaCollectionManager.itemsModified(aMessage.NOUN_ID, [aMessage]);
   },
+  
+  /**
+   * Populate a gloda representation of an item given the thus-far built
+   *  representation, the previous representation, and one or more raw
+   *  representations.
+   *
+   * The result of the processing ends up with attributes in 3 different forms:
+   * - Database attribute rows (to be added and removed).
+   * - In-memory representation.
+   * - JSON-able representation.
+   */
+  grokNounItem: function gloda_ns_grokNounItem(aItem, aRawReps, aIsNew,
+      aCallbackHandle) {
+    let itemNounDef = this._nounIDToMeta[aItem.NOUN_ID];
+    let attribsByBoundName = itemNounDef.attribsByBoundName;
+    
+    let addDBAttribs = [];
+    let removeDBAttribs = [];
+    
+    let jsonDict = {};
+    
+    let aOldItem = aItem;
+    // we want to create a clone of the existing item so that we can know the
+    //  deltas that happened for indexing purposes
+    aItem = aItem._clone();
+  
+    // Have the attribute providers directly set properties on the aItem
+    let attrProviders = this._attrProviderOrderByNoun[aItem.NOUN_ID];
+    for (let iProvider = 0; iProvider < attrProviders.length; iProvider++) {
+      yield aCallbackHandle.pushAndGo(
+        attrProviders[iProvider].process(aItem, aRawItem, aIsNew,
+                                         aCallbackHandle));
+    }
+  
+    // Iterate over the attributes on the item
+    for each (let [key, value] in Iterator(aItem)) {
+      // ignore keys that start with underscores, they are private and not
+      //  persisted by our attribute mechanism.  (they are directly handled by
+      //  the object implementation.)
+      if (key[0] == "_")
+        continue;
+      // find the attribute definition that corresponds to this key
+      let attrib = attribsByBoundName[key];
+      // if there's no attribute, that's not good, but not horrible.
+      if (attrib === undefined)
+        continue;
+      
+      let objNounDef = attrib.objectNounDef;
+      
+      // - translate for our JSON rep
+      if (attrib.singular) {
+        if (obnNounDef.toJSON)
+          jsonDict[attrib.id] = objNounDef.toJSON(value);
+        else
+          jsonDict[attrib.id] = value; 
+      }
+      else {
+        if (objNounDef.toJSON) {
+          toJSON = objNounDef.toJSON;
+          jsonDict[attrib.id] = [toJSON(subValue) for each
+                           ([, subValue] in Iterator(value))] ;
+        }
+        else
+          jsonDict[attrib.id] = value;
+      }
+      
+      // perform a delta analysis against the old value, if we have one
+      let oldValue = aOldItem[key];
+      if (oldValue !== undefined) {
+        // in the singular case if they don't match, it's one add and one remove
+        if (attrib.singular) {
+          if (value != oldValue) {
+            addDBAttribs.push(attrib.convertValuesToDBAttributes(value)[0]);
+            removeDBAttribs.push(
+              attrib.convertValuesToDBAttributes(oldValue)[0]);
+          }
+        }
+        // in the plural case, we have to figure the deltas accounting for
+        //  possible changes in ordering (which is insignificant from an
+        //  indexing perspective)
+        // some nouns may not meet === equivalence needs, so must provide a
+        //  custom computeDelta method to help us out
+        else if (objNounDef.computeDelta) {
+          let [valuesAdded, valuesRemoved] = 
+            objNounDef.computeDelta(value, oldValue);
+          // convert the values to database-style attribute rows
+          addDBAttribs.push.apply(addDBAttribs,
+            attrib.convertValuesToDBAttributes(valuesAdded));
+          removeDBAttribs.push.apply(removeDBAttribs,
+            attrib.convertValuesToDBAttributes(valuesRemoved));
+        }
+        else {
+          // build a map of the previous values; we will delete the values as
+          //  we see them so that we will know what old values are no longer
+          //  present in the current set of values.
+          let oldValueMap = {};
+          for each (let [iAnOldValue, anOldValue] in Iterator(oldValue)) {
+            oldValueMap[anOldValue] = true;
+          }
+          // traverse the current values...
+          let valuesAdded = [];
+          for each (let [iCurValue, curValue] in Iterator(value)) {
+            if (curValue in oldValueMap)
+              delete oldValueMap[curValue];
+            else
+              valuesAdded.push(curValue);
+          }
+          // anything still on oldValueMap was removed.
+          let valuesRemoved = [val for val in Iterator(oldValueMap, true)];
+          // convert the values to database-style attribute rows
+          addDBAttribs.push.apply(addDBAttribs,
+            attrib.convertValuesToDBAttributes(valuesAdded));
+          removeDBAttribs.push.apply(removeDBAttribs,
+            attrib.convertValuesToDBAttributes(valuesRemoved));
+        }
+      
+        // delete the old values to mark that we have processed them
+        delete aOldItem[key];
+      }
+      // no old value, all attributes are new
+      else {
+        addDBAttribs.push.apply(addDBAttribs,
+                                attrib.convertValuesToDBAttributes(value));
+      }
+    }
+    
+    // Iterate over any remaining values in old items for purge purposes.
+    for each (let [key, value] in Iterator(aOldItem)) {
+      // ignore keys that start with underscores, they are private and not
+      //  persisted by our attribute mechanism.  (they are directly handled by
+      //  the object implementation.)
+      if (key[0] == "_")
+        continue;
+      // find the attribute definition that corresponds to this key
+      let attrib = attribsByBoundName[key];
+      // if there's no attribute, that's not good, but not horrible.
+      if (attrib === undefined)
+        continue;
+      
+      removeDBAttribs.push.apply(removeDBAttribs,
+                                 attrib.convertValuesToDBAttributes(value));
+    }
+    
+    aItem._jsonText = this._json.encode(jsonDict);
+    
+    if (aIsNew) {
+      itemNounDef.objInsert.call(itemNounDef.datastore, aItem);
+    }
+    else {
+      itemNounDef.objUpdate.call(itemNounDef.datastore, aItem);
+    }
+    
+    yield this.kWorkDone;
+  },
 
   _processNounItem: function gloda_ns_processNounItem(aItem, aRawItem, aIsNew) {
     // For now, we are ridiculously lazy and simply nuke all existing attributes
     //  before applying the new attributes.
     aItem._datastore.clearAttributes(aItem);
 
     let allAttribs = [];
 
@@ -1398,24 +1583,16 @@ var Gloda = {
     GlodaDatastore.insertAttributes(aItem, outAttribs);
     aItem._replaceAttributes(memAttribs);
 
     if (aIsNew)
       GlodaCollectionManager.itemsAdded(aItem.NOUN_ID, [aItem]);
     else
       GlodaCollectionManager.itemsModified(aItem.NOUN_ID, [aItem]);
   },
-
-  /**
-   * Deprecated mechanism for querying for messages.  Use newQuery now,
-   *  specifying the message noun id.  Still works for now, but not for long.
-   */
-  queryMessagesAPV: function gloda_ns_queryMessagesAPV(aAPVs) {
-    return GlodaDatastore.queryMessagesAPV(aAPVs);
-  }
 };
 
 /* and initialize the Gloda object/NS before we return... */
 try {
   Gloda._init();
 }
 catch (ex) {
   Gloda._log.debug("Exception during Gloda init (" + ex.fileName + ":" +
--- a/modules/index_ab.js
+++ b/modules/index_ab.js
@@ -217,41 +217,38 @@ var GlodaABAttrs = {
     // we need to find any existing bound freetag attributes, and use them to
     //  populate to FreeTagNoun's understanding
     for (let freeTagName in this._attrFreeTag.parameterBindings) {
       this._log.debug("Telling FreeTagNoun about: " + freeTagName);
       FreeTagNoun.getFreeTag(freeTagName);
     }
   },
   
-  process: function(aContact, aCard) {
+  process: function(aContact, aCard, aIsNew, aCallbackHandle) {
     if (aContact.NOUN_ID != Gloda.NOUN_CONTACT) {
       this._log.warning("Somehow got a non-contact: " + aContact);
-      return [];
+      return Gloda.kWorkDone;
     }
+    
+    // update the name
+    if (aCard.displayName && aCard.displayName != aContact.name)
+      aContact.name = aCard.displayName;
   
-    this._log.debug("Processing a contact and card.");
-    let attribs = [];
+    aContact.freeTags = [];
     
     let tags = null;
     try {
-      tags = aCard.getProperty("tags", null);
+      tags = aCard.getProperty("Categories", null);
     } catch (ex) {
       this._log.error("Problem accessing property: " + ex);
     }
     if (tags) {
-      this._log.debug("Found tags: " + tags);
       for each (let [iTagName, tagName] in Iterator(tags.split(","))) {
         tagName = tagName.trim();
-        // return attrib, param, value; we know the param to use because we know
-        //  how FreeTagNoun works, but this is a candidate for refactoring.
         if (tagName) {
-          FreeTagNoun.getFreeTag(tagName); // cause the tag to be known
-          attribs.push([this._attrFreeTag, tagName, null]);
+          aContact.freeTags.push(FreeTagNoun.getFreeTag(tagName));
         }
       }
     }
     
-    this._log.debug("Returning attributes: " + attribs);
-    
-    return attribs;
+    yield Gloda.kWorkDone;
   }
 };
--- a/modules/indexer.js
+++ b/modules/indexer.js
@@ -126,16 +126,33 @@ function fixIterator(aEnum, aIface) {
     let iter = function () {
       while (aEnum.hasMoreElements())
         yield aEnum.getNext().QueryInterface(face);
     }
     return { __iterator__: iter };
   } catch(ex) {}
 }
 
+function MakeCleanMsgHdrCallback(aMsgHdr) {
+  return function() {
+    // Mark this message as indexed
+    aMsgHdr.setUint32Property(this.GLODA_MESSAGE_ID_PROPERTY, curMsg.id);
+    // If there is a gloda-dirty flag on there, clear it by writing a 0.  (But
+    //  don't do this if we didn't have a dirty flag on there in the first
+    //  case.)  It sounds like we would actually prefer to "cut" the "cell",
+    //  but I don't see any in-domain means of doing that.
+    try {
+      let isDirty = aMsgHdr.getUint32Property(this.GLODA_DIRTY_PROPERTY);
+      if (isDirty)
+        aMsgHdr.setUint32Property(this.GLODA_DIRTY_PROPERTY, 0);
+    }
+    catch (ex) {}
+  };
+}
+
 const MSG_FLAG_OFFLINE = 0x80;
 const MSG_FLAG_EXPUNGED = 0x08;
 
 /**
  * @class Capture the indexing batch concept explicitly.
  *
  * @param aJobType The type of thing we are indexing.  Current choices are:
  *   "folder" and "message".  Previous choices included "account".  The indexer
@@ -275,16 +292,18 @@ var GlodaIndexer = {
     
     this._inited = true;
     
     // initialize our listeners' this pointers
     this._databaseAnnouncerListener.indexer = this;
     this._msgFolderListener.indexer = this;
     this._shutdownTask.indexer = this;
     
+    this._callbackHandler.init();
+    
     // create the timer that drives our intermittent indexing
     this._timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
 
 
     // figure out if event-driven indexing should be enabled...
     let prefService = Cc["@mozilla.org/preferences-service;1"].
                         getService(Ci.nsIPrefService);
     let branch = prefService.getBranch("mailnews.database.global.indexer");
@@ -514,33 +533,44 @@ var GlodaIndexer = {
    *  it means the value is known reliably.  If this value is null, it means
    *  that we don't know, likely because we have started up and have not checked
    *  the database.
    */
   pendingDeletions: null,
   
   GLODA_MESSAGE_ID_PROPERTY: "gloda-id",
   GLODA_DIRTY_PROPERTY: "gloda-dirty",
-
-  /** Synchronous activities performed, you can drive us more. */
-  kWorkSync: 0,
   /**
-   * Asynchronous activity performed, you need to relinquish flow control and
-   *  trust us to call callbackDriver later.
+   * The message (or folder state) is believed up-to-date.
    */
-  kWorkAsync: 1,
+  kMessageClean: 0,
   /**
-   * We are all done with our task, close us and figure out something else to do.
+   * The message (or folder) is known to not be up-to-date. In the case of
+   *  folders, this means that some of the messages in the folder may be dirty.
+   *  However, because of the way our indexing works, it is possible there may
+   *  actually be no dirty messages in a folder.  (We attempt to process
+   *  messages in an event-driven fashion for a finite number of messages, but
+   *  because we can quit without completing processing of the queue, we need to
+   *  mark the folder dirty, just-in-case.)  (We could do some extra leg-work
+   *  and do a better job of marking the folder clean again.)   
    */
-  kWorkDone: 2,
+  kMessageDirty: 1,
   /**
-   * We are not done with our task, but we think it's a good idea to take a
-   *  breather.
+   * We have not indexed the folder at all, but messages in the folder think
+   *  they are indexed.  Once we mark all the messages in the folder as being
+   *  dirty so that they don't confuse us, we downgrade the folder's dirty
+   *  status to just kMessageDirty.
    */
-  kWorkPause: 3,
+  kMessageFilthy: 2,
+
+  kWorkSync: Gloda.kWorkSync,
+  kWorkAsync: Gloda.kWorkAsync,
+  kWorkDone: Gloda.kWorkDone,
+  kWorkPause: Gloda.kWorkPause,
+  kWorkDoneWithResult: Gloda.kWorkDoneWithResult,
   
   /**
    * Our current job number, out of _indexingJobGoal.  Although our jobs comes
    *  from _indexQueue, this is not an offset into that list because we forget
    *  jobs once we complete them.  As such, this value is strictly for progress
    *  tracking.
    */ 
   _indexingJobCount: 0,
@@ -724,22 +754,21 @@ var GlodaIndexer = {
                                                                aNeedIterator) {
     // if leave folder was't cleared first, remove the listener; everyone else
     //  will be nulled out in the exception handler below if things go south
     //  on this folder.
     if (this._indexingFolder !== null) {
       this._indexingDatabase.RemoveListener(this._databaseAnnouncerListener);
     }
     
-    let folderURI = GlodaDatastore._mapFolderID(aFolderID);
-    //this._log.debug("Active Folder URI: " + folderURI);
+    let glodaFolder = GlodaDatastore._mapFolderID(aFolderID);
   
     let rdfService = Cc['@mozilla.org/rdf/rdf-service;1'].
                      getService(Ci.nsIRDFService);
-    let folder = rdfService.GetResource(folderURI);
+    let folder = rdfService.GetResource(glodaFolder.uri);
     folder.QueryInterface(Ci.nsIMsgFolder); // (we want to explode in the try
     // if this guy wasn't what we wanted)
     this._indexingFolder = folder;
     this._indexingFolderID = aFolderID;
 
     try {
       // The msf may need to be created or otherwise updated for local folders.
       // This may require yielding until such time as the msf has been created.
@@ -762,19 +791,17 @@ var GlodaIndexer = {
         return this.kWorkAsync;
       }
       // we get an nsIMsgDatabase out of this (unsurprisingly) which
       //  explicitly inherits from nsIDBChangeAnnouncer, which has the
       //  AddListener call we want.
       if (this._indexingDatabase == null)
         this._indexingDatabase = folder.getMsgDatabase(null);
       if (aNeedIterator)
-        this._indexingIterator = fixIterator(
-                                   this._indexingDatabase.EnumerateMessages(),
-                                   Ci.nsIMsgDBHdr);
+        this._indexerGetIterator();
       this._indexingDatabase.AddListener(this._databaseAnnouncerListener);
     }
     catch (ex) {
       this._log.error("Problem entering folder: " +
                       folder.prettiestName + ", skipping.");
       this._log.error("Error was: " + ex);
       this._indexingFolder = null;
       this._indexingFolderID = null;
@@ -784,16 +811,22 @@ var GlodaIndexer = {
       // re-throw, we just wanted to make sure this junk is cleaned up and
       //  get localized error logging...
       throw ex;
     }
     
     return this.kWorkSync;
   },
   
+  _indexerGetIterator: function gloda_indexer_indexerGetIterator() {
+    this._indexingIterator = fixIterator(
+                               this._indexingDatabase.EnumerateMessages(),
+                               Ci.nsIMsgDBHdr);
+  },
+  
   _indexerLeaveFolder: function gloda_index_indexerLeaveFolder(aExpected) {
     if (this._indexingFolder !== null) {
       // remove our listener!
       this._indexingDatabase.RemoveListener(this._databaseAnnouncerListener);
       // null everyone out
       this._indexingFolder = null;
       this._indexingFolderID = null;
       this._indexingDatabase = null;
@@ -831,49 +864,71 @@ var GlodaIndexer = {
   },
 
   /**
    * The current processing 'batch' generator, produced by a call to workBatch()
    *  and used by callbackDriver to drive execution.
    */
   _batch: null,
   _inCallback: false,
+  _savedCallbackArgs: null,
   /**
    * The root work-driver.  callbackDriver creates workBatch generator instances
    *  (stored in _batch) which run until they are done (kWorkDone) or they
    *  (really the embedded _actualWorker) encounter something asynchronous.
    *  The convention is that all the callback handlers end up calling us,
    *  ensuring that control-flow properly resumes.  If the batch completes,
    *  we re-schedule ourselves after a time delay (controlled by _indexInterval)
    *  and return.  (We use one-shot timers because repeating-slack does not
    *  know enough to deal with our (current) asynchronous nature.)
    */
   callbackDriver: function gloda_index_callbackDriver() {
     // it is conceivable that someone we call will call something that in some
     //  cases might be asynchronous, and in other cases immediately generate
     //  events without returning.  In the interest of (stack-depth) sanity,
     //  let's handle this by performing a minimal time-delay callback.
+    // this is also now a good thing sequencing-wise.  if we get our callback
+    //  with data before the underlying function has yielded, we obviously can't
+    //  cram the data in yet.  Our options in this case are to either mark the
+    //  fact that the callback has already happened and immediately return to
+    //  the iterator when it does bubble up the kWorkAsync, or we can do as we
+    //  have been doing, but save the 
     if (this._inCallback) {
+      this._savedCallbackArgs = arguments;
       this._timer.initWithCallback(this._wrapCallbackDriver,
                                    0,
                                    Ci.nsITimer.TYPE_ONE_SHOT);
       return;
     }
     this._inCallback = true;
 
     try {
       if (this._batch === null)
         this._batch = this.workBatch();
       
       // kWorkAsync, kWorkDone, kWorkPause are allowed out; kWorkSync is not
       // On kWorkDone, we want to schedule another timer to fire on us if we are
       //  not done indexing.  (On kWorkAsync, we don't care what happens, because
       //  someone else will be receiving the callback, and they will call us when
       //  they are done doing their thing.
-      switch (this._batch.next()) {
+      let result;
+      let args;
+      if (this._savedCallbackArgs != null) {
+        args = this._savedCallbackArgs;
+        this._savedCallbackArgs = null;
+      }
+      else
+        args = arguments;
+      if (args.length == 0)
+        result = this._batch.next();
+      else if (args.length == 1)
+        result = this._batch.send(args[0]);
+      else // arguments works with destructuring assignment
+        result = this._batch.send(args);
+      switch (result) {
         // job's done, close the batch and re-schedule ourselves if there's more
         //  to do.
         case this.kWorkDone:
           this._batch.close();
           this._batch = null;
           // (intentional fall-through to re-scheduling logic) 
         // the batch wants to get re-scheduled, do so.
         case this.kWorkPause:
@@ -890,74 +945,121 @@ var GlodaIndexer = {
           break;
       }
     }
     finally {    
       this._inCallback = false;
     }
   },
 
-  /**
-   * The generator we are using to perform processing of the current job
-   *  (this._curIndexingJob).  It differs from this._batch which is a generator
-   *  that takes care of the batching logistics (namely managing database
-   *  transactions and keeping track of how much work can be done with the
-   *  current allocation of processing "tokens".
-   * The generator is created by _hireJobWorker from one of the _worker_*
-   *  generator methods.
-   */
-  _actualWorker: null,
+  _callbackHandle: {
+    _init: function gloda_index_callbackhandle_init() {
+      this.wrappedCallback = GlodaIndexer._wrapCallbackDriver;
+      this.callback = GlodaIndexer.callbackDriver;
+      
+    },
+    activeStack: [],
+    activeIterator: null,
+    push: function gloda_index_callbackhandle_push(aIterator) {
+      this.activeStack.push(aIterator);
+      this.activeIterator = aIterator;
+    },
+    pushAndGo: function gloda_index_callbackhandle_pushAndGo(aIterator) {
+      this.push(aIterator);
+      return this.activeIterator.next();
+    },
+    pop: function gloda_index_callbackhandle_pop() {
+      this.activeIterator.close();
+      this.activeStack.pop();
+      if (this.activeStack.length)
+        this.activeIterator = this.activeStack[this.activeStack.length - 1];
+      else
+        this.activeIterator = null;
+    },
+    /**
+     * Someone propagated an exception and we need to clean-up all the active
+     *  logic as best we can.  Which is not really all that well.
+     */
+    cleanup: function gloda_index_callbackhandle_cleanup() {
+      while (this.activeIterator !== null) {
+        this.pop();
+      }
+    },
+    popWithResult: function gloda_index_callbackhandle_popWithResult() {
+      this.pop();
+      let reslt = this._result;
+      this._result = null;
+      return result;
+    },
+    _result: null,
+    doneWithResult: function gloda_index_callbackhandle_doneWithResult(aResult){
+      this._result = aResult;
+      yield Gloda.kWorkDoneWithResult;
+    },
+    
+    /* be able to serve as a collection listener, resuming the active iterator's
+       last yield kWorkAsync */
+    _onItemsAdded: function() {},
+    _onItemsModified: function() {},
+    _onItemsRemoved: function() {},
+    _onQueryCompleted: function(aCollection) {
+      GlodaIndexer.callbackDriver();
+    }
+  },
   /**
    * The workBatch generator handles a single 'batch' of processing, managing
    *  the database transaction and keeping track of "tokens".  It drives the
    *  _actualWorker generator which is doing the work.
    * workBatch will only produce kWorkAsync and kWorkDone notifications.
    *  If _actualWorker returns kWorkSync and there are still tokens available,
    *  workBatch will keep driving _actualWorker until it encounters a
    *  kWorkAsync (which workBatch will yield to callbackDriver), or it runs
    *  out of tokens and yields a kWorkDone. 
    */
   workBatch: function gloda_index_workBatch() {
     let commitTokens = this._indexCommitTokens;
     GlodaDatastore._beginTransaction();
 
+    let data = undefined;
     while (commitTokens > 0) {
       for (let tokensLeft = this._indexTokens; tokensLeft > 0;
           tokensLeft--, commitTokens--) {
-        if ((this._actualWorker === null) && !this._hireJobWorker()) {
+        if ((this._callbackHandle.activeIterator === null) &&
+            !this._hireJobWorker()) {
           commitTokens = 0;
           break;
         }
       
         // XXX for performance, we may want to move the try outside the for loop
         //  with a quasi-redundant outer loop that shunts control back inside
         //  if we left the loop due to an exception (without consuming all the
         //  tokens.)
         try {
-          switch (this._actualWorker.next()) {
+          switch (this._callbackHandler.activeIterator.send(data)) {
             case this.kWorkSync:
               break;
             case this.kWorkAsync:
-              yield this.kWorkAsync;
+              data = yield this.kWorkAsync;
               break;
             case this.kWorkDone:
-              this._actualWorker.close();
-              this._actualWorker = null;
+              this._callbackHandler.pop();
+              tokensLeft++; // don't eat a token for this pass
               break;
+            case this.kWorkDoneWithResult:
+              data = this._callbackHandler.popWithResult();
+              tokensLeft++; // don't eat a token for this pass
+              continue;
           }
         }
         catch (ex) {
           this._log.debug("Bailing on job (at " + ex.fileName + ":" +
               ex.lineNumber + ") because: " + ex);
           this._indexerLeaveFolder(true);
           this._curIndexingJob = null;
-          if (this._actualWorker !== null) {
-            this._actualWorker.close();
-            this._actualWorker = null;
-          }
+          this._callbackHandle.cleanup();
         }
       }
       
       // take a breather by having the caller re-schedule us sometime in the
       //  future, but only if we're going to perform another loop iteration.
       if (commitTokens > 0)
         yield this.kWorkPause;
     }
@@ -997,43 +1099,48 @@ var GlodaIndexer = {
     //                this._indexQueue.length);
     let job = this._curIndexingJob = this._indexQueue.shift();
     this._indexingJobCount++;
     //this._log.debug("++ Pulled job: " + job.jobType + ", " +
     //                job.deltaType + ", " + job.id);
     let generator = null;
     
     if (job.jobType == "sweep") {
-      this._actualWorker = this._worker_indexingSweep(job);
+      generator = this._worker_indexingSweep(job);
     }
     else if (job.jobType == "folder") {
-      this._actualWorker = this._worker_folderIndex(job);
+      generator = this._worker_folderIndex(job);
     }
     else if(job.jobType == "message") {
       if (job === this._pendingAddJob)
         this._pendingAddJob = null;
       // update our goal from the items length
       job.goal = job.items.length;
                   
-      this._actualWorker = this._worker_messageIndex(job);
+      generator = this._worker_messageIndex(job);
     }
     else if (job.jobType == "delete") {
       // we'll count the block processing as a cost of 1...
       job.goal = 1;
-      this._actualWorker = this._worker_processDeletes(job);
+      generator = this._worker_processDeletes(job);
     }
     else if (job.jobType in this._otherIndexerWorkers) {
       let [indexer, workerFunc] = this._otherIndexerWorkers[job.jobType];
-      this._actualWorker = workerFunc.call(indexer, job);
+      generator = workerFunc.call(indexer, job);
     }
     else {
       this._log.warning("Unknown job type: " + job.jobType);
     }
     
-    return true;
+    if (generator) {
+      this._callbackHandle.push(generator);
+      return true;
+    }
+    else
+      return false;
   },
 
   /**
    * Performs the folder sweep, locating folders that should be indexed, and
    *  creating a folder indexing job for them, and rescheduling itself for
    *  execution after that job is completed.  Once it indexes all the folders,
    *  if we believe we have deletions to process (or just don't know), it kicks
    *  off a deletion processing job.
@@ -1072,39 +1179,32 @@ var GlodaIndexer = {
       let numFolders = allFolders.Count();
       for (let folderIndex = 0; folderIndex < numFolders; folderIndex++)
       {
         let folder = allFolders.GetElementAt(folderIndex).QueryInterface(
                                                             Ci.nsIMsgFolder);
         // we could also check nsMsgFolderFlags.Mail conceivably...
         let isLocal = folder instanceof Ci.nsIMsgLocalMailFolder;
         // we only index local folders or IMAP folders that are marked offline.
-        if (!isLocal && !(folder.flags&Ci.nsMsgFolderFlags.Offline))
+        if (!isLocal && !(folder.flags & Ci.nsMsgFolderFlags.Offline))
           continue;
 
         // if no folder was indexed (or the pref's not set), just use the first folder
         if (!aJob.lastFolderIndexedUri || useNextFolder)
         {
           // make sure the folder is dirty before accepting this job...
-          let isDirty = true;
-          try {
-            isDirty = folder.GetStringProperty(this.GLODA_DIRTY_PROPERTY) !=
-                        "0"; 
-          }
-          catch (ex) {}
-          
-          if (!isDirty) {
-            continue; 
-          }
+          let glodaFolder = GlodaDatastore._mapFolder(folder);
+          if (!glodaFolder.dirtyStatus)
+            continue;
         
           aJob.lastFolderIndexedUri = folder.URI;
           this._indexingJobGoal += 2;
           // add a job for the folder indexing
           this._indexQueue.push(new IndexingJob("folder", 0,
-              this._datastore._mapFolderURI(aJob.lastFolderIndexedUri)));
+              this._datastore._mapFolder(folder).id));
           // re-schedule this job (although this worker will die)
           this._indexQueue.push(aJob);
           yield this.kWorkDone;
         }
         else
         {
           if (aJob.lastFolderIndexedUri == folder.URI)
             useNextFolder = true;
@@ -1151,45 +1251,80 @@ var GlodaIndexer = {
     
     // there is of course a cost to all this header investigation even if we
     //  don't do something.  so we will yield with kWorkSync for every block. 
     const HEADER_CHECK_BLOCK_SIZE = 100;
     
     let isLocal = this._indexingFolder instanceof Ci.nsIMsgLocalMailFolder;
     // we can safely presume if we are here that this folder has been selected
     //  for offline processing...
+
+    // Handle the filthy case.  A filthy folder may have misleading properties
+    //  on the message that claim the message is indexed.  They are misleading
+    //  because the database, for whatever reason, does not have the messages
+    //  (accurately) indexed.
+    // We need to walk all the messages and mark them filthy if they have a
+    //  dirty property.  Once we have done this, we can downgrade the folder's
+    //  dirty status to plain dirty.  We do this rather than trying to process
+    //  everyone in one go in a filthy context because if we have to terminate
+    //  indexing before we quit, we don't want to have to re-index messages next
+    //  time.  (This could even lead to never completing indexing in a
+    //  pathological situation.)
+    let glodaFolder = GlodaDatastore._mapFolder(this._indexingFolder);
+    if (glodaFolder.dirtyStatus == glodaFolder.kFolderFilthy) {
+      let count = 0;
+      for (let msgHdr in this._indexingIterator) {
+        // we still need to avoid locking up the UI, pause periodically...
+        if (++count % HEADER_CHECK_BLOCK_SIZE == 0)
+          yield this.kWorkSync;
+        
+        let glodaMessageId = msgHdr.getUint32Property(
+                             this.GLODA_MESSAGE_ID_PROPERTY);
+        // if it has a gloda message id, we need to mark it filthy
+        if (glodaMessageId != 0)
+          msgHdr.setUint32Property(this.GLODA_DIRTY_PROPERTY,
+                                   this.kMessageFilthy);
+        // if it doesn't have a gloda message id, we will definitely index it,
+        //  so no action is required.
+      }
+      // this will automatically persist to the database
+      glodaFolder.dirtyStatus = glodaFolder.kFolderDirty;
+      
+      // We used up the iterator, get a new one.
+      this._indexerGetIterator();
+    }
     
     for (let msgHdr in this._indexingIterator) {
       // per above, we want to periodically release control while doing all
       //  this header traversal/investigation.
       if (++aJob.offset % HEADER_CHECK_BLOCK_SIZE == 0)
         yield this.kWorkSync;
       
-      if ((isLocal || msgHdr.flags&MSG_FLAG_OFFLINE) &&
-          !(msgHdr.flags&MSG_FLAG_EXPUNGED)) {
+      if ((isLocal || (msgHdr.flags & MSG_FLAG_OFFLINE)) &&
+          !(msgHdr.flags & MSG_FLAG_EXPUNGED)) {
         // this returns 0 when missing
         let glodaMessageId = msgHdr.getUint32Property(
                              this.GLODA_MESSAGE_ID_PROPERTY);
         
         // if it has a gloda message id, it has been indexed, but it still
         //  could be dirty.
         if (glodaMessageId != 0) {
           // (returns 0 when missing)
           let isDirty = msgHdr.getUint32Property(this.GLODA_DIRTY_PROPERTY)!= 0;
 
           // it's up to date if it's not dirty 
           if (!isDirty)
             continue;
         }
         
-        yield this._indexMessage(msgHdr);
+        yield this._callbackHandle.pushAndGo(this._indexMessage(msgHdr));
       }
     }
     
-    this._indexingFolder.setStringProperty(this.GLODA_DIRTY_PROPERTY, "0");
+    glodaFolder.dirtyStatus = glodaFolder.kFolderClean;
     
     // by definition, it's not likely we'll visit this folder again anytime soon
     this._indexerLeaveFolder();
     
     yield this.kWorkDone;
   },
   
   /**
@@ -1276,17 +1411,17 @@ var GlodaIndexer = {
   indexAccount: function glodaIndexAccount(aAccount) {
     let rootFolder = aAccount.incomingServer.rootFolder;
     if (rootFolder instanceof Ci.nsIMsgFolder) {
       this._log.info("Queueing account folders for indexing: " + aAccount.key);
 
       GlodaDatastore._beginTransaction();
       let folderJobs =
               [new IndexingJob("folder", 1,
-                              GlodaDatastore._mapFolderURI(folder.URI)) for each
+                               GlodaDatastore._mapFolder(folder).id) for each
               (folder in fixIterator(rootFolder.subFolders, Ci.nsIMsgFolder))];
       GlodaDatastore._commitTransaction();
       
       this._indexingJobGoal += folderJobs.length;
       this._indexQueue = this._indexQueue.concat(folderJobs);
       this.indexing = true;
     }
     else {
@@ -1296,43 +1431,29 @@ var GlodaIndexer = {
 
   /**
    * Queue a single folder for indexing given an nsIMsgFolder.
    */
   indexFolder: function glodaIndexFolder(aFolder) {
     this._log.info("Queue-ing folder for indexing: " + aFolder.prettiestName);
     
     this._indexQueue.push(new IndexingJob("folder", 1,
-                          GlodaDatastore._mapFolderURI(aFolder.URI)));
+                          GlodaDatastore._mapFolder(aFolder).id));
     this._indexingJobGoal++;
     this.indexing = true;
   },
-
-  /**
-   * Queue a single folder for indexing given its URI.
-   */
-  indexFolderByURI: function gloda_index_indexFolderByURI(aURI) {
-    if (aURI !== null) {
-      this._log.info("Queue-ing folder URI for indexing: " + aURI);
-      
-      this._indexQueue.push(new IndexingJob("folder", 1,
-                            GlodaDatastore._mapFolderURI(aURI)));
-      this._indexingJobGoal++;
-      this.indexing = true;
-    }
-  },
   
   /**
    * Queue a list of messages for indexing.
    *
    * @param aFoldersAndMessages List of [nsIMsgFolder, message key] tuples.
    */
   indexMessages: function gloda_index_indexMessages(aFoldersAndMessages) {
     let job = new IndexingJob("message", 1, null);
-    job.items = [[GlodaDatastore._mapFolderURI(fm[0].URI), fm[1]] for each
+    job.items = [[GlodaDatastore._mapFolder(fm[0]).id, fm[1]] for each
                  ([i, fm] in Iterator(aFoldersAndMessages))];
     this._indexQueue.push(job);
     this._indexingJobGoal++;
     this.indexing = true;
   },
   
   /* *********** Event Processing *********** */
   observe: function gloda_indexer_observe(aSubject, aTopic, aData) {
@@ -1421,17 +1542,17 @@ var GlodaIndexer = {
         this.indexer._pendingAddJob = new IndexingJob("message", 1, null);
         this.indexer._indexQueue.push(this.indexer._pendingAddJob);
         this.indexer._indexingJobGoal++;
       }
       // only queue the message if we haven't overflowed our event-driven budget
       if (this.indexer._pendingAddJob.items.length <
           this.indexer._indexMaxEventQueueMessages) {
         this.indexer._pendingAddJob.items.push(
-          [GlodaDatastore._mapFolderURI(aMsgHdr.folder.URI),
+          [GlodaDatastore._mapFolder(aMsgHdr.folder).id,
            aMsgHdr.messageKey]);
         this.indexer.indexing = true;
         this.indexer._log.debug("msgAdded notification, event indexing");
       }
       else {
         this.indexer.indexingSweepNeeded = true;
         this.indexer._log.debug("msgAdded notification, sweep indexing");
       }
@@ -1532,17 +1653,17 @@ var GlodaIndexer = {
                 //  required.
                 catch (ex) {}
               }
             }
             
             // this method takes care to update the in-memory representations
             //  too; we don't need to do anything
             this.indexer._datastore.updateMessageLocations(glodaIds,
-              newMessageKeys, aDestFolder.URI);
+              newMessageKeys, aDestFolder);
           }
           // target is IMAP or something we equally don't understand
           else {
             // XXX the srcFolder will always be the same for now, but we
             //  probably don't want to depend on it, or at least want a unit
             //  test that will break if it changes...
             let srcFolder = aSrcMsgHdrs.queryElementAt(0,Ci.nsIMsgDBHdr).folder;
     
@@ -1551,17 +1672,17 @@ var GlodaIndexer = {
             let messageKeys = [];
             for (let iMsgHdr = 0; iMsgHdr < aSrcMsgHdrs.length; iMsgHdr++) {
               let msgHdr = aSrcMsgHdrs.queryElementAt(iMsgHdr, Ci.nsIMsgDBHdr);
               messageKeys.push(msgHdr.messageKey);
             }
             // XXX we could extract the gloda message id's instead.
             // quickly move them to the right folder, zeroing their message keys
             this.indexer._datastore.updateMessageFoldersByKeyPurging(
-              srcFolder.URI, messageKeys, aDestFolder.URI);
+              srcFolder, messageKeys, aDestFolder);
             // we _do not_ need to mark the folder as dirty, because the
             //  message added events will cause that to happen.
           }
         }
        // copy case
         else {
           // mark the folder as dirty; we'll get to it later.
           aDestFolder.setStringProperty(this.indexer.GLODA_DIRTY_PROPERTY, "1");
@@ -1582,21 +1703,20 @@ var GlodaIndexer = {
      *  underlying account implementation, so we explicitly handle each case.
      *  Namely, we treat it as if we're only planning on getting one, but we
      *  handle if the children are already gone for some reason.
      */
     folderDeleted: function gloda_indexer_folderDeleted(aFolder) {
       this.indexer._log.debug("folderDeleted notification");
       
       delFunc = function(folder) {
-        let folderURI = aFolder.URI;
-        if (this._datastore._folderURIKnown(aFolder.URI)) {
-          let folderID = GlodaDatastore._mapFolderURI(aFolder.URI);
-          this._datastore.markMessagesDeletedByID(folderID);
-          this._datastore.deleteFolderByID(folderID);
+        if (this._datastore._folderKnown(aFolder)) {
+          let folder = GlodaDatastore._mapFolder(aFolder);
+          this._datastore.markMessagesDeletedByID(folder.id);
+          this._datastore.deleteFolderByID(folder.id);
         }
       };
 
       let descendentFolders = Cc["@mozilla.org/supports-array;1"].
                                 createInstance(Ci.nsISupportsArray);
       aFolder.ListDescendents(descendentFolders);
       
       // (the order of operations does not matter; child, non-child, whatever.)
@@ -1642,17 +1762,17 @@ var GlodaIndexer = {
     _folderRenameHelper: function gloda_indexer_folderRenameHelper(aOrigFolder,
                                                                    aNewURI) {
       let descendentFolders = Cc["@mozilla.org/supports-array;1"].
                                 createInstance(Ci.nsISupportsArray);
       aOrigFolder.ListDescendents(descendentFolders);
       
       let origURI = aOrigFolder.URI;
       // this rename is straightforward.
-      GlodaDatastore.renameFolder(origURI, aNewURI);
+      GlodaDatastore.renameFolder(aOrigFolder, aNewURI);
       
       for (let folder in fixIterator(descendentFolders, Ci.nsIMsgFolder)) {
         let oldSubURI = folder.URI;
         // mangle a new URI from the old URI.  we could also try and do a
         //  parallel traversal of the new folder hierarchy, but that seems like
         //  more work.
         let newSubURI = aNewURI + oldSubURI.substring(origURI.length)
         this.indexer._datastore.renameFolder(oldSubURI, newSubURI);
@@ -1728,17 +1848,17 @@ var GlodaIndexer = {
         this.indexer._pendingAddJob = new IndexingJob("message", 1, null);
         this.indexer._indexQueue.push(this.indexer._pendingAddJob);
         this.indexer._indexingJobGoal++;
       }
       // only queue the message if we haven't overflowed our event-driven budget
       if (this.indexer._pendingAddJob.items.length <
           this.indexer._indexMaxEventQueueMessages)
         this.indexer._pendingAddJob.items.push(
-          [GlodaDatastore._mapFolderURI(msgFolder.URI),
+          [GlodaDatastore._mapFolder(msgFolder).id,
            aMsgHdr.messageKey]);
       this.indexer.indexing = true;
     },
   
     OnItemAdded: function gloda_indexer_OnItemAdded(aParentItem, aItem) {
     },
     OnItemRemoved: function gloda_indexer_OnItemRemoved(aParentItem, aItem) {
     },
@@ -1846,48 +1966,37 @@ var GlodaIndexer = {
       return !this.indexer._shutdown(aUrlListener);
     },
     
     getCurrentTaskName: function gloda_indexer_getCurrentTaskName() {
       return "Global Database Indexer"; // L10n-me
     },
   }, 
   
-  _indexMessage: function gloda_indexMessage(aMsgHdr) {
+  _indexMessage: function gloda_indexMessage(aMsgHdr, aCallbackHandle) {
     this._log.debug("*** Indexing message: " + aMsgHdr.messageKey + " : " +
                     aMsgHdr.subject);
-    MsgHdrToMimeMessage(aMsgHdr, this, this._indexMessageWithBody);
-    return this.kWorkAsync;
-  },
-  
-  _indexMessageWithBody: function gloda_index_indexMessageWithBody(
-       aMsgHdr, aMimeMsg) {
+    MsgHdrToMimeMessage(aMsgHdr, aCallbackHandle, aCallbackHandle.callback);
+    let aMimeMsg = yield this.kWorkAsync;
 
     // -- Find/create the conversation the message belongs to.
     // Our invariant is that all messages that exist in the database belong to
     //  a conversation.
     
     // - See if any of the ancestors exist and have a conversationID...
     // (references are ordered from old [0] to new [n-1])
     let references = [aMsgHdr.getStringReference(i) for each
                       (i in range(0, aMsgHdr.numReferences))];
     // also see if we already know about the message...
     references.push(aMsgHdr.messageId);
     
-    this._datastore.getMessagesByMessageID(references,
-      this._indexMessageWithBodyAndAncestors, this,
-      [references, aMsgHdr, aMimeMsg]);
-    
-    return this.kWorkAsync;
-  },
-  
-  _indexMessageWithBodyAndAncestors:
-    function gloda_index_indexMessageWithBodyAndAncestors(ancestorLists,
-      references, aMsgHdr, aMimeMsg) {
+    this._datastore.getMessagesByMessageID(references, aCallbackHandle.callback,
+      aCallbackHandle);
     // (ancestorLists has a direct correspondence to the message ids)
+    let ancestorLists = yield kWorkAsync; 
     
     // pull our current message lookup results off
     references.pop();
     let candidateCurMsgs = ancestorLists.pop();
     
     let conversationID = null;
     // -- figure out the conversation ID
     // if we have a clone/already exist, just use his conversation ID
@@ -1996,57 +2105,48 @@ var GlodaIndexer = {
       if (allAttachmentNames)
         attachmentNames = allAttachmentNames.join("\n");
     } 
     
     let isNew;
     if (curMsg === null) {
       this._log.debug("...creating new message.  body length: " +
                       (aMimeMsg ? aMimeMsg.body.length : null));
-      curMsg = this._datastore.createMessage(aMsgHdr.folder.URI,
+      curMsg = this._datastore.createMessage(aMsgHdr.folder,
                                              aMsgHdr.messageKey,                
                                              conversationID,
                                              aMsgHdr.date,
                                              aMsgHdr.messageId,
                                              aMsgHdr.subject,
                                              aMimeMsg ? aMimeMsg.body : null,
                                              attachmentNames);
       isNew = true;
     }
     else {
       isNew = (curMsg._folderID === null); // aka was-a-ghost
       // (messageKey can be null if it's not new in the move-case)
-      curMsg._folderID = this._datastore._mapFolderURI(aMsgHdr.folder.URI);
+      curMsg._folderID = this._datastore._mapFolder(aMsgHdr.folder).id;
       curMsg._messageKey = aMsgHdr.messageKey;
       curMsg.date = new Date(aMsgHdr.date / 1000); 
       // note: we are assuming that our matching logic is flawless in that
       //  if this message was not a ghost, we are assuming the 'body'
       //  associated with the id is still exactly the same.  It is conceivable
       //  that there are cases where this is not true.
       this._datastore.updateMessage(curMsg, isNew ? aMsgHdr.subject : null,
         (isNew && aMimeMsg) ? aMimeMsg.body : null,
         isNew ? attachmentNames : null);
     }
     
-    // TODO: provide the parent gloda message if we can conjure it up.
-    Gloda.processMessage(curMsg, aMsgHdr, aMimeMsg, isNew,
-                         /* parent gloda message */ null);
+    yield aCallbackHandle.pushAndGo("Process Message Attributes",
+      Gloda.grokNounItem(curMsg, {header: aMsgHdr, mime: aMimeMsg}, isNew));
     
-    // Mark this message as indexed
-    aMsgHdr.setUint32Property(this.GLODA_MESSAGE_ID_PROPERTY, curMsg.id);
-    // If there is a gloda-dirty flag on there, clear it by writing a 0.  (But
-    //  don't do this if we didn't have a dirty flag on there in the first
-    //  case.)  It sounds like we would actually prefer to "cut" the "cell",
-    //  but I don't see any in-domain means of doing that.
-    try {
-      let isDirty = aMsgHdr.getUint32Property(this.GLODA_DIRTY_PROPERTY);
-      if (isDirty)
-        aMsgHdr.setUint32Property(this.GLODA_DIRTY_PROPERTY, 0);
-    }
-    catch (ex) {}
+    // we want to update the header for messages only after the transaction
+    //  irrevocably hits the disk.  otherwise we could get confused if the
+    //  transaction rolls back or what not.
+    GlodaDatastore.runPostCommit(MakeCleanMsgHdrCallback(aMsgHdr));
     
     this.callbackDriver();
   },
   
   /**
    * Wipe a message out of existence from our index.  This is slightly more
    *  tricky than one would first expect because there are potentially
    *  attributes not immediately associated with this message that reference
--- a/modules/noun_freetag.js
+++ b/modules/noun_freetag.js
@@ -56,17 +56,17 @@ FreeTag.prototype = {
 
 /**
  * @namespace Tag noun provider.  Since the tag unique value is stored as a
  *  parameter, we are an odd case and semantically confused.
  */
 var FreeTagNoun = {
   name: "freetag",
   class: FreeTag,
-  firstClass: false,
+  allowsArbitraryAttrs: false,
   
   _listeners: [],
   addListener: function(aListener) {
     this._listeners.push(aListener);
   },
   removeListener: function(aListener) {
     let index = this._listeners.indexOf(aListener);
     if (index >=0)
@@ -79,19 +79,25 @@ var FreeTagNoun = {
     if (!tag) {
       tag = this.knownFreeTags[aTagName] = new FreeTag(aTagName);
       for each (let [iListener, listener] in Iterator(this._listeners))
         listener.onFreeTagAdded(tag);
     }
     return tag;
   },
 
-  toParamAndValue: function gloda_noun_tag_toParamAndValue(aTag) {
+  toParamAndValue: function gloda_noun_freetag_toParamAndValue(aTag) {
     return [aTag.name, null];
   },
+  fromParamAndValue: function gloda_noun_freetag_fromParameterValue(aTagName,
+                                                                    aIgnored) {
+    return this.getFreeTag(aTagName);
+  },
   
-  fromParamAndValue: function gloda_noun_tag_fromParameterValue(aTagName,
-                                                                aIgnored) {
+  toJSON: function gloda_noun_freetag_toJSON(aTag) {
+    return aTag.name;
+  },
+  fromJSON: function gloda_noun_freetag_fromJSON(aTagName) {
     return this.getFreeTag(aTagName);
   },
 };
 
 Gloda.defineNoun(FreeTagNoun);
--- a/modules/noun_tag.js
+++ b/modules/noun_tag.js
@@ -30,74 +30,69 @@
  * use your version of this file under the terms of the MPL, indicate your
  * decision by deleting the provisions above and replace them with the notice
  * and other provisions required by the GPL or the LGPL. If you do not delete
  * the provisions above, a recipient may use your version of this file under
  * the terms of any one of the MPL, the GPL or the LGPL.
  * 
  * ***** END LICENSE BLOCK ***** */
 
-EXPORTED_SYMBOLS = ['Tagged', 'TagNoun'];
+EXPORTED_SYMBOLS = ['TagNoun'];
 
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cr = Components.results;
 const Cu = Components.utils;
 
 Cu.import("resource://gloda/modules/gloda.js");
 
 /**
- * @class Represents a tag applied at a certain time.  Or rather it would if we
- *  were clever enough to track and maintain that time accurately.
- */
-function Tagged(aTag, aDate) {
-  this.tag = aTag;
-  this.date = aDate;
-}
-
-Tagged.prototype = {
-  toString: function () {
-    return this.tag.tag;
-  }
-};
-
-/**
- * @namespace Tag noun provider.  Since the tag unique value is stored as a
- *  parameter, we are an odd case and semantically confused.
+ * @namespace Tag noun provider.
  */
 var TagNoun = {
   name: "tag",
-  class: Tagged,
-  firstClass: false,
+  class: Ci.nsIMsgTag,
+  allowsArbitraryAttrs: false,
   _msgTagService: null,
+  _tagMap: null,
   
   _init: function () {
     this._msgTagService = Cc["@mozilla.org/messenger/tagservice;1"].
                           getService(Ci.nsIMsgTagService);
+    this._updateTagMap();
+    
+    this.fromJSON = this.fromParamAndValue;
+  },
+  
+  _updateTagMap: function gloda_noun_tag_updateTagMap() {
+    this._tagMap = {};
+    let tagArray = this._msgTagService.getAllTags({});
+    for (let iTag = 0; iTag < tagArray.length; iTag++) {
+      let tag = tagArray[iTag];
+      this._tagMap[tag.key] = tag;
+    }
   },
   
   // we cannot be an attribute value
   
-  toParamAndValue: function gloda_noun_tag_toParamAndValue(aTagged, aGeneric) {
-    if (aGeneric)
-      return [aTagged.tag.key, null];
-    else
-      return [aTagged.tag.key, aTagged.date.valueOf() * 1000];
+  toParamAndValue: function gloda_noun_tag_toParamAndValue(aTag) {
+    return [aTag.key, null];
   },
-  
+  toJSON: function gloda_noun_tag_toJSON(aTag) {
+    return aTag.key;
+  }
   fromParamAndValue: function gloda_noun_tag_fromParameterValue(aTagKey,
-                                                                aPRTime) {
-    // we have to walk the array to find our tag.  curse you, tag service!
-    let tagService = Cc["@mozilla.org/messenger/tagservice;1"].
-                          getService(Ci.nsIMsgTagService);
-    let tagArray = tagService.getAllTags({});
-    for (let iTag = 0; iTag < tagArray.length; iTag++) {
-      let tag = tagArray[iTag];
-      if (tag.key == aTagKey)
-        return new Tagged(tag, new Date(aPRTime/1000));
+                                                                aIgnored) {
+    let tag = this._tagMap[aTagKey];
+    // you will note that if a tag is removed, we are unable to aggressively
+    //  deal with this.  we are okay with this, but it would be nice to be able
+    //  to listen to the message tag service to know when we should rebuild.
+    if ((tag === undefined) && this._msgTagService.isValidKey(aTagKey)) {
+      this._updateTagMap();
+      tag = this._tagMap[aTagKey];
     }
-    // the tag has gone a-way, null is probably the safest thing to do.
-    return null;
+    // we intentionally are returning undefined if the tag doesn't exist
+    return tag;
   },
 };
 
 TagNoun._init();
 Gloda.defineNoun(TagNoun, Gloda.NOUN_TAG);
--- a/modules/query.js
+++ b/modules/query.js
@@ -60,16 +60,19 @@ Cu.import("resource://gloda/modules/log4
 function GlodaQueryClass() {
   // if we are an 'or' clause, who is our parent whom other 'or' clauses should
   //  spawn from...
   this._owner = null;
   // our personal chain of and-ing.
   this._constraints = [];
   // the other instances we union with
   this._unions = [];
+  
+  this._order = [];
+  this._limit = 0;
 }
 
 GlodaQueryClass.prototype = {
   WILDCARD: {},
   
   get constraintCount() {
     return this._constraints.length;
   },
@@ -77,26 +80,38 @@ GlodaQueryClass.prototype = {
   or: function gloda_query_or() {
     let owner = this._owner || this;
     let orQuery = new this._queryClass();
     orQuery._owner = owner;
     owner._unions.push(orQuery);
     return orQuery;
   },
   
+  orderBy: function gloda_query_orderBy() {
+    for (let iArg = 0; iArg < arguments.length; iArg++) {
+      let arg = arguments[iArg];
+      this._order.push(arg);
+    }
+  },
+  
+  limit: function gloda_query_limit(aLimit) {
+    this._limit = aLimit;
+  },
+  
   /**
    * Return a collection asynchronously populated by this collection.  You must
    *  provide a listener to receive notifications from the collection as it
    *  receives updates.  The listener object should implement onItemsAdded,
    *  onItemsModified, and onItemsRemoved methods, all of which take a single
    *  argument which is the list of items which have been added, modified, or
    *  removed respectively.
    */
-  getCollection: function gloda_query_getAll(aListener) {
-    return this._nounMeta.datastore.queryFromQuery(this, aListener);
+  getCollection: function gloda_query_getCollection(aListener, aData) {
+    return this._nounMeta.datastore.queryFromQuery(this, aListener, false,
+      aData);
   },
   
   getAllSync: function gloda_query_getAllSync(aListener) {
     return this._nounMeta.datastore.queryFromQuery(this, aListener, true);
   },
   
   /**
    * Test whether the given first-class noun instance satisfies this query.
--- a/modules/utils.js
+++ b/modules/utils.js
@@ -105,9 +105,30 @@ var GlodaUtils = {
      // return the two-digit hexadecimal code for a byte
     function toHexString(charCode) {
       return ("0" + charCode.toString(16)).slice(-2);
     }
 
     // convert the binary hash data to a hex string.
     return [toHexString(hash.charCodeAt(i)) for (i in hash)].join("");
   },
+  
+  getCardForEmail: function gloda_utils_getCardForEmail(aAddress) {
+    // search through all of our local address books looking for a match.
+    let enumerator = Components.classes["@mozilla.org/abmanager;1"]
+                               .getService(Ci.nsIAbManager)
+                               .directories;
+    let cardForEmailAddress;
+    let addrbook;
+    while (!cardForEmailAddress && enumerator.hasMoreElements())
+    {
+      addrbook = enumerator.getNext().QueryInterface(Ci.nsIAbDirectory);
+      try
+      {
+        cardForEmailAddress = addrbook.cardForEmailAddress(aAddress);
+        if (cardForEmailAddress)
+          return cardForEmailAddress;
+      } catch (ex) {}
+    }
+
+    return null;
+  },
 };