fix Bug 464359 - gloda query logic for collection manager for newly indexed messages insufficient r/a=bienvenu
authorAndrew Sutherland bugmail@asutherland.org
Wed, 26 Nov 2008 11:37:25 -0800
changeset 1240 6463e9ac9b66b8ac26a862ff62ce20bd0ca3c11b
parent 1239 e8e722742c7090805c01ec9b209c136d75ad5447
child 1241 4baeb39a60ad941b8e1f542fe5be6c1d899832a8
push idunknown
push userunknown
push dateunknown
bugs464359
fix Bug 464359 - gloda query logic for collection manager for newly indexed messages insufficient r/a=bienvenu
mailnews/db/gloda/modules/datastore.js
mailnews/db/gloda/modules/gloda.js
mailnews/db/gloda/modules/indexer.js
mailnews/db/gloda/modules/query.js
mailnews/db/gloda/test/resources/glodaTestHelper.js
mailnews/db/gloda/test/unit/test_query_messages.js
--- a/mailnews/db/gloda/modules/datastore.js
+++ b/mailnews/db/gloda/modules/datastore.js
@@ -2672,16 +2672,24 @@ var GlodaDatastore = {
       aListenerData, aExistingCollection, aMasterCollection, aBecomeExplicit) {
     // when changing this method, be sure that GlodaQuery's testMatch function
     //  likewise has its changes made.
     let nounDef = aQuery._nounDef;
 
     let whereClauses = [];
     let unionQueries = [aQuery].concat(aQuery._unions);
     let boundArgs = [];
+    
+    // Use the dbQueryValidityConstraintSuffix to provide constraints that
+    //  filter items down to those that are valid for the query mechanism to
+    //  return.  For example, in the case of messages, deleted or ghost
+    //  messages should not be returned by this query layer.  We require
+    //  hand-rolled SQL to do that for now.
+    let validityConstraintSuffix  =
+      nounDef.dbQueryValidityConstraintSuffix || "";
 
     for (let iUnion = 0; iUnion < unionQueries.length; iUnion++) {
       let curQuery = unionQueries[iUnion];
       let selects = [];
       
       let lastConstraintWasSpecial = false;
       let curConstraintIsSpecial;
 
@@ -2798,17 +2806,18 @@ var GlodaDatastore = {
         else
           this._log.warning("Unable to translate constraint of type " + 
             constraintType + " on attribute bound as " + aAttrDef.boundName);
 
         lastConstraintWasSpecial = curConstraintIsSpecial;
       }
 
       if (selects.length)
-        whereClauses.push("id IN (" + selects.join(" INTERSECT ") + ")");
+        whereClauses.push("id IN (" + selects.join(" INTERSECT ") + ")" +
+                          validityConstraintSuffix);
     }
 
     let sqlString = "SELECT * FROM " + nounDef.tableName;
     if (whereClauses.length)
       sqlString += " WHERE " + whereClauses.join(" OR ");
     
     if (aQuery._order.length) {
       let orderClauses = [];
--- a/mailnews/db/gloda/modules/gloda.js
+++ b/mailnews/db/gloda/modules/gloda.js
@@ -972,16 +972,18 @@ var Gloda = {
       name: "message",
       class: GlodaMessage,
       allowsArbitraryAttrs: true,
       cache: true, cacheCost: 2048,
       tableName: "messages",
       attrTableName: "messageAttributes", attrIDColumnName: "messageID",
       datastore: GlodaDatastore, objFromRow: GlodaDatastore._messageFromRow,
       dbAttribAdjuster: GlodaDatastore.adjustMessageAttributes,
+      dbQueryValidityConstraintSuffix:
+        " AND deleted = 0 AND folderID IS NOT NULL AND messageKey IS NOT NULL",
       objInsert: GlodaDatastore.insertMessage,
       objUpdate: GlodaDatastore.updateMessage,
       toParamAndValue: function(aMessage) {
         if (aMessage instanceof GlodaMessage)
           return [null, aMessage.id];
         else // assume they're just passing the id directly
           return [null, aMessage];
       }}, this.NOUN_MESSAGE);
@@ -1152,17 +1154,17 @@ var Gloda = {
       // - ranged value helper: fooRange
       if (objectNounDef.continuous) {
         // takes one or more tuples of [lower bound, upper bound]
         let rangedConstrainer = function() {
           let constraint = [GlodaDatastore.kConstraintRanges, aAttrDef];
           for (let iArg = 0; iArg < arguments.length; iArg++ ) {
             constraint.push(arguments[iArg]);
           }
-          this._constraints.push(constraints);
+          this._constraints.push(constraint);
           return this;
         }
 
         aSubjectNounDef.queryClass.prototype[aAttrDef.boundName + "Range"] =
           rangedConstrainer;
       }
 
       // - string LIKE helper for special on-row attributes: fooLike
--- a/mailnews/db/gloda/modules/indexer.js
+++ b/mailnews/db/gloda/modules/indexer.js
@@ -2127,45 +2127,48 @@ var GlodaIndexer = {
     this._log.debug("references: " + Log4Moz.enumerateProperties(references).join(","));
     this._log.debug("ancestors: " + Log4Moz.enumerateProperties(ancestorLists).join(","));
     
     // pull our current message lookup results off
     references.pop();
     let candidateCurMsgs = ancestorLists.pop();
     
     let conversationID = null;
+    let conversation = null;
     // -- figure out the conversation ID
     // if we have a clone/already exist, just use his conversation ID
     if (candidateCurMsgs.length > 0) {
       conversationID = candidateCurMsgs[0].conversationID;
+      conversation = candidateCurMsgs[0].conversation;
     }
     // otherwise check out our ancestors
     else {
       // (walk from closest to furthest ancestor)
       for (let iAncestor = ancestorLists.length-1; iAncestor >= 0;
           --iAncestor) {
         let ancestorList = ancestorLists[iAncestor];
         
         if (ancestorList.length > 0) {
           // we only care about the first instance of the message because we are
           //  able to guarantee the invariant that all messages with the same
           //  message id belong to the same conversation. 
           let ancestor = ancestorList[0];
-          if (conversationID === null)
+          if (conversationID === null) {
             conversationID = ancestor.conversationID;
+            conversation = ancestor.conversation;
+          }
           else if (conversationID != ancestor.conversationID)
             this._log.error("Inconsistency in conversations invariant on " +
                             ancestor.headerMessageID + ".  It has conv id " +
                             ancestor.conversationID + " but expected " + 
                             conversationID + ". ID: " + ancestor.id);
         }
       }
     }
     
-    let conversation = null;
     // nobody had one?  create a new conversation
     if (conversationID === null) {
       // (the create method could issue the id, making the call return
       //  without waiting for the database...)
       conversation = this._datastore.createConversation(
           aMsgHdr.mime2DecodedSubject, null, null);
       conversationID = conversation.id;
     }
--- a/mailnews/db/gloda/modules/query.js
+++ b/mailnews/db/gloda/modules/query.js
@@ -39,16 +39,20 @@ EXPORTED_SYMBOLS = ["GlodaQueryClassFact
 
 const Cc = Components.classes;
 const Ci = Components.interfaces;
 const Cr = Components.results;
 const Cu = Components.utils;
 
 Cu.import("resource://app/modules/gloda/log4moz.js");
 
+// GlodaDatastore has some constants we need, and oddly enough, there was no
+//  load dependency preventing us from doing this.
+Cu.import("resource://app/modules/gloda/datastore.js");
+
 /**
  * @class Query class core; each noun gets its own sub-class where attributes
  *  have helper methods bound.
  * 
  * @property _owner The query instance that holds the list of unions...
  * @property _constraints A list of (lists of OR constraints) that are ANDed
  *     together.  For example [[FROM bob, FROM jim], [DATE last week]] would
  *     be requesting us to find all the messages from either bob or jim, and
@@ -124,25 +128,25 @@ GlodaQueryClass.prototype = {
 
       // assume success until a specific (or) constraint proves us wrong
       let querySatisfied = true;
       for (let iConstraint = 0; iConstraint < curQuery._constraints.length; 
            iConstraint++) {
         let constraint = curQuery._constraints[iConstraint];
         let [constraintType, attrDef] = constraint;
         let constraintValues = constraint.slice(2);
-        
-        if (constraintType === this.kConstraintIdIn) {
+
+        if (constraintType === GlodaDatastore.kConstraintIdIn) {
           if (constraintValues.indexOf(aObj.id) == -1) {
             querySatisfied = false;
             break;
           }
         }
-        else if ((constraintType === this.kConstraintIn) ||
-                 (constraintType === this.kConstraintEquals)) {
+        else if ((constraintType === GlodaDatastore.kConstraintIn) ||
+                 (constraintType === GlodaDatastore.kConstraintEquals)) {
           let objectNounDef = attrDef.objectNounDef;
           
           // if they provide an equals comparator, use that.
           // (note: the next case has better optimization possibilities than
           //  this mechanism, but of course has higher initialization costs or
           //  code complexity costs...)
           if (objectNounDef.equals) {
             let testValues;
@@ -179,43 +183,45 @@ GlodaQueryClass.prototype = {
             else
               testValues = aObj[attrDef.boundName];
 
             let foundMatch = false;
             for each (let [,testValue] in Iterator(testValues)) {
               let [aParam, aValue] = objectNounDef.toParamAndValue(testValue);
               for each (let [,value] in Iterator(constraintValues)) {
                 let [bParam, bValue] = objectNounDef.toParamAndValue(value);
-                if (aParam == bParam && aVAlue == bValue) {
+                if (aParam == bParam && aValue == bValue) {
                   foundMatch = true;
                   break;
                 }
               }
               if (foundMatch)
                 break;
             }
             if (!foundMatch) {
               querySatisfied = false;
               break;
             }
           }
         }
-        else if (constraintType === this.kConstraintRanges) {
+        else if (constraintType === GlodaDatastore.kConstraintRanges) {
+          let objectNounDef = attrDef.objectNounDef;
+          
           let testValues;
           if (attrDef.singular)
             testValues = [aObj[attrDef.boundName]];
           else
             testValues = aObj[attrDef.boundName];
 
           let foundMatch = false;
           for each (let [,testValue] in Iterator(testValues)) {
             let [tParam, tValue] = objectNounDef.toParamAndValue(testValue);
             for each (let [,rangeTuple] in Iterator(constraintValues)) {
-              let [lowRValue, upperRValue] = rangeTuple;
-              if (lowRValue == null) {
+              let [lowerRValue, upperRValue] = rangeTuple;
+              if (lowerRValue == null) {
                 let [upperParam, upperValue] =
                   objectNounDef.toParamAndValue(upperRValue);
                 if (tParam == upperParam && tValue <= upperValue) {
                   foundMatch = true;
                   break;
                 }
               }
               else if (upperRValue == null) {
@@ -241,17 +247,17 @@ GlodaQueryClass.prototype = {
             if (foundMatch)
               break;
           }
           if (!foundMatch) {
             querySatisfied = false;
             break;
           }
         }
-        else if (constraintType === this.kConstraintStringLike) {
+        else if (constraintType === GlodaDatastore.kConstraintStringLike) {
           let curIndex = 0;
           let value = aObj[attrDef.boundName];
           // the attribute must be singular, we don't support arrays of strings.
           for each (let [iValuePart, valuePart] in Iterator(constraintValues)) {
             if (typeof valuePart == "string") {
               let index = value.indexOf(valuePart);
               // if curIndex is null, we just need any match
               // if it's not null, it must match the offset of our found match
@@ -272,29 +278,28 @@ GlodaQueryClass.prototype = {
             }
             else // wild!
               curIndex = null;
           }
           // curIndex must be null or equal to the length of the string
           if (querySatisfied && curIndex !== null && curIndex != value.length)
             querySatisfied = false;
         }
-        else if (constraintType === this.kConstraintFulltext) {
+        else if (constraintType === GlodaDatastore.kConstraintFulltext) {
           // this is beyond our powers.  don't match.
           querySatisfied = false;
         }
         
         if (!querySatisfied)
           break;
       }
       
       if (querySatisfied)
         return true;
     }
-    
     return false;
   },
 };
 
 /**
  * @class A query that never matches anything.
  * 
  * Collections corresponding to this query are intentionally frozen in time and
--- a/mailnews/db/gloda/test/resources/glodaTestHelper.js
+++ b/mailnews/db/gloda/test/resources/glodaTestHelper.js
@@ -54,16 +54,142 @@ Components.utils.import("resource://app/
 Components.utils.import("resource://app/modules/gloda/indexer.js");
 
 // -- Add a logger listener that throws when we give it a warning/error.
 Components.utils.import("resource://app/modules/gloda/log4moz.js");
 let throwingAppender = new Log4Moz.ThrowingAppender(do_throw);
 throwingAppender.level = Log4Moz.Level.Warn;
 Log4Moz.Service.rootLogger.addAppender(throwingAppender);
 
+/**
+ * davida's patented dump function for what ails you. 
+ */
+function ddumpObject(obj, name, maxDepth, curDepth)
+{
+  if (curDepth == undefined)
+    curDepth = 0;
+  if (maxDepth != undefined && curDepth > maxDepth)
+    return;
+
+  var i = 0;
+  for (prop in obj)
+  {
+    i++;
+    try {
+      if (typeof(obj[prop]) == "object")
+      {
+        if (obj[prop] && obj[prop].length != undefined)
+          ddump(name + "." + prop + "=[probably array, length "
+                + obj[prop].length + "]");
+        else
+          ddump(name + "." + prop + "=[" + typeof(obj[prop]) + "] (" +
+                obj[prop] + ")");
+        ddumpObject(obj[prop], name + "." + prop, maxDepth, curDepth+1);
+      }
+      else if (typeof(obj[prop]) == "function")
+        ddump(name + "." + prop + "=[function]");
+      else
+        ddump(name + "." + prop + "=" + obj[prop]);
+    } catch (e) {
+      ddump(name + "." + prop + "-> Exception(" + e + ")");
+    }
+  }
+  if (!i)
+    ddump(name + " is empty");
+}
+/** its kid brother */
+function ddump(text)
+{
+    dump(text + "\n");
+}
+
+function dumpExc(e, message) {
+  var objDump = getObjectTree(e,1);
+  if (typeof(e) == 'object' && 'stack' in e)
+      objDump += e.stack;
+  if (typeof(message)=='undefined' || !message)
+      message='';
+  dump(message+'\n-- EXCEPTION START --\n'+objDump+'-- EXCEPTION END --\n');
+}
+
+function getObjectTree(o, recurse, compress, level)
+{
+    var s = "";
+    var pfx = "";
+
+    if (typeof recurse == "undefined")
+        recurse = 0;
+    if (typeof level == "undefined")
+        level = 0;
+    if (typeof compress == "undefined")
+        compress = true;
+
+    for (var i = 0; i < level; i++)
+        pfx += (compress) ? "| " : "|  ";
+
+    var tee = (compress) ? "+ " : "+- ";
+
+    if (typeof(o) != 'object') {
+        s += pfx + tee + i + " (" + typeof(o) + ") " + o + "\n";
+    } else
+    for (i in o)
+    {
+        var t;
+        try
+        {
+            t = typeof o[i];
+
+            switch (t)
+            {
+                case "function":
+                    var sfunc = String(o[i]).split("\n");
+                    if (sfunc[2] == "    [native code]")
+                        sfunc = "[native code]";
+                    else
+                        sfunc = sfunc.length + " lines";
+                    s += pfx + tee + i + " (function) " + sfunc + "\n";
+                    break;
+
+                case "object":
+                    s += pfx + tee + i + " (object) " + o[i] + "\n";
+                    if (!compress)
+                        s += pfx + "|\n";
+                    if ((i != "parent") && (recurse))
+                        s += getObjectTree(o[i], recurse - 1,
+                                             compress, level + 1);
+                    break;
+
+                case "string":
+                    if (o[i].length > 200)
+                        s += pfx + tee + i + " (" + t + ") " +
+                            o[i].length + " chars\n";
+                    else
+                        s += pfx + tee + i + " (" + t + ") '" + o[i] + "'\n";
+                    break;
+
+                default:
+                    s += pfx + tee + i + " (" + t + ") " + o[i] + "\n";
+            }
+        }
+        catch (ex)
+        {
+            s += pfx + tee + i + " (exception) " + ex + "\n";
+        }
+
+        if (!compress)
+            s += pfx + "|\n";
+
+    }
+
+    s += pfx + "*\n";
+
+    return s;
+}
+
+
 /** Inject messages using a POP3 fake-server. */
 const INJECT_FAKE_SERVER = 1;
 /** Inject messages using freshly created mboxes. */
 const INJECT_MBOX = 2;
 
 /**
  * Convert a list of synthetic messages to a form appropriate to feed to the
  *  POP3 fakeserver.
@@ -580,16 +706,162 @@ function twiddleAndTest(aSynthMsg, aActi
       twiddle_next_attr(smsg, gmsg);
     else
       next_test();
   }
   
   indexMessages([aSynthMsg], twiddle_next_attr);
 }
 
+_defaultExpectationExtractors = {};
+_defaultExpectationExtractors[Gloda.NOUN_MESSAGE] = [
+  function expectExtract_message_gloda(aGlodaMessage) {
+    return aGlodaMessage.headerMessageID;
+  },
+  function expectExtract_message_synth(aSynthMessage) {
+    return aSynthMessage.messageId;
+  }
+];
+_defaultExpectationExtractors[Gloda.NOUN_CONTACT] = [
+  function expectExtract_contact_gloda(aGlodaContact) {
+    return aGlodaContact.name;
+  },
+  function expectExtract_contact_name(aName) {
+    return aName;
+  }
+];
+_defaultExpectationExtractors[Gloda.NOUN_IDENTITY] = [
+  function expectExtract_identity_gloda(aGlodaIdentity) {
+    return aGlodaIdentity.value;
+  },
+  function expectExtract_identity_address(aAddress) {
+    return aAddress;
+  }
+];
+
+function expectExtract_default_toString(aThing) {
+  return aThing.toString();
+}
+
+function QueryExpectationListener(aExpectedSet, aGlodaExtractor) {
+  this.expectedSet = aExpectedSet;
+  this.glodaExtractor = aGlodaExtractor;
+  this.completed = false;
+}
+
+QueryExpectationListener.prototype = {
+  onItemsAdded: function query_expectation_onItemsAdded(aItems, aCollection) {
+    for each (let [, item] in Iterator(aItems)) {
+      let glodaStringRep;
+      try {
+        glodaStringRep = this.glodaExtractor(item);
+      }
+      catch (ex) {
+        do_throw("Gloda extractor threw during query expectation for item: " +
+                 item + " exception: " + ex);
+      }
+      
+      // make sure we were expecting this guy
+      if (glodaStringRep in this.expectedSet)
+        delete this.expectedSet[glodaStringRep];
+      else {
+        ddumpObject(item, "item", 0);
+        ddumpObject(this.expectedSet, "expectedSet", 1);
+        do_throw("Query returned unexpected result! gloda rep:" +
+                 glodaStringRep);
+      }
+      
+      // make sure the query's test method agrees with the database about this
+      if (!aCollection.query.test(item))
+        do_throw("Query test returned false when it should have been true on " +
+                 "extracted: " + glodaStringRep + " item: " + item);
+    }
+  },
+  onItemsModified: function query_expectation_onItemsModified(aItems,
+      aCollection) {
+  },
+  onItemsRemoved: function query_expectation_onItemsRemoved(aItems,
+      aCollection) {
+  },
+  onQueryCompleted: function query_expectation_onQueryCompleted(aCollection) {
+    // we may continue to match newly added items if we leave our query as it
+    //  is, so let's become explicit to avoid related troubles.
+    aCollection.becomeExplicit();
+    
+    // expectedSet should now be empty
+    for each (let [key, value] in this.expectedSet) {
+      do_throw("Query should have returned " + key + "(" + value + ")");
+    }
+    
+    next_test();
+  },
+}
+
+/**
+ * Execute the given query, verifying that the result set contains exactly the
+ *  contents of the expected set; no more, no less.  Since we expect that the
+ *  query will result in gloda objects, but your expectations will not be posed
+ *  in terms of gloda objects (though they could be), we rely on extractor
+ *  functions to take the gloda result objects and the expected result objects
+ *  into the same string.
+ * If you don't provide extractor functions, we will use our defaults (based on
+ *  the query noun type) if available, or assume that calling toString is
+ *  sufficient.
+ * Calls next_test automatically once the query completes and the results are
+ *  checked.
+ * 
+ * @param aQuery The query to execute.
+ * @param aExpectedSet The list of expected results from the query.
+ * @param aGlodaExtractor The extractor function to take an instance of the
+ *     gloda representation and return a string for comparison/equivalence
+ *     against that returned by the expected extractor (against the input
+ *     instance in aExpectedSet.)  The value returned must be unique for all
+ *     of the expected gloda representations of the expected set.  If omitted,
+ *     the default extractor for the gloda noun type is used.
+ * @param aExpectedExtractor The extractor function to take an instance from the
+ *     values in the aExpectedSet and return a string for comparison/equivalence
+ *     against that returned by the gloda extractor.  The value returned must
+ *     be unique for all of the values in the expected set.  If omitted, the
+ *     default extractor for the presumed input type based on the gloda noun
+ *     type used for the query is used.
+ * @returns The collection created from the query.
+ */
+function queryExpect(aQuery, aExpectedSet, aGlodaExtractor,
+    aExpectedExtractor) {
+  // - set extractor functions to defaults if omitted
+  if (aGlodaExtractor == null) {
+    if (_defaultExpectationExtractors[aQuery._nounDef.id] !== undefined)
+      aGlodaExtractor = _defaultExpectationExtractors[aQuery._nounDef.id][0];
+    else
+      aGlodaExtractor = expectExtract_default_toString;
+  }
+  if (aExpectedExtractor == null) {
+    if (_defaultExpectationExtractors[aQuery._nounDef.id] !== undefined)
+      aExpectedExtractor = _defaultExpectationExtractors[aQuery._nounDef.id][1];
+    else
+      aExpectedExtractor = expectExtract_default_toString;
+  }
+  
+  // - build the expected set
+  let expectedSet = {};
+  for each (let [, item] in Iterator(aExpectedSet)) {
+    try {
+      expectedSet[aExpectedExtractor(item)] = item;
+    }
+    catch (ex) {
+      do_throw("Expected extractor threw during query expectation for item: " +
+               item + " exception: " + ex);
+    }
+  }
+  
+  // - create the listener...
+  return aQuery.getCollection(new QueryExpectationListener(expectedSet,
+                                                           aGlodaExtractor));
+}
+
 var glodaHelperTests = [];
 var glodaHelperIterator = null;
 
 function _gh_test_iterator() {
   do_test_pending();
 
   for (let iTest=0; iTest < glodaHelperTests.length; iTest++) {
     dump("====== Test function: " + glodaHelperTests[iTest].name + "\n");
@@ -604,18 +876,33 @@ function _gh_test_iterator() {
   
   // once the control flow hits the root after do_test_finished, we're done,
   //  so let's just yield something to avoid callers having to deal with an
   //  exception indicating completion.
   glodaHelperIterator = null;
   yield null;
 }
 
+var _next_test_currently_in_test = false;
 function next_test() {
-  glodaHelperIterator.next();
+  // to avoid crazy messed up stacks, use a time-out to get us to our next thing
+  if (_next_test_currently_in_test) {
+    do_timeout(0, "next_test()");
+    return;
+  }
+  
+  _next_test_currently_in_test = true;
+  try {
+    glodaHelperIterator.next();
+  }
+  catch (ex) {
+    dumpExc(ex);
+    do_throw("Caught an exception during execution of next_test: " + ex);
+  }
+  _next_test_currently_in_test = false;
 }
 
 function glodaHelperRunTests(aTests) {
   imsInit();
   glodaHelperTests = aTests;
   glodaHelperIterator = _gh_test_iterator();
   next_test();
 }
--- a/mailnews/db/gloda/test/unit/test_query_messages.js
+++ b/mailnews/db/gloda/test/unit/test_query_messages.js
@@ -1,66 +1,359 @@
-/* This file tests our querying support, including full-text search.
+/* This file tests our querying support.
+ * This file is ugly.  Very ugly.  It's an equipotential problem; I'll refactor
+ *  this into something pretty later.  Sorry :(
  */
 
 do_import_script("../mailnews/db/gloda/test/resources/messageGenerator.js");
-
-//these are imported by glodaTestHelper's import of head_maillocal
-// do_import_script("../mailnews/test/resources/mailDirService.js");
-// do_import_script("../mailnews/test/resources/mailTestUtils.js");
 do_import_script("../mailnews/db/gloda/test/resources/glodaTestHelper.js");
 
 // Create a message generator
 var msgGen = new MessageGenerator();
 // Create a message scenario generator using that message generator
 var scenarios = new MessageScenarioFactory(msgGen);
 
+/* ===== Populate ===== */
+var world = {
+  phase: 0,
+    
+  peoples: null,
+  NUM_AUTHORS: 5,
+  authorGroups: {},
+  
+  NUM_CONVERSATIONS: 3,
+  lastMessagesInConvos: [],
+  conversationGroups: {},
+  conversationLists: [],
+  glodaConversationIds: [],
+  
+  NUM_FOLDERS: 2,
+  MESSAGES_PER_FOLDER: 11,
+  folderClumps: [],
+  folderGroups: {},
+  glodaFolders: [],
+  
+  outlierAuthor: null,
+  outlierFriend: null,
+  outliers: [],
+  
+  peoplesMessages: [],
+  outlierMessages: []
+};
+
+function lumpIt(aSynthMessage) {
+  // lump by author
+  let author = aSynthMessage.fromAddress;
+  if (!(author in world.authorGroups))
+    world.authorGroups[author] = [];
+  world.authorGroups[author].push(aSynthMessage);
+  
+  // lump by conversation, keying off of the originator's message id
+  let originator = aSynthMessage;
+  while (originator.parent) {
+    originator = originator.parent;
+  }
+  if (!(originator.messageId in world.conversationGroups))
+    world.conversationGroups[originator.messageId] = [];
+  world.conversationGroups[originator.messageId].push(aSynthMessage);
+  world.conversationLists[aSynthMessage.iConvo].push(aSynthMessage);
+  
+  // folder lumping happens in a big glob
+}
+
+function likeIt() {
+  let messages = [];
+  
+  let iAuthor = 0;
+  for (let iMessage = 0; iMessage < world.MESSAGES_PER_FOLDER; iMessage++) {
+    let iConvo = iMessage % world.NUM_CONVERSATIONS;
+    let smsg = msgGen.makeMessage(world.lastMessagesInConvos[iConvo]);
+    // we need missing messages to create ghosts, so periodically add an extra
+    //  unknown into the equation
+    if ((iMessage % 3) == 0)
+      smsg = msgGen.makeMessage(smsg);
+    
+    // makeMessage is not exceedingly clever right now, we need to overwrite
+    //  From and To...
+    smsg.from = world.peoples[iAuthor];
+    iAuthor = (iAuthor + iConvo + 1) % world.NUM_AUTHORS;
+    // so, everyone is talking to everyone for this stuff
+    smsg.to = world.peoples;
+    world.lastMessagesInConvos[iConvo] = smsg;
+    // simplify lumpIt and glodaInfoStasher's life
+    smsg.iConvo = iConvo;
+    
+    lumpIt(smsg);
+    messages.push(smsg);
+    world.peoplesMessages.push(smsg);
+  }
+  
+  smsg = msgGen.makeMessage();
+  smsg.from = world.outlierAuthor;
+  smsg.to = [world.outlierFriend];
+  // do not lump it
+  messages.push(smsg);
+  world.outlierMessages.push(smsg);
+  
+  world.folderClumps.push(messages);
+  
+  return messages;
+}
+
+/** 
+ * To save ourselves some lookup trouble, pretend to be a verification
+ *  function so we get easy access to the gloda translations of the messages so
+ *  we can cram this in various places. 
+ */
+function glodaInfoStasher(aSynthMessage, aGlodaMessage) {
+  if (aSynthMessage.iConvo !== undefined)
+    world.glodaConversationIds[aSynthMessage.iConvo] =
+      aGlodaMessage.conversation.id;
+  if (world.glodaFolders.length <= world.phase)
+    world.glodaFolders.push(aGlodaMessage.folder);
+}
+
+// first, we must populate our message store with delicious messages.
+function setup_populate() {
+  world.glodaHolderCollection = Gloda.explicitCollection(Gloda.NOUN_MESSAGE,
+    []);
+  
+  world.peoples = msgGen.makeNamesAndAddresses(world.NUM_AUTHORS);
+  world.outlierAuthor = msgGen.makeNameAndAddress();
+  world.outlierFriend = msgGen.makeNameAndAddress();
+  for (let iConvo = 0; iConvo < world.NUM_CONVERSATIONS; iConvo++) {
+    world.lastMessagesInConvos.push(null);
+    world.conversationLists.push([]);
+    world.glodaConversationIds.push(null);
+  }
+  
+  indexMessages(likeIt(), glodaInfoStasher, setup_populate_phase_two);
+}
+
+function setup_populate_phase_two() {
+  world.phase++;
+  indexMessages(likeIt(), glodaInfoStasher, next_test);
+}
+
 /* ===== Non-text queries ===== */
 
 /* === messages === */
 
+/**
+ * Takes a list of mutually exclusive queries and a list of the resulting
+ *  collections and ensures that the collections from one query do not pass the
+ *  query.test() method of one of the other queries.  To restate, the queries
+ *  must not have any overlapping results, or we will get angry without
+ *  justification. 
+ */
+function verify_nonMatches(aQueries, aCollections) {
+  for (let i = 0; i < aCollections.length; i++) {
+    let testQuery = aQueries[i];
+    let nonmatches =
+      aCollections[(i+1) % aCollections.length].items;
+    
+    for each (let [, item] in Iterator(nonmatches)) {
+      if (testQuery.test(item)) {
+        ddumpObject(item, "item", 0);
+        ddumpObject(testQuery._constraints, "constraints", 2);
+        do_throw("Something should not match query.test(), but it does: " +
+                 item);
+      }
+    }
+  }
+}
+
+var ts_convNum = 0;
+var ts_convQueries = [];
+var ts_convCollections = [];
 function test_query_messages_by_conversation() {
+  let convNum = ts_convNum++;
+  let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+  query.conversation(world.glodaConversationIds[convNum]);
+  
+  ts_convQueries.push(query);
+  ts_convCollections.push(queryExpect(query, world.conversationLists[convNum]));
+  // queryExpect calls next_test
+}
+
+function test_query_messages_by_conversation_nonmatches() {
+  verify_nonMatches(ts_convQueries, ts_convCollections);
+  next_test();
 }
 
+var ts_folderNum = 0;
+var ts_folderQueries = [];
+var ts_folderCollections = [];
 function test_query_messages_by_folder() {
+  let folderNum = ts_folderNum++;
+  let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+  query.folder(world.glodaFolders[folderNum]);
+  
+  ts_folderQueries.push(query);
+  ts_folderCollections.push(queryExpect(query, world.folderClumps[folderNum]));
+  // queryExpect calls next_test
+}
+
+function test_query_messages_by_folder_nonmatches() {
+  verify_nonMatches(ts_folderQueries, ts_folderCollections);
+  next_test();
 }
 
-function test_query_messages_by_identity() {
+// at this point we go run the identity and contact tests for side-effects
+
+var ts_messageIdentityQueries = [];
+var ts_messageIdentityCollections = [];
+function test_query_messages_by_identity_peoples() {
+  let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+  query.involves(peoplesIdentityCollection.items[0]);
+  
+  ts_messageIdentityQueries.push(query);
+  ts_messageIdentityCollections.push(queryExpect(query, world.peoplesMessages));
+  // queryExpect calls next_test
+}
+
+function test_query_messages_by_identity_outlier() {
+  let query = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+  query.involves(outlierIdentityCollection.items[0]);
+  // this also tests our ability to have two intersecting constraints! hooray!
+  query.involves(outlierIdentityCollection.items[1]);
+  
+  ts_messageIdentityQueries.push(query);
+  ts_messageIdentityCollections.push(queryExpect(query, world.outlierMessages));
+  // queryExpect calls next_test
+}
+
+function test_query_messages_by_identity_nonmatches() {
+  verify_nonMatches(ts_messageIdentityQueries, ts_messageIdentityCollections);
+  next_test();
 }
 
 function test_query_messages_by_contact() {
+  // IOU
+  next_test();
 }
 
+var ts_messagesDateQuery;
 function test_query_messages_by_date() {
+  ts_messagesDateQuery = Gloda.newQuery(Gloda.NOUN_MESSAGE);
+  // we are clearly relying on knowing the generation sequence here,
+  //  fuggedaboutit
+  ts_messagesDateQuery.dateRange([world.peoplesMessages[1].date,
+                                  world.peoplesMessages[2].date]);
+  queryExpect(ts_messagesDateQuery, world.peoplesMessages.slice(1, 3));
 }
 
+function test_query_messages_by_date_nonmatches() {
+  if (ts_messagesDateQuery.test(world.peoplesMessages[0]) ||
+      ts_messagesDateQuery.test(world.peoplesMessages[3])) {
+    do_throw("The date testing mechanism is busted.");
+  }
+  next_test();
+}
+
+
 /* === contacts === */
 function test_query_contacts_by_popularity() {
+  // IOU
+  next_test();
 }
 
 /* === identities === */
 
 /* ===== Text-based queries ===== */
 
 /* === conversations === */
 
 function test_query_conversations_by_subject_text() {
+  // IOU
+  next_test();
 }
 
 /* === messages === */
 
 function test_query_messages_by_body_text() {
+  // IOU
+  next_test();
 }
 
 /* === contacts === */
 
+var contactLikeQuery;
 function test_query_contacts_by_name() {
+  // let's use like... we need to test that...
+  contactLikeQuery = Gloda.newQuery(Gloda.NOUN_CONTACT);
+  let personName = world.peoples[0][0];
+  // chop off the first and last letter...  this isn't the most edge-case
+  //  handling way to roll, but LOOK OVER THERE? IS THAT ELVIS?
+  let personNameSubstring = personName.substring(1, personName.length-1);
+  contactLikeQuery.nameLike(contactLikeQuery.WILD, personNameSubstring,
+                            contactLikeQuery.WILD);
+  
+  queryExpect(contactLikeQuery, [personName]);
+}
+
+function test_query_contacts_by_name_nonmatch() {
+  let otherContact = outlierIdentityCollection.items[0].contact;
+  if (contactLikeQuery.test(otherContact)) {
+    do_throw("The string LIKE mechanism as applied to contacts does not work.");
+  }
+  next_test();
 }
 
 /* === identities === */
 
-function test_query_identities_by_kind_and_value() {
+var peoplesIdentityQuery;
+var peoplesIdentityCollection;
+function test_query_identities_for_peoples() {
+  peoplesIdentityQuery = Gloda.newQuery(Gloda.NOUN_IDENTITY);
+  peoplesIdentityQuery.kind("email");
+  let peopleAddrs = [nameAndAddr[1] for each (nameAndAddr in world.peoples)];
+  peoplesIdentityQuery.value.apply(peoplesIdentityQuery, peopleAddrs);
+  peoplesIdentityCollection = queryExpect(peoplesIdentityQuery, peopleAddrs);
 }
 
+var outlierIdentityQuery;
+var outlierIdentityCollection;
+function test_query_identities_for_outliers() {
+  outlierIdentityQuery = Gloda.newQuery(Gloda.NOUN_IDENTITY);
+  outlierIdentityQuery.kind("email");
+  let outlierAddrs = [world.outlierAuthor[1], world.outlierFriend[1]];
+  outlierIdentityQuery.value.apply(outlierIdentityQuery, outlierAddrs);
+  outlierIdentityCollection = queryExpect(outlierIdentityQuery, outlierAddrs);
+}
+
+function test_query_identities_by_kind_and_value_nonmatches() {
+  verify_nonMatches([peoplesIdentityQuery, outlierIdentityQuery],
+                    [peoplesIdentityCollection, outlierIdentityCollection]);
+  next_test();
+}
+
+
 /* ===== Driver ===== */
 
+var tests = [
+  setup_populate,
+  test_query_messages_by_conversation,
+  test_query_messages_by_conversation,
+  test_query_messages_by_conversation_nonmatches,
+  test_query_messages_by_folder,
+  test_query_messages_by_folder,
+  test_query_messages_by_folder_nonmatches,
+  // need to do the identity and contact lookups so we can have their results
+  //  for the other message-related queries
+  test_query_identities_for_peoples,
+  test_query_identities_for_outliers,
+  test_query_identities_by_kind_and_value_nonmatches,
+  // back to messages!
+  test_query_messages_by_identity_peoples,
+  test_query_messages_by_identity_outlier,
+  test_query_messages_by_identity_nonmatches,
+  test_query_messages_by_date,
+  test_query_messages_by_date_nonmatches,
+  test_query_contacts_by_name,
+  test_query_contacts_by_name_nonmatch
+];
+
 function run_test() {
+  // use mbox injection so we get multiple folders...
+  injectMessagesUsing(INJECT_MBOX);
+  glodaHelperRunTests(tests);
 }