indexing seems worky
authorAndrew Sutherland <asutherland@asutherland.org>
Thu, 12 Jun 2008 23:42:06 -0400
changeset 819 0108a2a0d06bad590b7f2ec58c0d12f71244e7ac
parent 818 d32b4cc6f46d1195ae66fd0cf8395c60b9259ca6
child 820 5254ff8c18e53d49a0ee4bd127299c010b999b8c
push idunknown
push userunknown
push dateunknown
indexing seems worky
chrome.manifest
content/overlay.js
install.rdf
locale/en-US/gloda.dtd
modules/datamodel.js
modules/datastore.js
modules/gloda.js
modules/indexer.js
modules/log4moz.js
--- a/chrome.manifest
+++ b/chrome.manifest
@@ -1,4 +1,5 @@
 content	gloda	content/
 locale	gloda	en-US	locale/en-US/
 skin	gloda	classic/1.0	skin/
 overlay	chrome://messenger/content/messenger.xul	chrome://gloda/content/thunderbirdOverlay.xul
+resource gloda ./
--- a/content/overlay.js
+++ b/content/overlay.js
@@ -29,23 +29,29 @@
  * use your version of this file under the terms of the MPL, indicate your
  * decision by deleting the provisions above and replace them with the notice
  * and other provisions required by the GPL or the LGPL. If you do not delete
  * the provisions above, a recipient may use your version of this file under
  * the terms of any one of the MPL, the GPL or the LGPL.
  * 
  * ***** END LICENSE BLOCK ***** */
 
+Components.utils.import("resource://gloda/modules/gloda.js");
+
+Components.utils.import("resource://gloda/modules/datastore.js");
+Components.utils.import("resource://gloda/modules/indexer.js");
+
 var gloda = {
   onLoad: function() {
     // initialization code
     this.initialized = true;
     this.strings = document.getElementById("gloda-strings");
   },
   onMenuItemCommand: function(e) {
+    GlodaIndexer.indexEverything();
     var promptService = Components.classes["@mozilla.org/embedcomp/prompt-service;1"]
                                   .getService(Components.interfaces.nsIPromptService);
     promptService.alert(window, this.strings.getString("helloMessageTitle"),
                                 this.strings.getString("helloMessage"));
   },
 
 };
 window.addEventListener("load", function(e) { gloda.onLoad(e); }, false);
--- a/install.rdf
+++ b/install.rdf
@@ -6,14 +6,14 @@
     <em:name>Global Database</em:name>
     <em:version>1.0</em:version>
     <em:creator>Andrew Sutherland</em:creator>
     <em:description>Adds a global database to Thunderbird</em:description>
     <em:optionsURL>chrome://gloda/content/options.xul</em:optionsURL>
     <em:targetApplication>
       <Description>
         <em:id>{3550f703-e582-4d05-9a08-453d09bdfdc6}</em:id> <!-- thunderbird -->
-        <em:minVersion>1.5</em:minVersion>
-        <em:maxVersion>2.0.0.*</em:maxVersion>
+        <em:minVersion>3.0a1</em:minVersion>
+        <em:maxVersion>3.0.*</em:maxVersion>
       </Description>
     </em:targetApplication>
   </Description>
 </RDF>
--- a/locale/en-US/gloda.dtd
+++ b/locale/en-US/gloda.dtd
@@ -1,1 +1,1 @@
-<!ENTITY gloda.label "Your localized menuitem">
+<!ENTITY gloda.label "Index Everything">
new file mode 100644
--- /dev/null
+++ b/modules/datamodel.js
@@ -0,0 +1,59 @@
+EXPORTED_SYMBOLS = ["GlodaConversation", "GlodaMessage"];
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
+
+Cu.import("resource://gloda/modules/log4moz.js");
+const LOG = Log4Moz.Service.getLogger("gloda.datamodel");
+
+function GlodaConversation (aID, aSubject, aOldestMessageDate,
+                            aNewestMessageDate) {
+  this._id = aID;
+  this._subject = aSubject;
+  this._oldestMessageDate = aOldestMessageDate;
+  this._newestMessageDate = aNewestMessageDate; 
+}
+
+GlodaConversation.prototype = {
+  get id() { return this._id; },
+  get subject() { return this._subject; },
+  get oldestMessageDate() { return this._oldestMessageDate; },
+  get newestMessageDate() { return this._newestMessageDate; },
+};
+
+
+function GlodaMessage (aID, aFolderID, aMessageKey, aConversationID,
+                       aConversation, aParentID, aHeaderMessageID,
+                       aBodySnippet) {
+  this._id = aID;
+  this._folderID = aFolderID;
+  this._messageKey = aMessageKey;
+  this._conversationID = aConversationID;
+  this._conversation = aConversation;
+  this._parentID = aParentID;
+  this._headerMessageID = aHeaderMessageID;
+  this._bodySnippet = aBodySnippet;
+}
+
+GlodaMessage.prototype = {
+  get id() { return this._id; },
+  get folderID() { return this._folderID; },
+  get messageKey() { return this._messageKey; },
+  get conversationID() { return this._conversationID; },
+  // conversation is special
+  get parentID() { return this._parentID; },
+  get headerMessageID() { return this._headerMessageID; },
+  get bodySnippet() { return this._boddySnippet; },
+  
+  get conversation() {
+  },
+  
+  /**
+   * Return the underlying nsIMsgDBHdr from the folder storage for this, or
+   *  null if the message does not exist for one reason or another.
+   */
+  get folderMessage() {
+  },
+};
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/modules/datastore.js
@@ -0,0 +1,372 @@
+EXPORTED_SYMBOLS = ["GlodaDatastore"];
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
+
+Cu.import("resource://gloda/modules/datamodel.js");
+
+let GlodaDatastore = {
+  _schemaVersion: 1,
+  _schema: {
+    tables: {
+      
+      // ----- Messages
+      folderLocations: {
+        columns: [
+          "id INTEGER PRIMARY KEY",
+          "folderURI TEXT",
+        ],
+      },
+      
+      conversations: {
+        columns: [
+          "id INTEGER PRIMARY KEY",
+          "subject TEXT",
+          "oldestMessageDate INTEGER",
+          "newestMessageDate INTEGER",
+        ],
+        
+        indices: {
+          subject: ['subject'],
+          oldestMessageDate: ['oldestMessageDate'],
+          newestMessageDate: ['newestMessageDate'],
+        }
+      },
+      
+      /**
+       * A message record correspond to an actual message stored in a folder
+       *  somewhere, or is a ghost record indicating a message that we know
+       *  should exist, but which we have not seen (and which we may never see).
+       *  We represent these ghost messages by storing NULL values in the
+       *  folderID and messageKey fields; this may need to change to other
+       *  sentinel values if this somehow impacts performance.
+       */
+      messages: {
+        columns: [
+          "id INTEGER PRIMARY KEY",
+          "folderID INTEGER REFERENCES folderLocations(id)",
+          "messageKey INTEGER",
+          "conversationID INTEGER NOT NULL REFERENCES conversations(id)",
+          "parentID INTEGER REFERENCES messages(id)",
+          "headerMessageID TEXT",
+          "bodySnippet TEXT",
+        ],
+        
+        indices: {
+          folderID: ['folderID'],
+          headerMessageID: ['headerMessageID'],
+          conversationID: ['conversationID'],
+        },
+      },
+      
+      // ----- Attributes
+      attributeDefinitions: {
+        columns: [
+          "id INTEGER PRIMARY KEY",
+          "attributeType TEXT",
+          "extensionName TEXT",
+          "name TEXT",
+          "parameter BLOB",
+        ],
+      },
+      
+      messageAttributes: {
+        columns: [
+          "conversationID INTEGER NOT NULL REFERENCES conversations(id)",
+          "messageID INTEGER NOT NULL REFERENCES messages(id)",
+          "attributeID INTEGER NOT NULL REFERENCES attributeDefinitions(id)",
+          "value NUMERIC",
+        ],
+        
+        indices: {
+          attribQuery: [
+            "attributeID", "value",
+            /* covering: */ "conversationID", "messageID"],
+          messageAttribFetch: [
+            "messageID",
+            /* covering: */ "conversationID", "messageID", "value"],
+          conversationAttribFetch: [
+            "conversationID",
+            /* covering: */ "messageID", "attributeID", "value"],
+        },
+      },
+    },
+  },
+  
+  _init: function glodaDBInit() {
+    // Get the path to our global database
+    var dirService = Cc["@mozilla.org/file/directory_service;1"].
+                     getService(Ci.nsIProperties);
+    var dbFile = dirService.get("ProfD", Ci.nsIFile);
+    dbFile.append("global-messages-db.sqlite");
+    
+    // Get the storage (sqlite) service
+    var dbService = Cc["@mozilla.org/storage/service;1"].
+                    getService(Ci.mozIStorageService);
+    
+    var dbConnection;
+    
+    // Create the file if it does not exist
+    if (!dbFile.exists()) {
+      dbConnection = this._createDB(dbService, dbFile);
+    }
+    // It does exist, but we (someday) might need to upgrade the schema
+    else {
+      // (Exceptions may be thrown if the database is corrupt)
+      try {
+        dbConnection = dbService.openDatabase(dbFile);
+      
+        if (dbConnection.schemaVersion != this._schemaVersion) {
+          this._migrate(dbConnection,
+                        dbConnection.schemaVersion, this._schemaVersion);
+        }
+      }
+      // Handle corrupt databases, other oddities
+      catch (ex) {
+        // TODO: handle them in the future.  let's die for now.
+        throw ex;
+      }
+    }
+    
+    this.dbConnection = dbConnection;
+  },
+  
+  _createDB: function glodaDBCreateDB(aDBService, aDBFile) {
+    var dbConnection = aDBService.openDatabase(aDBFile);
+    
+    dbConnection.beginTransaction();
+    try {
+      this._createSchema(dbConnection);
+      dbConnection.commitTransaction();
+    }
+    catch(ex) {
+      dbConnection.rollbackTransaction();
+      throw ex;
+    }
+    
+    return dbConnection;
+  },
+  
+  _createSchema: function glodaDBCreateSchema(aDBConnection) {
+    // -- For each table...
+    for (let tableName in this._schema.tables) {
+      let table = this._schema.tables[tableName];
+      
+      // - Create the table
+      aDBConnection.createTable(tableName, table.columns.join(", "));
+      
+      // - Create its indices
+      for (let indexName in table.indices) {
+        let indexColumns = table.indices[indexName];
+        
+        aDBConnection.executeSimpleSQL(
+          "CREATE INDEX " + indexName + " ON " + tableName +
+          "(" + indexColumns.join(", ") + ")"); 
+      }
+    }
+    
+    aDBConnection.schemaVersion = this._schemaVersion;  
+  },
+  
+  _migrate: function glodaDBMigrate(aDBConnection, aCurVersion, aNewVersion) {
+  },
+  
+  // cribbed from snowl
+  _createStatement: function glodaDBCreateStatement(aSQLString) {
+    let statement = null;
+    try {
+      statement = this.dbConnection.createStatement(aSQLString);
+    }
+    catch(ex) {
+       throw("error creating statement " + aSQLString + " - " +
+             this.dbConnection.lastError + ": " +
+             this.dbConnection.lastErrorString + " - " + ex);
+    }
+    
+    let wrappedStatement = Cc["@mozilla.org/storage/statement-wrapper;1"].
+                           createInstance(Ci.mozIStorageStatementWrapper);
+    wrappedStatement.initialize(statement);
+    return wrappedStatement;
+  },
+  
+  get _insertFolderLocationStatement() {
+    let statement = this._createStatement(
+      "INSERT INTO folderLocations (folderURI) VALUES (:folderURI)");
+    this.__defineGetter__("_insertFolderLocationStatement",
+      function() statement);
+    return this._insertFolderLocationStatement;
+  },
+  
+  get _selectFolderLocationByURIStatement() {
+    let statement = this._createStatement(
+      "SELECT id FROM folderLocations WHERE folderURI = :folderURI");
+    this.__defineGetter__("_selectFolderLocationByURIStatement",
+      function() statement);
+    return this._selectFolderLocationByURIStatement;
+  },
+  
+  _folderURIs: {},
+  
+  _mapFolderURI: function glodaDBMapFolderURI(aFolderURI) {
+    if (aFolderURI in this._folderURIs) {
+      return this._folderURIs[aFolderURI];
+    }
+    
+    var result;
+    this._selectFolderLocationByURIStatement.params.folderURI = aFolderURI;
+    if (this._selectFolderLocationByURIStatement.step()) {
+      result = this._selectFolderLocationByURIStatement.folderURI;
+    }
+    else {
+      this._insertFolderLocationStatement.params.folderURI = aFolderURI;
+      this._insertFolderLocationStatement.execute();
+      result = this.dbConnection.lastInsertRowID;
+    }
+    this._selectFolderLocationByURIStatement.reset();
+
+    this._folderURIs[aFolderURI] = result;
+    return result;
+  },
+  
+  // memoizing message statement creation
+  get _insertConversationStatement() {
+    let statement = this._createStatement(
+      "INSERT INTO conversations (subject, oldestMessageDate, \
+                                  newestMessageDate) \
+              VALUES (:subject, :oldestMessageDate, :newestMessageDate)");
+    this.__defineGetter__("_insertConversationStatement", function() statement);
+    return this._insertConversationStatement; 
+  }, 
+  
+  createConversation: function glodaDBCreateConversation(aSubject,
+        aOldestMessageDate, aNewestMessageDate) {
+    
+    let ics = this._insertConversationStatement;
+    ics.params.subject = aSubject;
+    ics.params.oldestMessageDate = aOldestMessageDate;
+    ics.params.newestMessageDate = aNewestMessageDate;
+        
+    ics.execute();
+    
+    return new GlodaConversation(this.dbConnection.lastInsertRowID,
+                                 aSubject, aOldestMessageDate,
+                                 aNewestMessageDate);
+  },
+  
+  
+  // memoizing message statement creation
+  get _insertMessageStatement() {
+    let statement = this._createStatement(
+      "INSERT INTO messages (folderID, messageKey, conversationID, parentID, \
+                             headerMessageID, bodySnippet) \
+              VALUES (:folderID, :messageKey, :conversationID, :parentID, \
+                      :headerMessageID, :bodySnippet)");
+    this.__defineGetter__("_insertMessageStatement", function() statement);
+    return this._insertMessageStatement; 
+  }, 
+  
+  createMessage: function glodaDBCreateMessage(aFolderURI, aMessageKey,
+                              aConversationID, aParentID, aHeaderMessageID,
+                              aBodySnippet) {
+    let folderID;
+    if (aFolderURI != null) {
+      folderID = this._mapFolderURI(aFolderURI);
+    }
+    else {
+      folderID = null;
+    }
+    
+    let ims = this._insertMessageStatement;
+    if (folderID != null)
+      ims.params.folderID = folderID;
+    if (aMessageKey != null)
+      ims.params.messageKey = aMessageKey;
+    ims.params.conversationID = aConversationID;
+    if (aParentID != null)
+      ims.params.parentID = aParentID;
+    ims.params.headerMessageID = aHeaderMessageID;
+    if (aBodySnippet != null)
+      ims.params.bodySnippet = aBodySnippet;
+
+
+    try {
+       ims.execute();
+    }
+    catch(ex) {
+       throw("error executing statement... " +
+             this.dbConnection.lastError + ": " +
+             this.dbConnection.lastErrorString + " - " + ex);
+    }
+    //ims.execute();
+    
+    return new GlodaMessage(this.dbConnection.lastInsertRowID, folderID,
+                            aMessageKey, aConversationID, aParentID,
+                            aHeaderMessageID, aBodySnippet);
+  },
+  
+  get _updateMessageStatement() {
+    let statement = this._createStatement(
+      "UPDATE messages SET folderID = :folderID, \
+                           messageKey = :messageKey, \
+                           conversationID = :conversationID, \
+                           parentID = :parentID, \
+                           headerMessageID = :headerMessageID, \
+                           bodySnippet = :boddySnippet \
+              WHERE id = :id");
+    this.__defineGetter__("_updateMessageStatement", function() statement);
+    return this._insertMessageStatement; 
+  }, 
+  
+  updateMessage: function glodaDBUpdateMessage(aMessage) {
+    let folderID = this._mapFolderURI(aFolderURI);
+    
+    let ums = this._updateMessageStatement;
+    ums.params.id = aMessage.id
+    ums.params.folderID = aMessage.folderID
+    ums.params.messageKey = aMessage.messageKey;
+    ums.params.conversationID = aMessage.conversationID;
+    ums.params.parentID = aMessage.parentID;
+    ums.params.headerMessageID = aMessage.headerMessageID;
+    ums.params.bodySnippet = aMessage.bodySnippet;
+    
+    ums.execute();
+  },
+  
+  _messageFromRow: function glodaDBMessageFromRow(aRow) {
+    return new GlodaMessage(aRow["id"], aRow["folderID"], aRow["messageKey"],
+                            aRow["conversationID"], aRow["parentID"],
+                            aRow["headerMessageID"], aRow["bodySnippet"]);
+  },
+  
+  getMessagesByMessageID: function glodaDBGetMessagesByMessageID(aMessageIDs) {
+    let msgIDToIndex = {};
+    let results = [];
+    for (let iID=0; iID < aMessageIDs.length; ++iID) {
+      let msgID = aMessageIDs[iID];
+      results.push(null);
+      msgIDToIndex[msgID] = iID;
+    } 
+
+    // Unfortunately, IN doesn't work with statement binding mechanisms, and
+    //  a chain of ORed tests really can't be bound unless we create one per
+    //  value of N (seems silly).
+    let quotedIDs = ["'" + msgID.replace("'", "\\'", "g") + "'" for each
+                     (msgID in aMessageIDs)]
+    let sqlString = "SELECT * FROM messages WHERE headerMessageID IN (" +
+                    quotedIDs + ")";
+    let statement = this._createStatement(sqlString);
+    
+    while (statement.step()) {
+      results[msgIDToIndex[statement.row["headerMessageID"]]] =
+        this._messageFromRow(statement.row);
+    }
+    statement.reset();
+    
+    return results;
+  },
+  
+};
+
+GlodaDatastore._init();
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/modules/gloda.js
@@ -0,0 +1,35 @@
+EXPORTED_SYMBOLS = ['Gloda'];
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
+
+Cu.import("resource://gloda/modules/log4moz.js");
+
+let Gloda = {
+  _init: function glodaNSInit() {
+    this._initLogging();
+  },
+  
+  _log: null,
+  _initLogging: function glodaNSInitLogging() {
+    let formatter = new Log4Moz.BasicFormatter();
+    let root = Log4Moz.Service.rootLogger;
+    root.level = Log4Moz.Level.Debug;
+
+    let capp = new Log4Moz.ConsoleAppender(formatter);
+    capp.level = Log4Moz.Level.Warn;
+    root.addAppender(capp);
+
+    let dapp = new Log4Moz.DumpAppender(formatter);
+    dapp.level = Log4Moz.Level.All;
+    root.addAppender(dapp);
+    
+    this._log = Log4Moz.Service.getLogger("Gloda.NS");
+    this._log.info("Logging Initialized");
+  },
+  
+};
+
+Gloda._init();
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/modules/indexer.js
@@ -0,0 +1,241 @@
+EXPORTED_SYMBOLS = ['GlodaIndexer'];
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
+
+Cu.import("resource://gloda/modules/log4moz.js");
+
+Cu.import("resource://gloda/modules/datastore.js");
+
+function range(begin, end) {
+  for (let i = begin; i < end; ++i) {
+    yield i;
+  }
+}
+
+// FROM STEEL
+/**
+ * This function will take a variety of xpcom iterators designed for c++ and turn
+ * them into a nice JavaScript style object that can be iterated using for...in
+ *
+ * Currently, we support the following types of xpcom iterators:
+ *   nsISupportsArray
+ *   nsIEnumerator
+ *   nsISimpleEnumerator
+ *
+ *   @param aEnum  the enumerator to convert
+ *   @param aIface (optional) an interface to QI each object to prior to returning
+ *
+ *   @note This does *not* return an Array object.  It returns an object that can
+ *         be use in for...in contexts only.  To create such an array, use
+ *         var array = [a for (a in fixIterator(xpcomEnumerator))];
+ */
+function fixIterator(aEnum, aIface) {
+  let face = aIface || Ci.nsISupports;
+  // Try to QI our object to each of the known iterator types.  If the QI does
+  // not throw, assign our iteration function
+  try {
+    aEnum.QueryInterface(Ci.nsISupportsArray);
+    let iter = function() {
+      let count = aEnum.Count();
+      for (let i = 0; i < count; i++)
+        yield aEnum.GetElementAt(i).QueryInterface(face);
+    }
+    return { __iterator__: iter };
+  } catch(ex) {}
+  
+  // Now try nsIEnumerator
+  try {
+    aEnum.QueryInterface(Ci.nsIEnumerator);
+    let done = false;
+    let iter = function() {
+      while (!done) {
+        try {
+          //rets.push(aEnum.currentItem().QueryInterface(face));
+          yield aEnum.currentItem().QueryInterface(face);
+          aEnum.next();
+        } catch(ex) {
+          done = true;
+        }
+      }
+    };
+
+    return { __iterator__: iter };
+  } catch(ex) {}
+  
+  // how about nsISimpleEnumerator? this one is nice and simple
+  try {
+    aEnum.QueryInterface(Ci.nsISimpleEnumerator);
+    let iter = function () {
+      while (aEnum.hasMoreElements())
+        yield aEnum.getNext().QueryInterface(face);
+    }
+    return { __iterator__: iter };
+  } catch(ex) {}
+}
+
+let GlodaIndexer = {
+  _datastore: GlodaDatastore,
+  _log: Log4Moz.Service.getLogger("gloda.indexer"),
+
+  indexEverything: function glodaIndexEverything() {
+  
+    this._log.info("Indexing Everything");
+    let msgAccountManager = Cc["@mozilla.org/messenger/account-manager;1"].
+                            getService(Ci.nsIMsgAccountManager);
+    
+    let blah = [this.indexAccount(account) for each
+                (account in fixIterator(msgAccountManager.accounts,
+                                        Ci.nsIMsgAccount))];
+
+    this._log.info("Indexing Everything");
+  },
+
+  indexAccount: function glodaIndexAccount(aAccount) {
+    this._log.info(">>> Indexing Account: " + aAccount.key);
+    
+    let rootFolder = aAccount.incomingServer.rootFolder;
+    if (rootFolder instanceof Ci.nsIMsgFolder) {
+      let blah = [this.indexFolder(folder) for each
+                  (folder in fixIterator(rootFolder.subFolders, Ci.nsIMsgFolder))];
+    }
+    else {
+      this._log.info("      Skipping Account, root folder not nsIMsgFolder");
+    }
+  
+    this._log.info("<<< Indexing Account: " + aAccount.key);
+  },
+
+  indexFolder: function glodaIndexFolder(aFolder) {
+    this._log.info("   >>> Indexing Folder: " + aFolder.prettiestName);
+    for each (let msgHdr in fixIterator(aFolder.getMessages(null),
+                                        Ci.nsIMsgDBHdr)) {
+      this._log.info("      >>> Indexing Message: " + msgHdr.messageId +
+                     " (" + msgHdr.subject + ")");
+      this.indexMessage(msgHdr);
+      this._log.info("      <<< Indexing Message: " + msgHdr.messageId);
+    }
+    this._log.info("   <<< Indexing Folder: " + aFolder.prettiestName);
+  },
+  
+  _mimeConverter: null,
+  _deMime: function glodaIndexDeMime(aString) {
+    if (this._mimeConverter == null) {
+      this._mimeConverter = Cc["@mozilla.org/messenger/mimeconverter;1"].
+                            getService(Ci.nsIMimeConverter);
+    }
+    
+    return this._mimeConverter.decodeMimeHeader(aString, null, false, true);
+  },
+  
+  /**
+   * Attempt to extract the original subject from a message.  For replies, this
+   *  means either taking off the 're[#]:' (or variant, including other language
+   *  variants), or in a Microsoft specific-ism, from the Thread-Topic header.
+   *
+   * Ideally, we would just be able to call NS_MsgStripRE to do the bulk of the
+   *  work for us, especially since the subject may be encoded.
+   */
+  _extractOriginalSubject: function glodaIndexExtractOriginalSubject(aMsgHdr) {
+    // mailnews.localizedRe contains a comma-delimited list of alternate
+    //  prefixes.
+    // NS_MsgStripRE does this, and bug 139317 proposes adding this to
+    //  nsIMimeConverter
+    
+    // HACK FIXME: for now, we just return the subject without any processing 
+    return this._deMime(aMsgHdr.subject);
+  },
+  
+  indexMessage: function glodaIndexMessage(aMsgHdr) {
+  
+    // -- Find/create the conversation the message belongs to.
+    // Our invariant is that all messages that exist in the database belong to
+    //  a conversation.
+    
+    // - See if any of the ancestors exist and have a conversationID...
+    // (references are ordered from old [0] to new [n-1])
+    let references = [aMsgHdr.getStringReference(i) for each
+                      (i in range(0, aMsgHdr.numReferences))];
+    // also see if we already know about the message...
+    references.push(aMsgHdr.messageId);
+    // (ancestors have a direct correspondence to the message id)
+    let ancestors = this._datastore.getMessagesByMessageID(references);
+    // pull our current message lookup results off
+    references.pop();
+    let curMsg = ancestors.pop();
+    
+    if (curMsg != null) {
+      this._log.warn("Attempting to re-index message: " + aMsgHdr.messageId
+                        + " (" + aMsgHdr.subject + ")");
+      return; 
+    }
+    
+    let conversationID = null;
+    
+    // (walk from closest to furthest ancestor)
+    for (let iAncestor=ancestors.length-1; iAncestor >= 0; --iAncestor) {
+      let ancestor = ancestors[iAncestor];
+      
+      if (ancestor != null) { // ancestor.conversationID cannot be null
+        if (conversationID == null)
+          conversationID = ancestor.conversationID;
+        else if (conversationID != ancestor.conversationID)
+          this._log.error("Inconsistency in conversations invariant on " +
+                          ancestor.messageID + ".  It has conv id " +
+                          ancestor.conversationID + " but expected " + 
+                          conversationID);
+      }
+    }
+    
+    let conversation = null;
+    if (conversationID == null) {
+      // (the create method could issue the id, making the call return
+      //  without waiting for the database...)
+      conversation = this._datastore.createConversation(
+          this._extractOriginalSubject(aMsgHdr), null, null);
+      conversationID = conversation.id;
+    }
+    
+    // Walk from furthest to closest ancestor, creating the ancestors that don't
+    //  exist, and updating any to have correct parentID's if they don't have
+    //  one.  (This is possible if previous messages that were consumed in this
+    //  thread only had an in-reply-to or for some reason did not otherwise
+    //  provide the full references chain.)
+    let lastAncestorId = null;
+    for (let iAncestor=0; iAncestor < ancestors.length; ++iAncestor) {
+      let ancestor = ancestors[iAncestor];
+      
+      if (ancestor == null) {
+        this._log.debug("creating message with: null, " + conversationID +
+                        ", " + lastAncestorId + ", " + references[iAncestor] +
+                        ", null.");
+        ancestor = this._datastore.createMessage(null, null, // no folder loc
+                                                 conversationID,
+                                                 lastAncestorId,
+                                                 references[iAncestor],
+                                                 null); // no snippet
+        ancestors[iAncestor] = ancestor;
+      }
+      else if (ancestor.parentID == null) {
+        ancestor.parentID = lastAncestorId;
+        this._datastore.updateMessage(ancestor);
+      }
+      
+      lastAncestorId = ancestor.id;
+    }
+    // now all our ancestors exist, though they may be ghost-like...
+    
+    this._log.debug("creating message with: " + aMsgHdr.folder.URI +
+                    ", " + conversationID +
+                    ", " + lastAncestorId + ", " + aMsgHdr.messageId +
+                    ", null.");
+    curMsg = this._datastore.createMessage(aMsgHdr.folder.URI,
+                                           aMsgHdr.messageKey,                
+                                           conversationID,
+                                           lastAncestorId,
+                                           aMsgHdr.messageId,
+                                           null); // no snippet
+  },
+};
new file mode 100644
--- /dev/null
+++ b/modules/log4moz.js
@@ -0,0 +1,489 @@
+/* ***** BEGIN LICENSE BLOCK *****
+ * Version: MPL 1.1/GPL 2.0/LGPL 2.1
+ *
+ * The contents of this file are subject to the Mozilla Public License Version
+ * 1.1 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the License is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
+ * for the specific language governing rights and limitations under the
+ * License.
+ *
+ * The Original Code is log4moz
+ *
+ * The Initial Developer of the Original Code is
+ * Michael Johnston
+ * Portions created by the Initial Developer are Copyright (C) 2006
+ * the Initial Developer. All Rights Reserved.
+ *
+ * Contributor(s):
+ * Michael Johnston <special.michael@gmail.com>
+ * Dan Mills <thunder@mozilla.com>
+ *
+ * Alternatively, the contents of this file may be used under the terms of
+ * either the GNU General Public License Version 2 or later (the "GPL"), or
+ * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
+ * in which case the provisions of the GPL or the LGPL are applicable instead
+ * of those above. If you wish to allow use of your version of this file only
+ * under the terms of either the GPL or the LGPL, and not to allow others to
+ * use your version of this file under the terms of the MPL, indicate your
+ * decision by deleting the provisions above and replace them with the notice
+ * and other provisions required by the GPL or the LGPL. If you do not delete
+ * the provisions above, a recipient may use your version of this file under
+ * the terms of any one of the MPL, the GPL or the LGPL.
+ *
+ * ***** END LICENSE BLOCK ***** */
+
+const EXPORTED_SYMBOLS = ['Log4Moz'];
+
+const Cc = Components.classes;
+const Ci = Components.interfaces;
+const Cr = Components.results;
+const Cu = Components.utils;
+
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
+
+const MODE_RDONLY   = 0x01;
+const MODE_WRONLY   = 0x02;
+const MODE_CREATE   = 0x08;
+const MODE_APPEND   = 0x10;
+const MODE_TRUNCATE = 0x20;
+
+const PERMS_FILE      = 0644;
+const PERMS_DIRECTORY = 0755;
+
+const ONE_BYTE = 1;
+const ONE_KILOBYTE = 1024 * ONE_BYTE;
+const ONE_MEGABYTE = 1024 * ONE_KILOBYTE;
+
+let Log4Moz = {
+  Level: {
+    Fatal:  70,
+    Error:  60,
+    Warn:   50,
+    Info:   40,
+    Config: 30,
+    Debug:  20,
+    Trace:  10,
+    All:    0,
+    Desc: {
+      70: "FATAL",
+      60: "ERROR",
+      50: "WARN",
+      40: "INFO",
+      30: "CONFIG",
+      20: "DEBUG",
+      10: "TRACE",
+      0:  "ALL"
+    }
+  },
+
+  get Service() {
+    delete Log4Moz.Service;
+    Log4Moz.Service = new Log4MozService();
+    return Log4Moz.Service;
+  },
+
+  get Formatter() { return Formatter; },
+  get BasicFormatter() { return BasicFormatter; },
+  get Appender() { return Appender; },
+  get DumpAppender() { return DumpAppender; },
+  get ConsoleAppender() { return ConsoleAppender; },
+  get FileAppender() { return FileAppender; },
+  get RotatingFileAppender() { return RotatingFileAppender; }
+};
+
+
+/*
+ * LogMessage
+ * Encapsulates a single log event's data
+ */
+function LogMessage(loggerName, level, message){
+  this.loggerName = loggerName;
+  this.message = message;
+  this.level = level;
+  this.time = Date.now();
+}
+LogMessage.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports]),
+
+  get levelDesc() {
+    if (this.level in Log4Moz.Level.Desc)
+      return Log4Moz.Level.Desc[this.level];
+    return "UNKNOWN";
+  },
+
+  toString: function LogMsg_toString(){
+    return "LogMessage [" + this._date + " " + this.level + " " +
+      this.message + "]";
+  }
+};
+
+/*
+ * Logger
+ * Hierarchical version.  Logs to all appenders, assigned or inherited
+ */
+
+function Logger(name, repository) {
+  this._name = name;
+  this._repository = repository;
+  this._appenders = [];
+}
+Logger.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports]),
+
+  parent: null,
+
+  _level: null,
+  get level() {
+    if (this._level != null)
+      return this._level;
+    if (this.parent)
+      return this.parent.level;
+    dump("log4moz warning: root logger configuration error: no level defined\n");
+    return Log4Moz.Level.All;
+  },
+  set level(level) {
+    this._level = level;
+  },
+
+  _appenders: null,
+  get appenders() {
+    if (!this.parent)
+      return this._appenders;
+    return this._appenders.concat(this.parent.appenders);
+  },
+
+  addAppender: function Logger_addAppender(appender) {
+    for (let i = 0; i < this._appenders.length; i++) {
+      if (this._appenders[i] == appender)
+        return;
+    }
+    this._appenders.push(appender);
+  },
+
+  log: function Logger_log(message) {
+    if (this.level > message.level)
+      return;
+    let appenders = this.appenders;
+    for (let i = 0; i < appenders.length; i++){
+      appenders[i].append(message);
+    }
+  },
+
+  fatal: function Logger_fatal(string) {
+    this.log(new LogMessage(this._name, Log4Moz.Level.Fatal, string));
+  },
+  error: function Logger_error(string) {
+    this.log(new LogMessage(this._name, Log4Moz.Level.Error, string));
+  },
+  warn: function Logger_warn(string) {
+    this.log(new LogMessage(this._name, Log4Moz.Level.Warn, string));
+  },
+  info: function Logger_info(string) {
+    this.log(new LogMessage(this._name, Log4Moz.Level.Info, string));
+  },
+  config: function Logger_config(string) {
+    this.log(new LogMessage(this._name, Log4Moz.Level.Config, string));
+  },
+  debug: function Logger_debug(string) {
+    this.log(new LogMessage(this._name, Log4Moz.Level.Debug, string));
+  },
+  trace: function Logger_trace(string) {
+    this.log(new LogMessage(this._name, Log4Moz.Level.Trace, string));
+  }
+};
+
+/*
+ * LoggerRepository
+ * Implements a hierarchy of Loggers
+ */
+
+function LoggerRepository() {}
+LoggerRepository.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports]),
+
+  _loggers: {},
+
+  _rootLogger: null,
+  get rootLogger() {
+    if (!this._rootLogger) {
+      this._rootLogger = new Logger("root", this);
+      this._rootLogger.level = Log4Moz.Level.All;
+    }
+    return this._rootLogger;
+  },
+  // FIXME: need to update all parent values if we do this
+  //set rootLogger(logger) {
+  //  this._rootLogger = logger;
+  //},
+
+  _updateParents: function LogRep__updateParents(name) {
+    let pieces = name.split('.');
+    let cur, parent;
+
+    // find the closest parent
+    for (let i = 0; i < pieces.length; i++) {
+      if (cur)
+        cur += '.' + pieces[i];
+      else
+        cur = pieces[i];
+      if (cur in this._loggers)
+        parent = cur;
+    }
+
+    // if they are the same it has no parent
+    if (parent == name)
+      this._loggers[name].parent = this.rootLogger;
+    else
+      this._loggers[name].parent = this._loggers[parent];
+
+    // trigger updates for any possible descendants of this logger
+    for (let logger in this._loggers) {
+      if (logger != name && name.indexOf(logger) == 0)
+        this._updateParents(logger);
+    }
+  },
+
+  getLogger: function LogRep_getLogger(name) {
+    if (!(name in this._loggers)) {
+      this._loggers[name] = new Logger(name, this);
+      this._updateParents(name);
+    }
+    return this._loggers[name];
+  }
+};
+
+/*
+ * Formatters
+ * These massage a LogMessage into whatever output is desired
+ * Only the BasicFormatter is currently implemented
+ */
+
+// Abstract formatter
+function Formatter() {}
+Formatter.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports]),
+  format: function Formatter_format(message) {}
+};
+
+// FIXME: should allow for formatting the whole string, not just the date
+function BasicFormatter(dateFormat) {
+  if (dateFormat)
+    this.dateFormat = dateFormat;
+}
+BasicFormatter.prototype = {
+  _dateFormat: null,
+
+  get dateFormat() {
+    if (!this._dateFormat)
+      this._dateFormat = "%Y-%m-%d %H:%M:%S";
+    return this._dateFormat;
+  },
+
+  set dateFormat(format) {
+    this._dateFormat = format;
+  },
+
+  format: function BF_format(message) {
+    let date = new Date(message.time);
+    return date.toLocaleFormat(this.dateFormat) + "\t" +
+      message.loggerName + "\t" + message.levelDesc + "\t" +
+      message.message + "\n";
+  }
+};
+BasicFormatter.prototype.__proto__ = new Formatter();
+
+/*
+ * Appenders
+ * These can be attached to Loggers to log to different places
+ * Simply subclass and override doAppend to implement a new one
+ */
+
+function Appender(formatter) {
+  this._name = "Appender";
+  this._formatter = formatter? formatter : new BasicFormatter();
+}
+Appender.prototype = {
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports]),
+
+  _level: Log4Moz.Level.All,
+  get level() { return this._level; },
+  set level(level) { this._level = level; },
+
+  append: function App_append(message) {
+    if(this._level <= message.level)
+      this.doAppend(this._formatter.format(message));
+  },
+  toString: function App_toString() {
+    return this._name + " [level=" + this._level +
+      ", formatter=" + this._formatter + "]";
+  },
+  doAppend: function App_doAppend(message) {}
+};
+
+/*
+ * DumpAppender
+ * Logs to standard out
+ */
+
+function DumpAppender(formatter) {
+  this._name = "DumpAppender";
+  this._formatter = formatter;
+}
+DumpAppender.prototype = {
+  doAppend: function DApp_doAppend(message) {
+    dump(message);
+  }
+};
+DumpAppender.prototype.__proto__ = new Appender();
+
+/*
+ * ConsoleAppender
+ * Logs to the javascript console
+ */
+
+function ConsoleAppender(formatter) {
+  this._name = "ConsoleAppender";
+  this._formatter = formatter;
+}
+ConsoleAppender.prototype = {
+  doAppend: function CApp_doAppend(message) {
+    if (message.level > Log4Moz.Level.Warn) {
+      Cu.reportError(message);
+      return;
+    }
+    Cc["@mozilla.org/consoleservice;1"].
+      getService(Ci.nsIConsoleService).logStringMessage(message);
+  }
+};
+ConsoleAppender.prototype.__proto__ = new Appender();
+
+/*
+ * FileAppender
+ * Logs to a file
+ */
+
+function FileAppender(file, formatter) {
+  this._name = "FileAppender";
+  this._file = file; // nsIFile
+  this._formatter = formatter;
+}
+FileAppender.prototype = {
+  __fos: null,
+  get _fos() {
+    if (!this.__fos)
+      this.openStream();
+    return this.__fos;
+  },
+
+  openStream: function FApp_openStream() {
+    this.__fos = Cc["@mozilla.org/network/file-output-stream;1"].
+      createInstance(Ci.nsIFileOutputStream);
+    let flags = MODE_WRONLY | MODE_CREATE | MODE_APPEND;
+    this.__fos.init(this._file, flags, PERMS_FILE, 0);
+  },
+
+  closeStream: function FApp_closeStream() {
+    if (!this.__fos)
+      return;
+    try {
+      this.__fos.close();
+      this.__fos = null;
+    } catch(e) {
+      dump("Failed to close file output stream\n" + e);
+    }
+  },
+
+  doAppend: function FApp_doAppend(message) {
+    if (message === null || message.length <= 0)
+      return;
+    try {
+      this._fos().write(message, message.length);
+    } catch(e) {
+      dump("Error writing file:\n" + e);
+    }
+  },
+
+  clear: function FApp_clear() {
+    this.closeStream();
+    this._file.remove(false);
+  }
+};
+FileAppender.prototype.__proto__ = new Appender();
+
+/*
+ * RotatingFileAppender
+ * Similar to FileAppender, but rotates logs when they become too large
+ */
+
+function RotatingFileAppender(file, formatter, maxSize, maxBackups) {
+  if (maxSize === undefined)
+    maxSize = ONE_MEGABYTE * 2;
+
+  if (maxBackups === undefined)
+    maxBackups = 0;
+
+  this._name = "RotatingFileAppender";
+  this._file = file; // nsIFile
+  this._formatter = formatter;
+  this._maxSize = maxSize;
+  this._maxBackups = maxBackups;
+}
+RotatingFileAppender.prototype = {
+  doAppend: function RFApp_doAppend(message) {
+    if (message === null || message.length <= 0)
+      return;
+    try {
+      this.rotateLogs();
+      this._fos.write(message, message.length);
+    } catch(e) {
+      dump("Error writing file:\n" + e);
+    }
+  },
+  rotateLogs: function RFApp_rotateLogs() {
+    if(this._file.exists() &&
+       this._file.fileSize < this._maxSize)
+      return;
+
+    this.closeStream();
+
+    for (let i = this.maxBackups - 1; i > 0; i--){
+      let backup = this._file.parent.clone();
+      backup.append(this._file.leafName + "." + i);
+      if (backup.exists())
+        backup.moveTo(this._file.parent, this._file.leafName + "." + (i + 1));
+    }
+
+    let cur = this._file.clone();
+    if (cur.exists())
+      cur.moveTo(cur.parent, cur.leafName + ".1");
+
+    // Note: this._file still points to the same file
+  }
+};
+RotatingFileAppender.prototype.__proto__ = new FileAppender();
+
+/*
+ * LoggingService
+ */
+
+function Log4MozService() {
+  this._repository = new LoggerRepository();
+}
+Log4MozService.prototype = {
+  //classDescription: "Log4moz Logging Service",
+  //contractID: "@mozilla.org/log4moz/service;1",
+  //classID: Components.ID("{a60e50d7-90b8-4a12-ad0c-79e6a1896978}"),
+  QueryInterface: XPCOMUtils.generateQI([Ci.nsISupports]),
+
+  get rootLogger() {
+    return this._repository.rootLogger;
+  },
+
+  getLogger: function LogSvc_getLogger(name) {
+    return this._repository.getLogger(name);
+  }
+};