Bug 782325 - Give Gloda a unique ID each time the index is regenerated, and ensure IM conversations are reindexed when the gloda database is reset. r=asuth, a=Standard8.
authorMike Conley <mconley@mozilla.com>
Mon, 20 Aug 2012 10:13:34 -0400
changeset 12551 24698041be04e99471b2ecab3411e8607a10f91e
parent 12546 36e2012538ac374ccbe1068746d39037d34b07dc
child 12552 0df4bccfc5f3c00cea38489e7e46fd84424310ee
push id641
push usermconley@mozilla.com
push dateMon, 20 Aug 2012 14:16:19 +0000
treeherdercomm-beta@0df4bccfc5f3 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersasuth, Standard8
bugs782325
Bug 782325 - Give Gloda a unique ID each time the index is regenerated, and ensure IM conversations are reindexed when the gloda database is reset. r=asuth, a=Standard8.
mail/components/im/modules/index_im.js
mailnews/db/gloda/modules/datastore.js
mailnews/db/gloda/modules/gloda.js
mailnews/db/gloda/test/unit/test_corrupt_database.js
--- a/mail/components/im/modules/index_im.js
+++ b/mail/components/im/modules/index_im.js
@@ -245,29 +245,33 @@ var GlodaIMIndexer = {
   _knownFiles: {},
   _cacheSaveTimer: null,
   _scheduleCacheSave: function() {
     if (this._cacheSaveTimer)
       return;
     this._cacheSaveTimer = setTimeout(this._saveCacheNow, 5000);
   },
   _saveCacheNow: function() {
-    let data = JSON.stringify(GlodaIMIndexer._knownFiles);
+    let data = {
+      knownFiles: GlodaIMIndexer._knownFiles,
+      datastoreID: Gloda.datastoreID,
+    };
+
     let file = FileUtils.getFile("ProfD", ["logs", kCacheFileName]);
     let ostream = FileUtils.openSafeFileOutputStream(file);
 
     // Obtain a converter to convert our data to a UTF-8 encoded input stream.
     let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].createInstance(Ci.nsIScriptableUnicodeConverter);
     converter.charset = "UTF-8";
 
     // Asynchronously copy the data to the file.
-    let istream = converter.convertToInputStream(data);
+    let istream = converter.convertToInputStream(JSON.stringify(data));
     NetUtil.asyncCopy(istream, ostream, function(rc) {
       if (!Components.isSuccessCode(rc)) {
-        dump("Failed to write cache file\n");
+        Cu.reportError("Failed to write cache file");
       }
     });
   },
 
   _knownConversations: {},
 
   _scheduleIndexingJob: function(aConversation) {
     let convId = aConversation.id;
@@ -456,17 +460,30 @@ var GlodaIMIndexer = {
     cacheFile.append(kCacheFileName);
     if (cacheFile.exists()) {
       const PR_RDONLY = 0x01;
       let fis = new FileInputStream(cacheFile, PR_RDONLY, 0444,
                                     Ci.nsIFileInputStream.CLOSE_ON_EOF);
       let sis = new ScriptableInputStream(fis);
       let text = sis.read(sis.available());
       sis.close();
-      this._knownFiles = JSON.parse(text);
+
+      let data = JSON.parse(text);
+
+      // Check to see if the Gloda datastore ID matches the one that we saved
+      // in the cache. If so, we can trust it. If not, that means that the
+      // cache is likely invalid now, so we ignore it (and eventually
+      // overwrite it).
+      if ("datastoreID" in data &&
+          Gloda.datastoreID &&
+          data.datastoreID === Gloda.datastoreID) {
+        // Ok, the cache's datastoreID matches the one we expected, so it's
+        // still valid.
+        this._knownFiles = data.knownFiles;
+      }
     }
 
     let children = dir.directoryEntries;
     while (children.hasMoreElements()) {
       let proto = children.getNext().QueryInterface(Ci.nsIFile);
       if (!proto.isDirectory())
         continue;
       let protoName = proto.leafName;
--- a/mailnews/db/gloda/modules/datastore.js
+++ b/mailnews/db/gloda/modules/datastore.js
@@ -996,16 +996,22 @@ var GlodaDatastore = {
 
   /**
    * Our "mailnews.database.global.datastore." preferences branch for debug
    * notification handling.  We register as an observer against this.
    */
   _prefBranch: null,
 
   /**
+   * The unique ID assigned to an index when it has been built. This value
+   * changes once the index has been rebuilt.
+   */
+  _datastoreID: null,
+
+  /**
    * Initialize logging, create the database if it doesn't exist, "upgrade" it
    *  if it does and it's not up-to-date, fill our authoritative folder uri/id
    *  mapping.
    */
   _init: function gloda_ds_init(aNounIDToDef) {
     this._log = Log4Moz.repository.getLogger("gloda.datastore");
     this._log.debug("Beginning datastore initialization.");
 
@@ -1096,16 +1102,24 @@ var GlodaDatastore = {
             this._schemaVersion);
           dbConnection = this._migrate(dbService, dbFile,
                                        dbConnection,
                                        this._actualSchemaVersion,
                                        this._schemaVersion);
           this._log.debug("Migration call completed.");
         }
         // else: this database is juuust right.
+
+        // If we never had a datastore ID, make sure to create one now.
+        if (!this._prefBranch.prefHasUserValue("id")) {
+          this._datastoreID = this._generateDatastoreID();
+          this._prefBranch.setCharPref("id", this._datastoreID);
+        } else {
+          this._datastoreID = this._prefBranch.getCharPref("id");
+        }
       }
       // Handle corrupt databases, other oddities
       catch (ex) {
         if (ex.result == Cr.NS_ERROR_FILE_CORRUPTED) {
           this._log.warn("Database was corrupt, removing the old one.");
           dbFile.remove(false);
           this._log.warn("Removed old database, creating a new one.");
           dbConnection = this._createDB(dbService, dbFile);
@@ -1243,16 +1257,29 @@ var GlodaDatastore = {
       this._log.debug("Potentially expected exception during connection " +
                       "closure: " + ex);
     }
 
     this.asyncConnection = null;
     this.syncConnection = null;
   },
 
+  /**
+   * Generates and returns a UUID.
+   *
+   * @return a UUID as a string, ex: "c4dd0159-9287-480f-a648-a4613e147fdb"
+   */
+  _generateDatastoreID: function gloda_ds_generateDatastoreID() {
+    let uuidGen = Cc["@mozilla.org/uuid-generator;1"]
+                    .getService(Ci.nsIUUIDGenerator);
+    let uuid = uuidGen.generateUUID().toString();
+    // We snip off the { and } from each end of the UUID.
+    return uuid.substring(1, uuid.length - 2);
+  },
+
   _determineCachePages: function gloda_ds_determineCachePages(aDBConn) {
     try {
       // For the details of the computations, one should read
       //  nsNavHistory::InitDB. We're slightly diverging from them in the sense
       //  that we won't allow gloda to use insane amounts of memory cache, and
       //  we start with 1% instead of 6% like them.
       let pageStmt = aDBConn.createStatement("PRAGMA page_size");
       pageStmt.executeStep();
@@ -1300,16 +1327,22 @@ var GlodaDatastore = {
     //  not enabling that is much more comprehensive.  We can think about
     //  turning that on after we've seen how this reduces our corruption count.
     dbConnection.executeSimpleSQL("PRAGMA synchronous = FULL");
     // Register custom tokenizer to index all language text
     var tokenizer = Cc["@mozilla.org/messenger/fts3tokenizer;1"].
                       getService(Ci.nsIFts3Tokenizer);
     tokenizer.registerTokenizer(dbConnection);
 
+    // We're creating a new database, so let's generate a new ID for this
+    // version of the datastore. This way, indexers can know when the index
+    // has been rebuilt in the event that they need to rebuild dependent data.
+    this._datastoreID = this._generateDatastoreID();
+    this._prefBranch.setCharPref("id", this._datastoreID);
+
     dbConnection.beginTransaction();
     try {
       this._createSchema(dbConnection);
       dbConnection.commitTransaction();
     }
     catch(ex) {
       dbConnection.rollbackTransaction();
       throw ex;
--- a/mailnews/db/gloda/modules/gloda.js
+++ b/mailnews/db/gloda/modules/gloda.js
@@ -251,16 +251,27 @@ var Gloda = {
    * We are done with our task, and have a result that we are returning.  This
    *  should only be used by your callback handler's doneWithResult method.
    *  Ex: you are passed aCallbackHandle, and you do
    *  "yield aCallbackHandle.doneWithResult(myResult);".
    */
   kWorkDoneWithResult: 4,
 
   /**
+   * Callers should access the unique ID for the GlodaDatastore
+   * with this getter. If the GlodaDatastore has not been
+   * initialized, this value is null.
+   *
+   * @return a UUID as a string, ex: "c4dd0159-9287-480f-a648-a4613e147fdb"
+   */
+  get datastoreID() {
+    return GlodaDatastore._datastoreID;
+  },
+
+  /**
    * Lookup a gloda message from an nsIMsgDBHdr, with the result returned as a
    *  collection.  Keep in mind that the message may not be indexed, so you
    *  may end up with an empty collection.  (Also keep in mind that this query
    *  is asynchronous, so you will want your action-taking logic to be found
    *  in your listener's onQueryCompleted method; the result will not be in
    *  the collection when this method returns.)
    *
    * @param aMsgHdr The header of the message you want the gloda message for.
--- a/mailnews/db/gloda/test/unit/test_corrupt_database.js
+++ b/mailnews/db/gloda/test/unit/test_corrupt_database.js
@@ -23,16 +23,21 @@ const gPrefs = Cc["@mozilla.org/preferen
 // yes to indexing
 gPrefs.setBoolPref("mailnews.database.global.indexer.enabled", true);
 // no to a sweep we don't control
 gPrefs.setBoolPref("mailnews.database.global.indexer.perform_initial_sweep",
     false);
 // yes to debug output
 gPrefs.setBoolPref("mailnews.database.global.logging.dump", true);
 
+// We'll start with this datastore ID, and make sure it gets overwritten
+// when the index is rebuilt.
+const kDatastoreIDPref = "mailnews.database.global.datastore.id";
+const kOriginalDatastoreID = "47e4bad6-fedc-4931-bf3f-d2f4146ac63e";
+gPrefs.setCharPref(kDatastoreIDPref, kOriginalDatastoreID);
 
 // -- Add a logger listener that throws when we give it a warning/error.
 Components.utils.import("resource:///modules/gloda/log4moz.js");
 
 /**
  * Count the type of each severity level observed.
  */
 function CountingAppender() {
@@ -103,16 +108,29 @@ function test_corrupt_databases_get_repo
   // we expect 2 warnings
   do_check_eq(countingAppender.getCountForLevel(Log4Moz.Level.Warn), 2);
   // and no errors
   do_check_eq(countingAppender.getCountForLevel(Log4Moz.Level.Error), 0);
 
   // - make sure the datastore has an actual database
   Components.utils.import("resource:///modules/gloda/datastore.js");
 
+  // Make sure that the datastoreID was overwritten
+  do_check_neq(Gloda.datastoreID, kOriginalDatastoreID);
+  // And for good measure, make sure that the pref was also overwritten
+  let currentDatastoreID = gPrefs.getCharPref(kDatastoreIDPref);
+  do_check_neq(currentDatastoreID, kOriginalDatastoreID);
+  // We'll also ensure that the Gloda.datastoreID matches the one stashed
+  // in prefs...
+  do_check_eq(currentDatastoreID, Gloda.datastoreID);
+  // And finally, we'll make sure that the datastoreID is a string with length
+  // greater than 0.
+  do_check_eq(typeof(Gloda.datastoreID), "string");
+  do_check_true(Gloda.datastoreID.length > 0);
+
   if (!GlodaDatastore.asyncConnection)
     do_throw("No database connection suggests no database!");
 }
 
 var tests = [
   test_corrupt_databases_get_reported_and_blown_away,
 ];