Bug 699856 - Refactor nsSearchService.js to not use a database engine. r=gavin
authorDavid Rajchenbach-Teller <dteller@mozilla.com>
Tue, 13 Mar 2012 23:32:53 +0100
changeset 92296 7df9f3c93402a92372a53846dd50386c5683a706
parent 92295 a301b875ba1608487d3d7b5c6c49ecb22c65564e
child 92297 89f530a84865d14f09573ac6406775d347ce349f
push id886
push userlsblakk@mozilla.com
push dateMon, 04 Jun 2012 19:57:52 +0000
treeherdermozilla-beta@bbd8d5efd6d1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgavin
bugs699856
milestone14.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 699856 - Refactor nsSearchService.js to not use a database engine. r=gavin
toolkit/components/search/nsSearchService.js
toolkit/components/search/tests/xpcshell/data/chrome.manifest
toolkit/components/search/tests/xpcshell/data/engine.src
toolkit/components/search/tests/xpcshell/data/engine.xml
toolkit/components/search/tests/xpcshell/data/ico-size-16x16-png.ico
toolkit/components/search/tests/xpcshell/data/search-metadata.json
toolkit/components/search/tests/xpcshell/data/search.sqlite
toolkit/components/search/tests/xpcshell/head_search.js
toolkit/components/search/tests/xpcshell/test_645970.js
toolkit/components/search/tests/xpcshell/test_migratedb.js
toolkit/components/search/tests/xpcshell/test_nodb.js
toolkit/components/search/tests/xpcshell/test_nodb_pluschanges.js
toolkit/components/search/tests/xpcshell/xpcshell.ini
--- a/toolkit/components/search/nsSearchService.js
+++ b/toolkit/components/search/nsSearchService.js
@@ -38,16 +38,17 @@
 #
 # ***** END LICENSE BLOCK *****
 
 const Ci = Components.interfaces;
 const Cc = Components.classes;
 const Cr = Components.results;
 
 Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+Components.utils.import("resource://gre/modules/Services.jsm");
 
 const PERMS_FILE      = 0644;
 const PERMS_DIRECTORY = 0755;
 
 const MODE_RDONLY   = 0x01;
 const MODE_WRONLY   = 0x02;
 const MODE_CREATE   = 0x08;
 const MODE_APPEND   = 0x10;
@@ -71,16 +72,29 @@ const SEARCH_ENGINE_TOPIC        = "brow
 const QUIT_APPLICATION_TOPIC     = "quit-application";
 
 const SEARCH_ENGINE_REMOVED      = "engine-removed";
 const SEARCH_ENGINE_ADDED        = "engine-added";
 const SEARCH_ENGINE_CHANGED      = "engine-changed";
 const SEARCH_ENGINE_LOADED       = "engine-loaded";
 const SEARCH_ENGINE_CURRENT      = "engine-current";
 
+// The following constants are left undocumented in nsIBrowserSearchService.idl
+// For the moment, they are meant for testing/debugging purposes only.
+
+/**
+ * Topic used for events involving the service itself.
+ */
+const SEARCH_SERVICE_TOPIC       = "browser-search-service";
+
+/**
+ * Sent whenever metadata is fully written to disk.
+ */
+const SEARCH_SERVICE_METADATA_WRITTEN  = "write-metadata-to-disk-complete";
+
 const SEARCH_TYPE_MOZSEARCH      = Ci.nsISearchEngine.TYPE_MOZSEARCH;
 const SEARCH_TYPE_OPENSEARCH     = Ci.nsISearchEngine.TYPE_OPENSEARCH;
 const SEARCH_TYPE_SHERLOCK       = Ci.nsISearchEngine.TYPE_SHERLOCK;
 
 const SEARCH_DATA_XML            = Ci.nsISearchEngine.DATA_XML;
 const SEARCH_DATA_TEXT           = Ci.nsISearchEngine.DATA_TEXT;
 
 // File extensions for search plugin description files
@@ -213,16 +227,22 @@ function isUsefulLine(aLine) {
 });
 
 __defineGetter__("gPrefSvc", function() {
   delete this.gPrefSvc;
   return this.gPrefSvc = Cc["@mozilla.org/preferences-service;1"].
                          getService(Ci.nsIPrefBranch);
 });
 
+__defineGetter__("FileUtils", function() {
+  delete this.FileUtils;
+  Components.utils.import("resource://gre/modules/FileUtils.jsm");
+  return FileUtils;
+});
+
 __defineGetter__("NetUtil", function() {
   delete this.NetUtil;
   Components.utils.import("resource://gre/modules/NetUtil.jsm");
   return NetUtil;
 });
 
 __defineGetter__("gChromeReg", function() {
   delete this.gChromeReg;
@@ -304,16 +324,74 @@ function FAIL(message, resultCode) {
  * @throws resultCode
  */
 function ENSURE_WARN(assertion, message, resultCode) {
   NS_ASSERT(assertion, SEARCH_LOG_PREFIX + message);
   if (!assertion)
     throw Components.Exception(message, resultCode);
 }
 
+/**
+ * A delayed treatment that may be delayed even further.
+ *
+ * Use this for instance if you write data to a file and you expect
+ * that you may have to rewrite data very soon afterwards. With
+ * |Lazy|, the treatment is delayed by a few milliseconds and,
+ * should a new change to the data occur during this period,
+ * 1/ only the final version of the data is actually written;
+ * 2/ a further grace delay is added to take into account other
+ * changes.
+ *
+ * @constructor
+ * @param {Function} code The code to execute after the delay.
+ * @param {number=} delay An optional delay, in milliseconds.
+ */
+function Lazy(code, delay) {
+  LOG("Lazy: Creating a Lazy");
+  this._callback =
+    (function(){
+       code();
+       this._timer = null;
+     }).bind(this);
+  this._delay = delay || LAZY_SERIALIZE_DELAY;
+  this._timer = null;
+}
+Lazy.prototype = {
+  /**
+   * Start (or postpone) treatment.
+   */
+  go: function Lazy_go() {
+    LOG("Lazy_go: starting");
+    if (this._timer) {
+      LOG("Lazy_go: reusing active timer");
+      this._timer.delay = this._delay;
+    } else {
+      LOG("Lazy_go: creating timer");
+      this._timer = Cc["@mozilla.org/timer;1"].createInstance(Ci.nsITimer);
+      this._timer.
+        initWithCallback(this._callback,
+                         this._delay,
+                         Ci.nsITimer.TYPE_ONE_SHOT);
+    }
+  },
+  /**
+   * Perform any postponed treatment immediately.
+   */
+  flush: function Lazy_flush() {
+    LOG("Lazy_flush: starting");
+    if (!this._timer) {
+      return;
+    }
+    this._timer.cancel();
+    this._timer = null;
+    this._callback();
+  }
+};
+
+
 function loadListener(aChannel, aEngine, aCallback) {
   this._channel = aChannel;
   this._bytes = [];
   this._engine = aEngine;
   this._callback = aCallback;
 }
 loadListener.prototype = {
   _callback: null,
@@ -713,16 +791,22 @@ function getMozParamPref(prefName)
 let gEnginesLoaded = false;
 function notifyAction(aEngine, aVerb) {
   if (gEnginesLoaded) {
     LOG("NOTIFY: Engine: \"" + aEngine.name + "\"; Verb: \"" + aVerb + "\"");
     gObsSvc.notifyObservers(aEngine, SEARCH_ENGINE_TOPIC, aVerb);
   }
 }
 
+function  parseJsonFromStream(aInputStream) {
+  const json = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
+  const data = json.decodeFromStream(aInputStream, aInputStream.available());
+  return data;
+}
+
 /**
  * Simple object representing a name/value pair.
  */
 function QueryParameter(aName, aValue) {
   if (!aName || (aValue == null))
     FAIL("missing name or value for QueryParameter!");
 
   this.name = aName;
@@ -2571,16 +2655,17 @@ SearchService.prototype = {
           LOG("_buildCache: failure during asyncCopy: " + rv);
       });
     } catch (ex) {
       LOG("_buildCache: Could not write to cache file: " + ex);
     }
   },
 
   _loadEngines: function SRCH_SVC__loadEngines() {
+    LOG("_loadEngines: start");
     // See if we have a cache file so we don't have to parse a bunch of XML.
     let cache = {};
     let cacheEnabled = getBoolPref(BROWSER_SEARCH_PREF + "cache.enabled", true);
     if (cacheEnabled) {
       let cacheFile = getDir(NS_APP_USER_PROFILE_50_DIR);
       cacheFile.append("search.json");
       if (cacheFile.exists())
         cache = this._readCacheFile(cacheFile);
@@ -2628,18 +2713,21 @@ SearchService.prototype = {
 
       this._loadFromChromeURLs(chromeURIs);
 
       if (cacheEnabled)
         this._buildCache();
       return;
     }
 
+    LOG("_loadEngines: loading from cache directories");
     for each (let dir in cache.directories)
       this._loadEnginesFromCache(dir);
+
+    LOG("_loadEngines: done");
   },
 
   _readCacheFile: function SRCH_SVC__readCacheFile(aFile) {
     let stream = Cc["@mozilla.org/network/file-input-stream;1"].
                  createInstance(Ci.nsIFileInputStream);
     let json = Cc["@mozilla.org/dom/json;1"].createInstance(Ci.nsIJSON);
 
     try {
@@ -2805,17 +2893,17 @@ SearchService.prototype = {
         continue;
       }
 
       if (fileExtension == SHERLOCK_FILE_EXT) {
         if (isWritable) {
           try {
             this._convertSherlockFile(addedEngine, fileURL.fileBaseName);
           } catch (ex) {
-            LOG("_loadEnginesFromDir: Failed to convert: " + fileURL.path + "\n" + ex);
+            LOG("_loadEnginesFromDir: Failed to convert: " + fileURL.path + "\n" + ex + "\n" + ex.stack);
             // The engine couldn't be converted, mark it as read-only
             addedEngine._readOnly = true;
           }
         }
 
         // If the engine still doesn't have an icon, see if we can find one
         if (!addedEngine._iconURI) {
           var icon = this._findSherlockIcon(file, fileURL.fileBaseName);
@@ -2902,45 +2990,52 @@ SearchService.prototype = {
       names.forEach(function (n) uris.push(root + n + ".xml"));
     });
     
     return [chromeFiles, uris];
   },
 
   _saveSortedEngineList: function SRCH_SVC_saveSortedEngineList() {
     // We only need to write the prefs. if something has changed.
+    LOG("SRCH_SVC_saveSortedEngineList: starting");
     if (!this._needToSetOrderPrefs)
       return;
 
+    LOG("SRCH_SVC_saveSortedEngineList: something to do");
+
     // Set the useDB pref to indicate that from now on we should use the order
     // information stored in the database.
     gPrefSvc.setBoolPref(BROWSER_SEARCH_PREF + "useDBForOrder", true);
 
     var engines = this._getSortedEngines(true);
-    var values = [];
-    var names = [];
-
+
+    let instructions = [];
     for (var i = 0; i < engines.length; ++i) {
-      names[i] = "order";
-      values[i] = i + 1;
+      instructions.push(
+        {key: "order",
+         value: i+1,
+         engine: engines[i]
+        });
     }
 
-    engineMetadataService.setAttrs(engines, names, values);
+    engineMetadataService.setAttrs(instructions);
+    LOG("SRCH_SVC_saveSortedEngineList: done");
   },
 
   _buildSortedEngineList: function SRCH_SVC_buildSortedEngineList() {
     LOG("_buildSortedEngineList: building list");
     var addedEngines = { };
     this.__sortedEngines = [];
     var engine;
 
     // If the user has specified a custom engine order, read the order
     // information from the engineMetadataService instead of the default
     // prefs.
     if (getBoolPref(BROWSER_SEARCH_PREF + "useDBForOrder", false)) {
+      LOG("_buildSortedEngineList: using db for order");
       for each (engine in this._engines) {
         var orderNumber = engineMetadataService.getAttr(engine, "order");
 
         // Since the DB isn't regularly cleared, and engine files may disappear
         // without us knowing, we may already have an engine in this slot. If
         // that happens, we just skip it - it will be added later on as an
         // unsorted engine. This problem will sort itself out when we call
         // _saveSortedEngineList at shutdown.
@@ -3429,17 +3524,17 @@ SearchService.prototype = {
       case QUIT_APPLICATION_TOPIC:
         this._removeObservers();
         this._saveSortedEngineList();
         if (this._batchTimer) {
           // Flush to disk immediately
           this._batchTimer.cancel();
           this._buildCache();
         }
-        engineMetadataService.closeDB();
+        engineMetadataService.flush();
         break;
     }
   },
 
   // nsITimerCallback
   notify: function SRCH_SVC_notify(aTimer) {
     LOG("_notify: checking for updates");
 
@@ -3494,145 +3589,226 @@ SearchService.prototype = {
         aIID.equals(Ci.nsITimerCallback)        ||
         aIID.equals(Ci.nsISupports))
       return this;
     throw Cr.NS_ERROR_NO_INTERFACE;
   }
 };
 
 var engineMetadataService = {
-  get mDB() {
-    var engineDataTable = "id INTEGER PRIMARY KEY, engineid STRING, name STRING, value STRING";
-    var file = getDir(NS_APP_USER_PROFILE_50_DIR);
-    file.append("search.sqlite");
-    var dbService = Cc["@mozilla.org/storage/service;1"].
-                    getService(Ci.mozIStorageService);
-    var db;
-    try {
-      db = dbService.openDatabase(file);
-    } catch (ex) {
-      if (ex.result == 0x8052000b) { /* NS_ERROR_FILE_CORRUPTED */
-        // delete and try again
-        file.remove(false);
-        db = dbService.openDatabase(file);
-      } else {
-        throw ex;
+  /**
+   * @type {nsIFile|null} The file holding the metadata.
+   */
+  get _jsonFile() {
+    delete this._jsonFile;
+    return this._jsonFile = FileUtils.getFile(NS_APP_USER_PROFILE_50_DIR,
+                                              ["search-metadata.json"]);
+  },
+
+  /**
+   * Lazy getter for the file containing json data.
+   */
+  get _store() {
+    delete this._store;
+    return this._store = this._loadStore();
+  },
+
+  // Perform loading the first time |_store| is accessed.
+  _loadStore: function() {
+    let jsonFile = this._jsonFile;
+    if (!jsonFile.exists()) {
+      LOG("loadStore: search-metadata.json does not exist");
+
+      // First check to see whether there's an existing SQLite DB to migrate
+      let store = this._migrateOldDB();
+      if (store) {
+        // Commit the migrated store to disk immediately
+        LOG("Committing the migrated store to disk");
+        this._commit(store);
+        return store;
       }
+
+       // Migration failed, or this is a first-run - just use an empty store
+      return {};
     }
 
+    LOG("loadStore: attempting to load store from JSON file");
     try {
-      db.createTable("engine_data", engineDataTable);
-    } catch (ex) {
-      // Fails if the table already exists, which is fine
+      return parseJsonFromStream(NetUtil.newChannel(jsonFile).open());
+    } catch (x) {
+      LOG("loadStore failed to load file: "+x);
+      return {};
     }
-
-    delete this.mDB;
-    return this.mDB = db;
-  },
-
-  get mGetData() {
-    delete this.mGetData;
-    return this.mGetData = this.mDB.createStatement(
-      "SELECT value FROM engine_data WHERE engineid = :engineid AND name = :name");
-  },
-  get mDeleteData() {
-    delete this.mDeleteData;
-    return this.mDeleteData = this.mDB.createStatement(
-      "DELETE FROM engine_data WHERE engineid = :engineid AND name = :name");
-  },
-  get mInsertData() {
-    delete this.mInsertData;
-    return this.mInsertData = this.mDB.createStatement(
-      "INSERT INTO engine_data (engineid, name, value) " +
-      "VALUES (:engineid, :name, :value)");
   },
 
   getAttr: function epsGetAttr(engine, name) {
+    let record = this._store[engine._id];
+    if (!record) {
+      return null;
+    }
+
     // attr names must be lower case
-    name = name.toLowerCase();
-
-    var stmt = this.mGetData;
-    stmt.reset();
-    var pp = stmt.params;
-    pp.engineid = engine._id;
-    pp.name = name;
-
-    var value = null;
-    if (stmt.executeStep())
-      value = stmt.row.value;
-    stmt.reset();
-    return value;
+    return record[name.toLowerCase()];
   },
 
-  setAttr: function epsSetAttr(engine, name, value) {
+  _setAttr: function epsSetAttr(engine, name, value) {
     // attr names must be lower case
     name = name.toLowerCase();
-
-    this.mDB.beginTransaction();
-
-    var pp = this.mDeleteData.params;
-    pp.engineid = engine._id;
-    pp.name = name;
-    this.mDeleteData.executeStep();
-    this.mDeleteData.reset();
-
-    pp = this.mInsertData.params;
-    pp.engineid = engine._id;
-    pp.name = name;
-    pp.value = value;
-    this.mInsertData.executeStep();
-    this.mInsertData.reset();
-
-    this.mDB.commitTransaction();
+    let db = this._store;
+    let record = db[engine._id];
+    if (!record) {
+      record = db[engine._id] = {};
+    }
+    if (record[name] != value) {
+      record[name] = value;
+      return true;
+    }
+    return false;
+  },
+
+  /**
+   * Set one metadata attribute for an engine.
+   *
+   * If an actual change has taken place, the attribute is committed
+   * automatically (and lazily), using this._commit.
+   *
+   * @param {nsISearchEngine} engine The engine to update.
+   * @param {string} key The name of the attribute. Case-insensitive. In
+   * the current implementation, this _must not_ conflict with properties
+   * of |Object|.
+   * @param {*} value A value to store.
+   */
+  setAttr: function epsSetAttr(engine, key, value) {
+    if (this._setAttr(engine, key, value)) {
+      this._commit();
+    }
+  },
+
+  /**
+   * Bulk set metadata attributes for a number of engines.
+   *
+   * If actual changes have taken place, the store is committed
+   * automatically (and lazily), using this._commit.
+   *
+   * @param {Array.<{engine: nsISearchEngine, key: string, value: *}>} changes
+   * The list of changes to effect. See |setAttr| for the documentation of
+   * |engine|, |key|, |value|.
+   */
+  setAttrs: function epsSetAttrs(changes) {
+    let self = this;
+    let changed = false;
+    changes.forEach(function(change) {
+      changed |= self._setAttr(change.engine, change.key, change.value);
+    });
+    if (changed) {
+      this._commit();
+    }
+  },
+
+  /**
+   * Flush any waiting write.
+   */
+  flush: function epsFlush() {
+    if (this._lazyWriter) {
+      this._lazyWriter.flush();
+    }
   },
 
-  setAttrs: function epsSetAttrs(engines, names, values) {
-    this.mDB.beginTransaction();
-
-    for (var i = 0; i < engines.length; i++) {
-      // attr names must be lower case
-      var name = names[i].toLowerCase();
-
-      var pp = this.mDeleteData.params;
-      pp.engineid = engines[i]._id;
-      pp.name = names[i];
-      this.mDeleteData.executeStep();
-      this.mDeleteData.reset();
-
-      pp = this.mInsertData.params;
-      pp.engineid = engines[i]._id;
-      pp.name = names[i];
-      pp.value = values[i];
-      this.mInsertData.executeStep();
-      this.mInsertData.reset();
+  /**
+   * Migrate search.sqlite
+   *
+   * Notes:
+   * - we do not remove search.sqlite after migration, so as to allow
+   * downgrading and forensics;
+   */
+  _migrateOldDB: function SRCH_SVC_EMS_migrate() {
+    LOG("SRCH_SVC_EMS_migrate start");
+    let sqliteFile = FileUtils.getFile(NS_APP_USER_PROFILE_50_DIR,
+                                       ["search.sqlite"]);
+    if (!sqliteFile.exists()) {
+      LOG("SRCH_SVC_EMS_migrate search.sqlite does not exist");
+      return null;
+    }
+    let store = {};
+    try {
+      LOG("SRCH_SVC_EMS_migrate Migrating data from SQL");
+      const sqliteDb = Services.storage.openDatabase(sqliteFile);
+      const statement = sqliteDb.createStatement("SELECT * from engine_data");
+      while (statement.executeStep()) {
+        let row = statement.row;
+        let engine = row.engineid;
+        let name   = row.name;
+        let value  = row.value;
+        if (!store[engine]) {
+          store[engine] = {};
+        }
+        store[engine][name] = value;
+      }
+      statement.finalize();
+      sqliteDb.close();
+    } catch (ex) {
+      LOG("SRCH_SVC_EMS_migrate failed: " + ex);
+      return null;
     }
-
-    this.mDB.commitTransaction();
+    return store;
   },
 
-  deleteEngineData: function epsDelData(engine, name) {
-    // attr names must be lower case
-    name = name.toLowerCase();
-
-    var pp = this.mDeleteData.params;
-    pp.engineid = engine._id;
-    pp.name = name;
-    this.mDeleteData.executeStep();
-    this.mDeleteData.reset();
+  /**
+   * Commit changes to disk, asynchronously.
+   *
+   * Calls to this function are actually delayed by LAZY_SERIALIZE_DELAY
+   * (= 100ms). If the function is called again before the expiration of
+   * the delay, commits are merged and the function is again delayed by
+   * the same amount of time.
+   *
+   * @param aStore is an optional parameter specifying the object to serialize.
+   *               If not specified, this._store is used.
+   */
+  _commit: function epsCommit(aStore) {
+    LOG("epsCommit: start");
+
+    let store = aStore || this._store;
+    if (!store) {
+      LOG("epsCommit: nothing to do");
+      return;
+    }
+
+    if (!this._lazyWriter) {
+      LOG("epsCommit: initializing lazy writer");
+      let jsonFile = this._jsonFile;
+      function writeCommit() {
+        LOG("epsWriteCommit: start");
+        let ostream = FileUtils.
+          openSafeFileOutputStream(jsonFile,
+                                   MODE_WRONLY | MODE_CREATE | MODE_TRUNCATE);
+
+        // Obtain a converter to convert our data to a UTF-8 encoded input stream.
+        let converter = Cc["@mozilla.org/intl/scriptableunicodeconverter"].
+          createInstance(Ci.nsIScriptableUnicodeConverter);
+        converter.charset = "UTF-8";
+
+        let callback = function(result) {
+          if (Components.isSuccessCode(result)) {
+            gObsSvc.notifyObservers(null,
+                                    SEARCH_SERVICE_TOPIC,
+                                    SEARCH_SERVICE_METADATA_WRITTEN);
+          }
+          LOG("epsWriteCommit: done " + result);
+        };
+        // Asynchronously copy the data to the file.
+        let istream = converter.convertToInputStream(JSON.stringify(store));
+        NetUtil.asyncCopy(istream, ostream, callback);
+      }
+      this._lazyWriter = new Lazy(writeCommit);
+    }
+    LOG("epsCommit: (re)setting timer");
+    this._lazyWriter.go();
   },
-
-  closeDB: function epsCloseDB() {
-    ["mInsertData", "mDeleteData", "mGetData"].forEach(function(aStmt) {
-      if (Object.getOwnPropertyDescriptor(this, aStmt).value !== undefined)
-        this[aStmt].finalize();
-    }, this);
-    if (Object.getOwnPropertyDescriptor(this, "mDB").value !== undefined)
-      this.mDB.close();
-  }
-}
+  _lazyWriter: null,
+};
 
 const SEARCH_UPDATE_LOG_PREFIX = "*** Search update: ";
 
 /**
  * Outputs aText to the JavaScript console as well as to stdout, if the search
  * logging pref (browser.search.update.log) is set to true.
  */
 function ULOG(aText) {
--- a/toolkit/components/search/tests/xpcshell/data/chrome.manifest
+++ b/toolkit/components/search/tests/xpcshell/data/chrome.manifest
@@ -1,1 +1,3 @@
 locale testsearchplugin ar jar:jar:searchTest.jar!/chrome/searchTest.jar!/
+content testsearchplugin ./
+
new file mode 100644
--- /dev/null
+++ b/toolkit/components/search/tests/xpcshell/data/engine.src
@@ -0,0 +1,18 @@
+<SEARCH
+   version="1"
+   name="Sherlock test search engine"
+   description="A test search engine for testing sherlock"
+   method="GET"
+   action="http://getfirefox.com"
+   searchform="http://getfirefox.com"
+>
+
+<input name="q" user>
+
+</search>
+
+<BROWSER
+   update="http://getfirefox.com"
+   updateIcon="http://getfirefox.com"
+   updatecheckdays="7"
+>
new file mode 100644
--- /dev/null
+++ b/toolkit/components/search/tests/xpcshell/data/engine.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<SearchPlugin xmlns="http://www.mozilla.org/2006/browser/search/">
+<ShortName>Test search engine</ShortName>
+<Description>A test search engine (based on Google search)</Description>
+<InputEncoding>UTF-8</InputEncoding>
+<Image width="16" height="16">data:image/png;base64,AAABAAEAEBAAAAEAGABoAwAAFgAAACgAAAAQAAAAIAAAAAEAGAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADs9Pt8xetPtu9FsfFNtu%2BTzvb2%2B%2Fne4dFJeBw0egA%2FfAJAfAA8ewBBegAAAAD%2B%2FPtft98Mp%2BwWsfAVsvEbs%2FQeqvF8xO7%2F%2F%2F63yqkxdgM7gwE%2FggM%2BfQA%2BegBDeQDe7PIbotgQufcMufEPtfIPsvAbs%2FQvq%2Bfz%2Bf%2F%2B%2B%2FZKhR05hgBBhQI8hgBAgAI9ewD0%2B%2Fg3pswAtO8Cxf4Kw%2FsJvvYAqupKsNv%2B%2Fv7%2F%2FP5VkSU0iQA7jQA9hgBDgQU%2BfQH%2F%2Ff%2FQ6fM4sM4KsN8AteMCruIqqdbZ7PH8%2Fv%2Fg6Nc%2Fhg05kAA8jAM9iQI%2BhQA%2BgQDQu6b97uv%2F%2F%2F7V8Pqw3eiWz97q8%2Ff%2F%2F%2F%2F7%2FPptpkkqjQE4kwA7kAA5iwI8iAA8hQCOSSKdXjiyflbAkG7u2s%2F%2B%2F%2F39%2F%2F7r8utrqEYtjQE8lgA7kwA7kwA9jwA9igA9hACiWSekVRyeSgiYSBHx6N%2F%2B%2Fv7k7OFRmiYtlAA5lwI7lwI4lAA7kgI9jwE9iwI4iQCoVhWcTxCmb0K%2BooT8%2Fv%2F7%2F%2F%2FJ2r8fdwI1mwA3mQA3mgA8lAE8lAE4jwA9iwE%2BhwGfXifWvqz%2B%2Ff%2F58u%2Fev6Dt4tr%2B%2F%2F2ZuIUsggA7mgM6mAM3lgA5lgA6kQE%2FkwBChwHt4dv%2F%2F%2F728ei1bCi7VAC5XQ7kz7n%2F%2F%2F6bsZkgcB03lQA9lgM7kwA2iQktZToPK4r9%2F%2F%2F9%2F%2F%2FSqYK5UwDKZAS9WALIkFn%2B%2F%2F3%2F%2BP8oKccGGcIRJrERILYFEMwAAuEAAdX%2F%2Ff7%2F%2FP%2B%2BfDvGXQLIZgLEWgLOjlf7%2F%2F%2F%2F%2F%2F9QU90EAPQAAf8DAP0AAfMAAOUDAtr%2F%2F%2F%2F7%2B%2Fu2bCTIYwDPZgDBWQDSr4P%2F%2Fv%2F%2F%2FP5GRuABAPkAA%2FwBAfkDAPAAAesAAN%2F%2F%2B%2Fz%2F%2F%2F64g1C5VwDMYwK8Yg7y5tz8%2Fv%2FV1PYKDOcAAP0DAf4AAf0AAfYEAOwAAuAAAAD%2F%2FPvi28ymXyChTATRrIb8%2F%2F3v8fk6P8MAAdUCAvoAAP0CAP0AAfYAAO4AAACAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAQAA</Image>
+<Url type="application/x-suggestions+json" method="GET" template="http://suggestqueries.google.com/complete/search?output=firefox&amp;client=firefox&amp;hl={moz:locale}&amp;q={searchTerms}"/>
+<Url type="text/html" method="GET" template="http://www.google.com/search">
+  <Param name="q" value="{searchTerms}"/>
+  <Param name="ie" value="utf-8"/>
+  <Param name="oe" value="utf-8"/>
+  <Param name="aq" value="t"/>
+  <!-- Dynamic parameters -->
+  <Param name="rls" value="{moz:distributionID}:{moz:locale}:{moz:official}"/>
+  <MozParam name="client" condition="defaultEngine" trueValue="firefox-a" falseValue="firefox"/>
+</Url>
+<SearchForm>http://www.google.com/</SearchForm>
+</SearchPlugin>
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..442ab4dc8093602a04c8e5affb02510d84853e9c
GIT binary patch
literal 901
zc$@)+1A6=b0096201yxW0096X0B8gN02TlM0EtjeM-2)Z3IG5A4M|8uQUCw|5C8xG
z5C{eU001BJ|6u?C12#!SK~y-61;I^h+XNVZ;m`SLoU}q>u_w6ECYgFo7iWmuLSuez
zw8RmJ)1t=HxUFs6Nt|efBaK@baX{P(Ns4wHtD}lrrNw39HmOdL;DWRm0hhEXmzU>}
ztN-Ts=PJSPvpIg9CHQ$Z$KzR&?`BE9oF)0}D#g9o9G_e!d4Cq?t;;0KvpHV7RC#5Z
zLZ*Mmh$113gkU20k<g0-Zz8n+5}YW89mS|e3hzXM5yi+w39^$k^Di^V^f!e`D1@Of
z3WcLk=!JrJCb(k-XRKg{3iVK^js+u(kvmh-LY>Un>&V6PI8h*s17Q>hejxM%g-#&2
zf#3wfMj+Gz!3+`@ry262EZS&^%(G?W;#eW_g|RQ3`ohsr=naLAFSx$2?F$>eLhUGy
zd6c2->*W1Kw4)-Kr?-%cA7ey)VcZu^2MWW1LjO?k4u$qWXboaC2MMfR2D6{#R)2}Y
zpvd*XZ5Dd(A=7UZqK*)Dgi}u#c7=f_bUmT%DL7u7W=F;9=v2KW%3hIt_cpq>#(d{v
z<nm#HNn04Z!pIeTR~WcL*Hv)cIF6g4>0YDm<}lku%5H^1dkx)fu;6}*Ts%;iIKsFk
z1diZ4Lf;WOtr%Zyr)jk`Hk?H)XN9U$!En~eI}J2vi%jbQGQAfkx+|R9LSQQlZH2xa
zr_)U1+8R6d61H8WZdWkvHOh8_g1w2}+-0Hp5Se}yBdRN$TMEC_6nraAua@TPT9&qz
zW5+6DTNSMO8r6D(l69YaeGkprC-dRA$n@Pf6H_=}RS2pH{HjXN%;HsZw5uyPW(C{)
z0LyGpHSbe4Kc`^sqnqC|zxo)NzNs)Vgs>Fj*-aI{l%;Ph(J@wVjkj@1>oko!)Ql}w
z%exrmee&f4G~*{SrDNn`InJaI=PaKlC|qM$SY%LG!7Eg7Z>-}K8r&^xVdZx*Z#=@t
zACNCRLC>EspL>o>uP1q#RS7knKwIQkzsaFip{uRa)*EbVo7nmuR`x5Z+5si)2|4Wq
zO?yEmdyf2@R`^p*@j}%(QS&@amH9!f@<^?-r#9J4?{FvWvaWWisC|m+kT=pJURB4W
bQh)G2Kinzq-|<En00000NkvXXu0mjftTUs#
new file mode 100644
--- /dev/null
+++ b/toolkit/components/search/tests/xpcshell/data/search-metadata.json
@@ -0,0 +1,28 @@
+{
+    "[app]/amazondotcom.xml": {
+        "used": 0,
+        "order": 4
+    },
+    "[app]/eBay.xml": {
+        "used": 0,
+        "order": 1
+    },
+    "[app]/wikipedia.xml": {
+        "used": 0,
+        "order": 6
+    },
+    "[app]/twitter.xml": {
+        "used": 0,
+        "order": 7
+    },
+    "[app]/google.xml": {
+        "used": 1,
+        "order": 2
+    },
+    "[app]/bing.xml": {
+        "order": 5
+    },
+    "[app]/yahoo.xml": {
+        "order": 3
+    }
+}
new file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..983bb831a8aed422790006c0bd16f430b1b043ff
GIT binary patch
literal 65536
zc%1Ff!D<sx6b9fs#i0xBQWQnF*o}qOF4~PdQyD_B7EQA$A~KpW3{EnkPFodZ?9ylP
zIeZcEL5$Fe0T(U>`9AJ}d-(sw<sLmdD4Qa_sZS<(6L-SrkR;(@j3I>G`K*NBlFM1%
z|7JIAeZRZ*tK5DH^8)|?000000000000000000000000000000000000Drm&$?A>u
z_2i=8<gcotn2yV7@p6<m`O<ae(ID#$vpDSS9b|DSi1*4-+&>&<{cI4Q4)&k)2FLMn
zcD#96nx{v@!Tw=?GfwkKu~^RX>a^Q7*5=Ph+bXnQ+OO@mwrYR;>rVjy0000000000
z00000000000000000000000000N{Uj`)V4tuP4v*+3dx3b6z%0adQ9tq&j_9j6%GU
zh6i1kPx6oTbW}I5>&c>VGfl%r*C_V#4~yi^Y8rOB<h*=a&Wcf)FXCI*(y-OV<GLPK
IMVBQ%0bgHcF#rGn
--- a/toolkit/components/search/tests/xpcshell/head_search.js
+++ b/toolkit/components/search/tests/xpcshell/head_search.js
@@ -1,42 +1,15 @@
 /* -*- Mode: Java; tab-width: 2; indent-tabs-mode: nil; c-basic-offset: 2 -*- */
 /* vim:set ts=2 sw=2 sts=2 et: */
-/* ***** BEGIN LICENSE BLOCK *****
- * Version: MPL 1.1/GPL 2.0/LGPL 2.1
- *
- * The contents of this file are subject to the Mozilla Public License Version
- * 1.1 (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- * http://www.mozilla.org/MPL/
- *
- * Software distributed under the License is distributed on an "AS IS" basis,
- * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
- * for the specific language governing rights and limitations under the
- * License.
- *
- * The Initial Developer of the Original Code is POTI Inc.
- * Portions created by the Initial Developer are Copyright (C) 2007
- * the Initial Developer. All Rights Reserved.
- *
- * Contributor(s):
- *
- * Alternatively, the contents of this file may be used under the terms of
- * either the GNU General Public License Version 2 or later (the "GPL"), or
- * the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
- * in which case the provisions of the GPL or the LGPL are applicable instead
- * of those above. If you wish to allow use of your version of this file only
- * under the terms of either the GPL or the LGPL, and not to allow others to
- * use your version of this file under the terms of the MPL, indicate your
- * decision by deleting the provisions above and replace them with the notice
- * and other provisions required by the GPL or the LGPL. If you do not delete
- * the provisions above, a recipient may use your version of this file under
- * the terms of any one of the MPL, the GPL or the LGPL.
- *
- * ***** END LICENSE BLOCK ***** */
+
+Components.utils.import("resource://gre/modules/Services.jsm");
+Components.utils.import("resource://gre/modules/NetUtil.jsm");
+Components.utils.import("resource://gre/modules/XPCOMUtils.jsm");
+
 
 const XULAPPINFO_CONTRACTID = "@mozilla.org/xre/app-info;1";
 const XULAPPINFO_CID = Components.ID("{c763b610-9d49-455a-bbd2-ede71682a1ac}");
 
 var gXULAppInfo = null;
 
 /**
  * Creates an nsIXULAppInfo
@@ -85,8 +58,54 @@ function createAppInfo(id, name, version
   var registrar = Components.manager.QueryInterface(Components.interfaces.nsIComponentRegistrar);
   registrar.registerFactory(XULAPPINFO_CID, "XULAppInfo",
                             XULAPPINFO_CONTRACTID, XULAppInfoFactory);
 }
 
 // Need to create and register a profile folder.
 var gProfD = do_get_profile();
 
+function dumpn(text)
+{
+  dump(text+"\n");
+}
+
+/**
+ * Clean the profile of any metadata files left from a previous run.
+ */
+function removeMetadata()
+{
+  let file = gProfD.clone();
+  file.append("search-metadata.json");
+  if (file.exists()) {
+    file.remove(false);
+  }
+
+  file = gProfD.clone();
+  file.append("search.sqlite");
+  if (file.exists()) {
+    file.remove(false);
+  }
+}
+
+/**
+ * Run some callback once metadata has been committed to disk.
+ */
+function afterCommit(callback)
+{
+  let obs = function(result, topic, verb) {
+    if (verb == "write-metadata-to-disk-complete") {
+      callback(result);
+    } else {
+      dump("TOPIC: " + topic+ "\n");
+    }
+  }
+  Services.obs.addObserver(obs, "browser-search-service", false);
+}
+
+function  parseJsonFromStream(aInputStream) {
+  const json = Cc["@mozilla.org/dom/json;1"].createInstance(Components.interfaces.nsIJSON);
+  const data = json.decodeFromStream(aInputStream, aInputStream.available());
+  return data;
+}
+
+Services.prefs.setBoolPref("browser.search.log", true);
+//Otherwise, error logs contain no useful data
--- a/toolkit/components/search/tests/xpcshell/test_645970.js
+++ b/toolkit/components/search/tests/xpcshell/test_645970.js
@@ -28,40 +28,28 @@
  * use your version of this file under the terms of the MPL, indicate your
  * decision by deleting the provisions above and replace them with the notice
  * and other provisions required by the GPL or the LGPL. If you do not delete
  * the provisions above, a recipient may use your version of this file under
  * the terms of any one of the MPL, the GPL or the LGPL.
  *
  * ***** END LICENSE BLOCK ***** */
 
-let Cc = Components.classes;
-let Ci = Components.interfaces;
-let Cu = Components.utils;
-
-Cu.import("resource://gre/modules/Services.jsm");
-
-
-var gPrefService = Cc["@mozilla.org/preferences-service;1"]
-                    .getService(Ci.nsIPrefService)
-                    .QueryInterface(Ci.nsIPrefBranch);
 /**
  * Test nsSearchService with nested jar: uris
  */
 function run_test() {
   createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "2");
 
   do_load_manifest("data/chrome.manifest");
 
   let url  = "chrome://testsearchplugin/locale/searchplugins/";
-  gPrefService.setCharPref("browser.search.jarURIs", url);
+  Services.prefs.setCharPref("browser.search.jarURIs", url);
 
-  gPrefService.setBoolPref("browser.search.loadFromJars", true);
+  Services.prefs.setBoolPref("browser.search.loadFromJars", true);
 
   // The search service needs to be started after the jarURIs pref has been
   // set in order to initiate it correctly
-  let searchService = Cc["@mozilla.org/browser/search-service;1"]
-                       .getService(Ci.nsIBrowserSearchService);
-  let engine = searchService.getEngineByName("bug645970");
+  let engine = Services.search.getEngineByName("bug645970");
   do_check_neq(engine, null);
   Services.obs.notifyObservers(null, "quit-application", null);
 }
 
new file mode 100644
--- /dev/null
+++ b/toolkit/components/search/tests/xpcshell/test_migratedb.js
@@ -0,0 +1,38 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * test_migratedb: Start search engine
+ * - without search-metadata.json
+ * - with search.sqlite
+ *
+ * Ensure that nothing explodes.
+ */
+
+function run_test()
+{
+  removeMetadata();
+
+  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "2");
+
+  let search_sqlite = do_get_file("data/search.sqlite");
+  search_sqlite.copyTo(gProfD, "search.sqlite");
+
+  let search = Services.search;
+
+  do_test_pending();
+  afterCommit(
+    function()
+    {
+      //Check that search-metadata.json has been created
+      let metadata = gProfD.clone();
+      metadata.append("search-metadata.json");
+      do_check_true(metadata.exists());
+
+      removeMetadata();
+      do_test_finished();
+    }
+  );
+
+  search.getEngines();
+}
new file mode 100644
--- /dev/null
+++ b/toolkit/components/search/tests/xpcshell/test_nodb.js
@@ -0,0 +1,37 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+/*
+ * test_nodb: Start search engine
+ * - without search-metadata.json
+ * - without search.sqlite
+ *
+ * Ensure that :
+ * - nothing explodes;
+ * - no search-metadata.json is created.
+ */
+
+
+function run_test()
+{
+  removeMetadata();
+  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "2");
+
+  let search = Services.search; // Cause service initialization
+
+  do_test_pending();
+  do_timeout(500,
+             function()
+             {
+               // Check that search-metadata.json has not been created
+               // Note that we cannot du much better than a timeout for
+               // checking a non-event.
+               let metadata = gProfD.clone();
+               metadata.append("search-metadata.json");
+               do_check_true(!metadata.exists());
+               removeMetadata();
+
+               do_test_finished();
+             }
+            );
+}
new file mode 100644
--- /dev/null
+++ b/toolkit/components/search/tests/xpcshell/test_nodb_pluschanges.js
@@ -0,0 +1,87 @@
+/* Any copyright is dedicated to the Public Domain.
+   http://creativecommons.org/publicdomain/zero/1.0/ */
+
+
+/*
+ * test_nodb: Start search engine
+ * - without search-metadata.json
+ * - without search.sqlite
+ *
+ * Ensure that :
+ * - nothing explodes;
+ * - if we change the order, search-metadata.json is created;
+ * - this search-medata.json can be parsed;
+ * - the order stored in search-metadata.json is consistent.
+ *
+ * Notes:
+ * - we install the search engines of test "test_downloadAndAddEngines.js"
+ * to ensure that this test is independent from locale, commercial agreements
+ * and configuration of Firefox.
+ */
+
+do_load_httpd_js();
+
+function run_test()
+{
+  removeMetadata();
+  createAppInfo("xpcshell@tests.mozilla.org", "XPCShell", "1", "2");
+  do_load_manifest("data/chrome.manifest");
+
+  let httpServer = new nsHttpServer();
+  httpServer.start(4444);
+  httpServer.registerDirectory("/", do_get_cwd());
+
+  let search = Services.search;
+
+  function observer(aSubject, aTopic, aData) {
+    if ("engine-added" == aData) {
+      let engine1 = search.getEngineByName("Test search engine");
+      let engine2 = search.getEngineByName("Sherlock test search engine");
+      dumpn("Got engine 2: "+engine2);
+      if(engine1 && engine2)
+      {
+        search.moveEngine(engine1, 0);
+        search.moveEngine(engine2, 1);
+        do_timeout(0,
+                   function() {
+                     // Force flush
+                     // Note: the timeout is needed, to avoid some reentrency
+                     // issues in nsSearchService.
+                     search.QueryInterface(Ci.nsIObserver).
+                       observe(observer, "quit-application", "<no verb>");
+                   });
+        afterCommit(
+          function()
+          {
+            // Check that search-metadata.json has been created
+            let metadata = gProfD.clone();
+            metadata.append("search-metadata.json");
+            do_check_true(metadata.exists());
+
+            // Check that the entries are placed as specified correctly
+	    let stream = NetUtil.newChannel(metadata).open();
+            let json = parseJsonFromStream(stream);
+            do_check_eq(json["[app]/test-search-engine.xml"].order, 1);
+            do_check_eq(json["[profile]/sherlock-test-search-engine.xml"].order, 2);
+	    httpServer.stop(function() {});
+	    stream.close(); // Stream must be closed under Windows
+	    removeMetadata();
+            do_test_finished();
+          }
+        );
+      }
+    }
+  };
+  Services.obs.addObserver(observer, "browser-search-engine-modified",
+                           false);
+
+  do_test_pending();
+
+  search.addEngine("http://localhost:4444/data/engine.xml",
+                   Ci.nsISearchEngine.DATA_XML,
+                   null, false);
+  search.addEngine("http://localhost:4444/data/engine.src",
+                   Ci.nsISearchEngine.DATA_TEXT,
+                   "http://localhost:4444/data/ico-size-16x16-png.ico",
+                   false);
+}
--- a/toolkit/components/search/tests/xpcshell/xpcshell.ini
+++ b/toolkit/components/search/tests/xpcshell/xpcshell.ini
@@ -1,5 +1,8 @@
 [DEFAULT]
 head = head_search.js
 tail = 
 
+[test_nodb.js]
+[test_nodb_pluschanges.js]
+[test_migratedb.js]
 [test_645970.js]