Bug 674210 - Reduce places.sqlite cache size and reorganize history expiration around the new value.
authorMarco Bonardo <mbonardo@mozilla.com>
Tue, 30 Aug 2011 16:23:59 +0200
changeset 77528 ed85e33792e086643c1ff66b65e7909525efe34e
parent 77527 fdfc74d7e8268802102a28977b00204b89421140
child 77529 fea3711b548963e6b81e2f94bc63ba9852dce6f7
push id78
push userclegnitto@mozilla.com
push dateFri, 16 Dec 2011 17:32:24 +0000
treeherdermozilla-release@79d24e644fdd [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs674210
milestone9.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 674210 - Reduce places.sqlite cache size and reorganize history expiration around the new value. r=dietrich
browser/app/profile/firefox.js
toolkit/components/downloads/test/unit/test_history_expiration.js
toolkit/components/places/nsNavHistory.cpp
toolkit/components/places/nsPlacesExpiration.js
toolkit/components/places/tests/expiration/head_expiration.js
toolkit/components/places/tests/expiration/test_annos_expire_history.js
toolkit/components/places/tests/expiration/test_annos_expire_never.js
toolkit/components/places/tests/expiration/test_debug_expiration.js
toolkit/components/places/tests/expiration/test_notifications_onDeleteURI.js
toolkit/components/places/tests/expiration/test_notifications_onDeleteVisits.js
toolkit/components/places/tests/expiration/test_pref_maxpages.js
toolkit/components/places/tests/unit/test_history_removeAllPages.js
toolkit/components/places/tests/unit/test_removeVisitsByTimeframe.js
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -793,22 +793,16 @@ pref("browser.sessionstore.restore_on_de
 pref("browser.sessionstore.restore_hidden_tabs", false);
 
 // allow META refresh by default
 pref("accessibility.blockautorefresh", false);
 
 // Whether history is enabled or not.
 pref("places.history.enabled", true);
 
-// The percentage of system memory that the Places database can use.  Out of the
-// allowed cache size it will at most use the size of the database file.
-// Changes to this value are effective after an application restart.
-// Acceptable values are between 0 and 50.
-pref("places.database.cache_to_memory_percentage", 6);
-
 // the (maximum) number of the recent visits to sample
 // when calculating frecency
 pref("places.frecency.numVisits", 10);
 
 // buckets (in days) for frecency calculation
 pref("places.frecency.firstBucketCutoff", 4);
 pref("places.frecency.secondBucketCutoff", 14);
 pref("places.frecency.thirdBucketCutoff", 31);
--- a/toolkit/components/downloads/test/unit/test_history_expiration.js
+++ b/toolkit/components/downloads/test/unit/test_history_expiration.js
@@ -87,18 +87,19 @@ function run_test()
     stmt.reset();
     stmt.finalize();
   }
 
   let histsvc = Cc["@mozilla.org/browser/nav-history-service;1"].
                 getService(Ci.nsINavHistoryService);
   // Add the download to places
   // Add the visit in the past to circumvent possible VM timing bugs
-  let yesterday = Date.now() - 24 * 60 * 60 * 1000;
-  histsvc.addVisit(theURI, yesterday * 1000, null,
+  // Go back by 8 days, since expiration ignores history in the last 7 days.
+  let expirableTime = Date.now() - 8 * 24 * 60 * 60 * 1000;
+  histsvc.addVisit(theURI, expirableTime * 1000, null,
                    histsvc.TRANSITION_DOWNLOAD, false, 0);
 
   // Get the download manager as history observer and batch expirations
   let histobs = dm.QueryInterface(Ci.nsINavHistoryObserver);
   histobs.onBeginUpdateBatch();
 
   // Look for the removed download notification
   let obs = Cc["@mozilla.org/observer-service;1"].
--- a/toolkit/components/places/nsNavHistory.cpp
+++ b/toolkit/components/places/nsNavHistory.cpp
@@ -123,24 +123,37 @@ using namespace mozilla::places;
 #define PREF_FRECENCY_BOOKMARK_VISIT_BONUS      "frecency.bookmarkVisitBonus"
 #define PREF_FRECENCY_DOWNLOAD_VISIT_BONUS      "frecency.downloadVisitBonus"
 #define PREF_FRECENCY_PERM_REDIRECT_VISIT_BONUS "frecency.permRedirectVisitBonus"
 #define PREF_FRECENCY_TEMP_REDIRECT_VISIT_BONUS "frecency.tempRedirectVisitBonus"
 #define PREF_FRECENCY_DEFAULT_VISIT_BONUS       "frecency.defaultVisitBonus"
 #define PREF_FRECENCY_UNVISITED_BOOKMARK_BONUS  "frecency.unvisitedBookmarkBonus"
 #define PREF_FRECENCY_UNVISITED_TYPED_BONUS     "frecency.unvisitedTypedBonus"
 
-#define PREF_CACHE_TO_MEMORY_PERCENTAGE         "database.cache_to_memory_percentage"
-
 #define PREF_FORCE_DATABASE_REPLACEMENT         "database.replaceOnStartup"
 
-// Default integer value for PREF_CACHE_TO_MEMORY_PERCENTAGE.
-// This is 6% of machine memory, giving 15MB for a user with 256MB of memory.
-// Out of this cache, SQLite will use at most the size of the database file.
-#define DATABASE_DEFAULT_CACHE_TO_MEMORY_PERCENTAGE 6
+// To calculate the cache size we take into account the available physical
+// memory and the current database size.  This is the percentage of memory
+// we reserve for the former case.
+#define DATABASE_CACHE_TO_MEMORY_PERC 2
+// The minimum size of the cache.  We should never work without a cache, since
+// that would badly hurt WAL journaling mode.
+#define DATABASE_CACHE_MIN_BYTES (PRUint64)5242880 // 5MiB
+
+// We calculate an optimal database size, based on hardware specs.  This
+// pertains more to expiration, but the code is pretty much the same used for
+// cache_size, so it's here to reduce code duplication.
+// This percentage of disk size is used to protect against calculating a too
+// large size on disks with tiny quota or available space.
+#define DATABASE_TO_DISK_PERC 2
+// Maximum size of the optimal database.  High-end hardware has plenty of
+// memory and disk space, but performances don't grow linearly.
+#define DATABASE_MAX_SIZE (PRInt64)167772160 // 160MiB
+// Used to share the calculated optimal database size with other components.
+#define PREF_OPTIMAL_DATABASE_SIZE "history.expiration.transient_optimal_database_size"
 
 // If the physical memory size is not available, use MEMSIZE_FALLBACK_BYTES
 // instead.  Must stay in sync with the code in nsPlacesExpiration.js.
 #define MEMSIZE_FALLBACK_BYTES 268435456 // 256 M
 
 // Maximum size for the WAL file.  It should be small enough since in case of
 // crashes we could lose all the transactions in the file.  But a too small
 // file could hurt performance.
@@ -676,61 +689,102 @@ nsNavHistory::SetJournalMode(enum Journa
 
   return NS_OK;
 }
 
 
 nsresult
 nsNavHistory::InitDB()
 {
+  // WARNING: any statement executed before setting the journal mode must be
+  // finalized, since SQLite doesn't allow changing the journal mode if there
+  // is any outstanding statement.
+
   {
     // Get the page size.  This may be different than the default if the
     // database file already existed with a different page size.
     nsCOMPtr<mozIStorageStatement> statement;
-    nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"),
-                                  getter_AddRefs(statement));
+    nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
+      "PRAGMA page_size"
+    ), getter_AddRefs(statement));
     NS_ENSURE_SUCCESS(rv, rv);
-
-    PRBool hasResult;
-    mozStorageStatementScoper scoper(statement);
+    PRBool hasResult = PR_FALSE;
     rv = statement->ExecuteStep(&hasResult);
     NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
     rv = statement->GetInt32(0, &mDBPageSize);
     NS_ENSURE_SUCCESS(rv, rv);
     NS_ENSURE_TRUE(mDBPageSize > 0, NS_ERROR_UNEXPECTED);
   }
 
   // Ensure that temp tables are held in memory, not on disk.
   nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
       "PRAGMA temp_store = MEMORY"));
   NS_ENSURE_SUCCESS(rv, rv);
 
-  // Compute the size of the database cache using the device's memory size.
+  // We want to work with a cache that is at a maximum half of the database
+  // size.  We also want it to respect the available memory size.
+
+  // Calculate memory size, fallback to a meaningful value if it fails.
+  PRUint64 memSizeBytes = PR_GetPhysicalMemorySize();
+  if (memSizeBytes == 0) {
+    memSizeBytes = MEMSIZE_FALLBACK_BYTES;
+  }
+
+  PRUint64 cacheSize = memSizeBytes * DATABASE_CACHE_TO_MEMORY_PERC / 100;
+
+  // Calculate an optimal database size for expiration purposes.
+  // We usually want to work with a cache that is half the database size.
+  // Limit the size to avoid extreme values on high-end hardware.
+  PRInt64 optimalDatabaseSize = NS_MIN(static_cast<PRInt64>(cacheSize) * 2,
+                                       DATABASE_MAX_SIZE);
+
+  // Protect against a full disk or tiny quota.
+  PRInt64 diskAvailableBytes = 0;
+  nsCOMPtr<nsILocalFile> localDB = do_QueryInterface(mDBFile);
+  if (localDB &&
+      NS_SUCCEEDED(localDB->GetDiskSpaceAvailable(&diskAvailableBytes)) &&
+      diskAvailableBytes > 0) {
+    optimalDatabaseSize = NS_MIN(optimalDatabaseSize,
+                                 diskAvailableBytes * DATABASE_TO_DISK_PERC / 100);
+  }
+
+  // Share the calculated size if it's meaningful.
+  if (optimalDatabaseSize < PR_INT32_MAX) {
+    (void)mPrefBranch->SetIntPref(PREF_OPTIMAL_DATABASE_SIZE,
+                                  static_cast<PRInt32>(optimalDatabaseSize));
+  }
+
+  // Get the current database size. Due to chunked growth we have to use
+  // page_count to evaluate it.
+  PRUint64 databaseSizeBytes = 0;
+  {
+    nsCOMPtr<mozIStorageStatement> statement;
+    nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING(
+      "PRAGMA page_count"
+    ), getter_AddRefs(statement));
+    NS_ENSURE_SUCCESS(rv, rv);
+    PRBool hasResult = PR_FALSE;
+    rv = statement->ExecuteStep(&hasResult);
+    NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE);
+    PRInt32 pageCount = 0;
+    rv = statement->GetInt32(0, &pageCount);
+    NS_ENSURE_SUCCESS(rv, rv);
+    databaseSizeBytes = pageCount * mDBPageSize;
+  }
+
+  // Set cache to a maximum of half the database size.
+  cacheSize = NS_MIN(cacheSize, databaseSizeBytes / 2);
+  // Ensure we never work without a minimum cache.
+  cacheSize = NS_MAX(cacheSize, DATABASE_CACHE_MIN_BYTES);
+
+  // Set the number of cached pages.
   // We don't use PRAGMA default_cache_size, since the database could be moved
   // among different devices and the value would adapt accordingly.
-  PRInt32 cachePercentage;
-  if (NS_FAILED(mPrefBranch->GetIntPref(PREF_CACHE_TO_MEMORY_PERCENTAGE,
-                                        &cachePercentage)))
-    cachePercentage = DATABASE_DEFAULT_CACHE_TO_MEMORY_PERCENTAGE;
-  // Sanity checks, we allow values between 0 (disable cache) and 50%.
-  if (cachePercentage > 50)
-    cachePercentage = 50;
-  if (cachePercentage < 0)
-    cachePercentage = 0;
-
-  static PRUint64 physMem = PR_GetPhysicalMemorySize();
-  if (physMem == 0)
-    physMem = MEMSIZE_FALLBACK_BYTES;
-
-  PRUint64 cacheSize = physMem * cachePercentage / 100;
-
-  // Compute number of cached pages, this will be our cache size.
-  PRUint64 cachePages = cacheSize / mDBPageSize;
   nsCAutoString cacheSizePragma("PRAGMA cache_size = ");
-  cacheSizePragma.AppendInt(cachePages);
+  cacheSizePragma.AppendInt(cacheSize / mDBPageSize);
   rv = mDBConn->ExecuteSimpleSQL(cacheSizePragma);
   NS_ENSURE_SUCCESS(rv, rv);
 
   // Be sure to set journal mode after page_size.  WAL would prevent the change
   // otherwise.
   if (NS_SUCCEEDED(SetJournalMode(JOURNAL_WAL))) {
     // Set the WAL journal size limit.  We want it to be small, since in
     // synchronous = NORMAL mode a crash could cause loss of all the
@@ -4125,19 +4179,20 @@ nsNavHistory::PreparePlacesForVisitsDele
     return NS_OK;
 
   // if a moz_place is annotated or was a bookmark,
   // we won't delete it, but we will delete the moz_visits
   // so we need to reset the frecency.  Note, we set frecency to
   // -visit_count, as we use that value in our "on idle" query
   // to figure out which places to recalculate frecency first.
   // Pay attention to not set frecency = 0 if visit_count = 0
+  // TODO (bug 487809): we don't need anymore to set frecency here.
   nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
       "UPDATE moz_places "
-      "SET frecency = -MAX(visit_count, 1) "
+      "SET frecency = -(visit_count + 1) "
       "WHERE id IN ( "
         "SELECT h.id " 
         "FROM moz_places h "
         "WHERE h.id IN ( ") + aPlaceIdsQueryString + NS_LITERAL_CSTRING(") "
           "AND ( "
             "EXISTS (SELECT b.id FROM moz_bookmarks b WHERE b.fk =h.id) "
             "OR EXISTS (SELECT a.id FROM moz_annos a WHERE a.place_id = h.id) "
           ") "        
@@ -4558,18 +4613,19 @@ nsNavHistory::RemoveAllPages()
   NS_ASSERTION(NS_IsMainThread(), "This can only be called on the main thread");
 
   mozStorageTransaction transaction(mDBConn, PR_FALSE);
 
   // reset frecency for all items that will _not_ be deleted
   // Note, we set frecency to -visit_count since we use that value in our
   // idle query to figure out which places to recalcuate frecency first.
   // We must do this before deleting visits.
+  // TODO (bug 487809): we don't need anymore to set frecency here.
   nsresult rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
-    "UPDATE moz_places SET frecency = -MAX(visit_count, 1) "
+    "UPDATE moz_places SET frecency = -(visit_count + 1) "
     "WHERE id IN(SELECT b.fk FROM moz_bookmarks b WHERE b.fk NOTNULL)"));
   NS_ENSURE_SUCCESS(rv, rv);
 
   // Expire visits, then let the paranoid functions do the cleanup for us.
   rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
       "DELETE FROM moz_historyvisits"));
   NS_ENSURE_SUCCESS(rv, rv);
 
--- a/toolkit/components/places/nsPlacesExpiration.js
+++ b/toolkit/components/places/nsPlacesExpiration.js
@@ -104,26 +104,20 @@ const PREF_MAX_URIS_NOTSET = -1; // Use 
 // We save the current unique URIs limit to this pref, to make it available to
 // other components without having to duplicate the full logic.
 const PREF_READONLY_CALCULATED_MAX_URIS = "transient_current_max_pages";
 
 // Seconds between each expiration step.
 const PREF_INTERVAL_SECONDS = "interval_seconds";
 const PREF_INTERVAL_SECONDS_NOTSET = 3 * 60;
 
-// The percentage of system memory we will use for the database's cache.
-// Use the same value set in nsNavHistory.cpp.  We use the size of the cache to
-// evaluate how many pages we can store before going over it.
-const PREF_DATABASE_CACHE_PER_MEMORY_PERCENTAGE =
-  "places.history.cache_per_memory_percentage";
-const PREF_DATABASE_CACHE_PER_MEMORY_PERCENTAGE_NOTSET = 6;
-
-// Minimum number of unique URIs to retain.  This is used when system-info
-// returns bogus values.
-const MIN_URIS = 1000;
+// An optimal database size calculated by history.  Used to evaluate a limit
+// to the number of pages we may retain before hitting performance issues.
+const PREF_OPTIMAL_DATABASE_SIZE = "transient_optimal_database_size";
+const PREF_OPTIMAL_DATABASE_SIZE_NOTSET = 167772160; // 160MiB
 
 // Max number of entries to expire at each expiration step.
 // This value is globally used for different kind of data we expire, can be
 // tweaked based on data type.  See below in getBoundStatement.
 const EXPIRE_LIMIT_PER_STEP = 6;
 // When we run a large expiration step, the above limit is multiplied by this.
 const EXPIRE_LIMIT_PER_LARGE_STEP_MULTIPLIER = 10;
 
@@ -136,24 +130,20 @@ const EXPIRE_LIMIT_PER_LARGE_STEP_MULTIP
 //   default number of entries.
 // 2. Dirty history:
 //   We expire at the default interval, but a greater number of entries
 //   (default number of entries * EXPIRE_AGGRESSIVITY_MULTIPLIER).
 const EXPIRE_AGGRESSIVITY_MULTIPLIER = 3;
 
 // This is the average size in bytes of an URI entry in the database.
 // Magic numbers are determined through analysis of the distribution of a ratio
-// between number of unique URIs and database size among our users.  We use a
-// more pessimistic ratio on single cores, since we handle some stuff in a
-// separate thread.
+// between number of unique URIs and database size among our users.
 // Based on these values we evaluate how many unique URIs we can handle before
-// going over the database maximum cache size.  If we are over the maximum
-// number of entries, we will expire.
-const URIENTRY_AVG_SIZE_MIN = 2000;
-const URIENTRY_AVG_SIZE_MAX = 3000;
+// starting expiring some.
+const URIENTRY_AVG_SIZE = 1600;
 
 // Seconds of idle time before starting a larger expiration step.
 // Notice during idle we stop the expiration timer since we don't want to hurt
 // stand-by or mobile devices batteries.
 const IDLE_TIMEOUT_SECONDS = 5 * 60;
 
 // If a clear history ran just before we shutdown, we will skip most of the
 // expiration at shutdown.  This is maximum number of seconds from last
@@ -205,23 +195,26 @@ const ACTION = {
                            // history
 };
 
 // The queries we use to expire.
 const EXPIRATION_QUERIES = {
 
   // Finds visits to be expired.  Will return nothing if we are not over the
   // unique URIs limit.
+  // This explicitly excludes any visits added in the last 7 days, to protect
+  // users with thousands of bookmarks from constantly losing history.
   QUERY_FIND_VISITS_TO_EXPIRE: {
     sql: "INSERT INTO expiration_notify "
        +   "(v_id, url, guid, visit_date, expected_results) "
        + "SELECT v.id, h.url, h.guid, v.visit_date, :limit_visits "
        + "FROM moz_historyvisits v "
        + "JOIN moz_places h ON h.id = v.place_id "
        + "WHERE (SELECT COUNT(*) FROM moz_places) > :max_uris "
+       + "AND visit_date < strftime('%s','now','localtime','start of day','-7 days','utc') * 1000000 "
        + "ORDER BY v.visit_date ASC "
        + "LIMIT :limit_visits",
     actions: ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | ACTION.IDLE |
              ACTION.DEBUG
   },
 
   // Removes the previously found visits.
   QUERY_EXPIRE_VISITS: {
@@ -230,26 +223,30 @@ const EXPIRATION_QUERIES = {
        + ")",
     actions: ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN | ACTION.IDLE |
              ACTION.DEBUG
   },
 
   // Finds orphan URIs in the database.
   // Notice we won't notify single removed URIs on removeAllPages, so we don't
   // run this query in such a case, but just delete URIs.
+  // This could run in the middle of adding a visit or bookmark to a new page.
+  // In such a case since it is async, could end up expiring the orphan page
+  // before it actually gets the new visit or bookmark.
+  // Thus, since new pages get frecency -1, we filter on that.
   QUERY_FIND_URIS_TO_EXPIRE: {
     sql: "INSERT INTO expiration_notify "
        +   "(p_id, url, guid, visit_date, expected_results) "
        + "SELECT h.id, h.url, h.guid, h.last_visit_date, :limit_uris "
        + "FROM moz_places h "
        + "LEFT JOIN moz_historyvisits v ON h.id = v.place_id "
        + "LEFT JOIN moz_bookmarks b ON h.id = b.fk "
        + "WHERE v.id IS NULL "
        +   "AND b.id IS NULL "
-       +   "AND h.ROWID <> IFNULL(:null_skips_last, (SELECT MAX(ROWID) FROM moz_places)) "
+       +   "AND frecency <> -1 "
        + "LIMIT :limit_uris",
     actions: ACTION.TIMED | ACTION.TIMED_OVERLIMIT | ACTION.SHUTDOWN |
              ACTION.IDLE | ACTION.DEBUG
   },
 
   // Expire found URIs from the database.
   QUERY_EXPIRE_URIS: {
     sql: "DELETE FROM moz_places WHERE id IN ( "
@@ -728,17 +725,18 @@ nsPlacesExpiration.prototype = {
     }
     return aNewStatus;
   },
   get status() this._status,
 
   _isIdleObserver: false,
   _expireOnIdle: false,
   set expireOnIdle(aExpireOnIdle) {
-    // Observe idle regardless, since we want to stop timed expiration.
+    // Observe idle regardless aExpireOnIdle, since we always want to stop
+    // timed expiration on idle, to preserve mobile battery life.
     if (!this._isIdleObserver && !this._shuttingDown) {
       this._idle.addIdleObserver(this, IDLE_TIMEOUT_SECONDS);
       this._isIdleObserver = true;
     }
     else if (this._isIdleObserver && this._shuttingDown) {
       this._idle.removeIdleObserver(this, IDLE_TIMEOUT_SECONDS);
       this._isIdleObserver = false;
     }
@@ -759,48 +757,25 @@ nsPlacesExpiration.prototype = {
     try {
       // We want to silently fail since getIntPref throws if it does not exist,
       // and use a default to fallback to.
       this._urisLimit = this._prefBranch.getIntPref(PREF_MAX_URIS);
     }
     catch(e) {}
 
     if (this._urisLimit < 0) {
-      // If physical memory size is not available, use MEMSIZE_FALLBACK_BYTES
-      // instead.  Must stay in sync with the code in nsNavHistory.cpp.
-      const MEMSIZE_FALLBACK_BYTES = 268435456; // 256 M
-
-      // The preference did not exist or has a negative value, so we calculate a
-      // limit based on hardware.
-      let memsize = this._sys.getProperty("memsize"); // Memory size in bytes.
-      if (memsize <= 0)
-        memsize = MEMSIZE_FALLBACK_BYTES;
-
-      let cpucount = this._sys.getProperty("cpucount"); // CPU count.
-      const AVG_SIZE_PER_URIENTRY = cpucount > 1 ? URIENTRY_AVG_SIZE_MIN
-                                                 : URIENTRY_AVG_SIZE_MAX;
-      // We will try to live inside the database cache size, since working out
-      // of it can be really slow.
-      let cache_percentage = PREF_DATABASE_CACHE_PER_MEMORY_PERCENTAGE_NOTSET;
+      // The preference did not exist or has a negative value.
+      // Calculate the number of unique places that may fit an optimal database
+      // size on this hardware.  If there are more than these unique pages,
+      // some will be expired.
+      let optimalDatabaseSize = PREF_OPTIMAL_DATABASE_SIZE_NOTSET;
       try {
-        let prefs = Cc["@mozilla.org/preferences-service;1"].
-                    getService(Ci.nsIPrefBranch);
-        cache_percentage =
-          prefs.getIntPref(PREF_DATABASE_CACHE_PER_MEMORY_PERCENTAGE);
-        if (cache_percentage < 0) {
-          cache_percentage = 0;
-        }
-        else if (cache_percentage > 50) {
-          cache_percentage = 50;
-        }
-      }
-      catch(e) {}
-      let cachesize = memsize * cache_percentage / 100;
-      this._urisLimit = Math.max(MIN_URIS,
-                                 parseInt(cachesize / AVG_SIZE_PER_URIENTRY));
+        optimalDatabaseSize = this._prefBranch.getIntPref(PREF_OPTIMAL_DATABASE_SIZE);
+      } catch (ex) {}
+      this._urisLimit = Math.ceil(optimalDatabaseSize / URIENTRY_AVG_SIZE);
     }
     // Expose the calculated limit to other components.
     this._prefBranch.setIntPref(PREF_READONLY_CALCULATED_MAX_URIS,
                                 this._urisLimit);
 
     // Get the expiration interval value.
     try {
       // We want to silently fail since getIntPref throws if it does not exist,
@@ -822,16 +797,21 @@ nsPlacesExpiration.prototype = {
    *        LIMIT const for values.
    */
   _expireWithActionAndLimit:
   function PEX__expireWithActionAndLimit(aAction, aLimit)
   {
     // Skip expiration during batch mode.
     if (this._inBatchMode)
       return;
+    // Don't try to further expire after shutdown.
+    if (this._shuttingDown &&
+        aAction != ACTION.SHUTDOWN && aAction != ACTION.CLEAN_SHUTDOWN) {
+      return;
+    }
 
     let boundStatements = [];
     for (let queryType in EXPIRATION_QUERIES) {
       if (EXPIRATION_QUERIES[queryType].actions & aAction)
         boundStatements.push(this._getBoundStatement(queryType, aLimit, aAction));
     }
 
     // Execute statements asynchronously in a transaction.
@@ -896,28 +876,16 @@ nsPlacesExpiration.prototype = {
       case "QUERY_FIND_VISITS_TO_EXPIRE":
         params.max_uris = this._urisLimit;
         // Avoid expiring all visits in case of an unlimited debug expiration,
         // just remove orphans instead.
         params.limit_visits =
           aLimit == LIMIT.DEBUG && baseLimit == -1 ? 0 : baseLimit;
         break;
       case "QUERY_FIND_URIS_TO_EXPIRE":
-        // We could run in the middle of adding a new visit or bookmark to
-        // a new page.  In such a case since we are async, we could end up
-        // expiring the page before it actually gets the visit or bookmark,
-        // thinking it's an orphan.  So we never expire the last added page
-        // when expiration does not run on user action.
-        if (aAction != ACTION.TIMED && aAction != ACTION.TIMED_OVERLIMIT &&
-            aAction != ACTION.IDLE) {
-          params.null_skips_last = -1;
-        }
-        else {
-          params.null_skips_last = null;
-        }
         params.limit_uris = baseLimit;
         break;
       case "QUERY_SILENT_EXPIRE_ORPHAN_URIS":
         params.limit_uris = baseLimit;
         break;
       case "QUERY_EXPIRE_FAVICONS":
         params.limit_favicons = baseLimit;
         break;
--- a/toolkit/components/places/tests/expiration/head_expiration.js
+++ b/toolkit/components/places/tests/expiration/head_expiration.js
@@ -123,8 +123,25 @@ function getHistoryEnabled() {
   return Services.prefs.getBoolPref("places.history.enabled");
 }
 function clearHistoryEnabled() {
   try {
     Services.prefs.clearUserPref("places.history.enabled");
   }
   catch(ex) {}
 }
+
+/**
+ * Returns a PRTime in the past usable to add expirable visits.
+ *
+ * @note Expiration ignores any visit added in the last 7 days, but it's
+ *       better be safe against DST issues, by going back one day more.
+ */
+function getExpirablePRTime() {
+  let dateObj = new Date();
+  // Normalize to midnight
+  dateObj.setHours(0);
+  dateObj.setMinutes(0);
+  dateObj.setSeconds(0);
+  dateObj.setMilliseconds(0);
+  dateObj = new Date(dateObj.getTime() - 8 * 86400000);
+  return dateObj.getTime() * 1000;
+}
--- a/toolkit/components/places/tests/expiration/test_annos_expire_history.js
+++ b/toolkit/components/places/tests/expiration/test_annos_expire_history.js
@@ -57,17 +57,17 @@ let as = Cc["@mozilla.org/browser/annota
 function run_test() {
   // Set interval to a large value so we don't expire on it.
   setInterval(3600); // 1h
 
   // Expire all expirable pages.
   setMaxPages(0);
 
   // Add some visited page and a couple expire with history annotations for each.
-  let now = Date.now() * 1000;
+  let now = getExpirablePRTime();
   for (let i = 0; i < 5; i++) {
     let pageURI = uri("http://page_anno." + i + ".mozilla.org/");
     hs.addVisit(pageURI, now++, null, hs.TRANSITION_TYPED, false, 0);
     as.setPageAnnotation(pageURI, "page_expire1", "test", 0, as.EXPIRE_WITH_HISTORY);
     as.setPageAnnotation(pageURI, "page_expire2", "test", 0, as.EXPIRE_WITH_HISTORY);
   }
 
   let pages = as.getPagesWithAnnotation("page_expire1");
--- a/toolkit/components/places/tests/expiration/test_annos_expire_never.js
+++ b/toolkit/components/places/tests/expiration/test_annos_expire_never.js
@@ -60,17 +60,17 @@ let as = Cc["@mozilla.org/browser/annota
 function run_test() {
   // Set interval to a large value so we don't expire on it.
   setInterval(3600); // 1h
 
   // Expire all expirable pages.
   setMaxPages(0);
 
   // Add some visited page and a couple expire never annotations for each.
-  let now = Date.now() * 1000;
+  let now = getExpirablePRTime();
   for (let i = 0; i < 5; i++) {
     let pageURI = uri("http://page_anno." + i + ".mozilla.org/");
     hs.addVisit(pageURI, now++, null, hs.TRANSITION_TYPED, false, 0);
     as.setPageAnnotation(pageURI, "page_expire1", "test", 0, as.EXPIRE_NEVER);
     as.setPageAnnotation(pageURI, "page_expire2", "test", 0, as.EXPIRE_NEVER);
   }
 
   let pages = as.getPagesWithAnnotation("page_expire1");
--- a/toolkit/components/places/tests/expiration/test_debug_expiration.js
+++ b/toolkit/components/places/tests/expiration/test_debug_expiration.js
@@ -3,17 +3,17 @@
 
 /**
  * What this is aimed to test:
  *
  * Expiration can be manually triggered through a debug topic, but that should
  * only expire orphan entries, unless -1 is passed as limit.
  */
 
-let gNow = Date.now() * 1000;
+let gNow = getExpirablePRTime();
 
 add_test(function test_expire_orphans()
 {
   // Add visits to 2 pages and force a orphan expiration. Visits should survive.
   PlacesUtils.history.addVisit(NetUtil.newURI("http://page1.mozilla.org/"),
                                gNow++, null,
                                PlacesUtils.history.TRANSITION_TYPED, false, 0);
   PlacesUtils.history.addVisit(NetUtil.newURI("http://page2.mozilla.org/"),
--- a/toolkit/components/places/tests/expiration/test_notifications_onDeleteURI.js
+++ b/toolkit/components/places/tests/expiration/test_notifications_onDeleteURI.js
@@ -95,17 +95,17 @@ function run_test() {
 function run_next_test() {
   if (gTests.length) {
     gCurrentTest = gTests.shift();
     gTestIndex++;
     print("\nTEST " + gTestIndex + ": " + gCurrentTest.desc);
     gCurrentTest.receivedNotifications = 0;
 
     // Setup visits.
-    let now = Date.now() * 1000;
+    let now = getExpirablePRTime();
     for (let i = 0; i < gCurrentTest.addPages; i++) {
       let page = "http://" + gTestIndex + "." + i + ".mozilla.org/";
       hs.addVisit(uri(page), now++, null, hs.TRANSITION_TYPED, false, 0);
     }
 
     // Setup bookmarks.
     gCurrentTest.bookmarks = [];
     for (let i = 0; i < gCurrentTest.addBookmarks; i++) {
--- a/toolkit/components/places/tests/expiration/test_notifications_onDeleteVisits.js
+++ b/toolkit/components/places/tests/expiration/test_notifications_onDeleteVisits.js
@@ -112,17 +112,17 @@ function run_test() {
 function run_next_test() {
   if (gTests.length) {
     gCurrentTest = gTests.shift();
     gTestIndex++;
     print("\nTEST " + gTestIndex + ": " + gCurrentTest.desc);
     gCurrentTest.receivedNotifications = 0;
 
     // Setup visits.
-    let now = Date.now() * 1000;
+    let now = getExpirablePRTime();
     for (let j = 0; j < gCurrentTest.visitsPerPage; j++) {
       for (let i = 0; i < gCurrentTest.addPages; i++) {
         let page = "http://" + gTestIndex + "." + i + ".mozilla.org/";
         hs.addVisit(uri(page), now++, null, hs.TRANSITION_TYPED, false, 0);
       }
     }
 
     // Setup bookmarks.
--- a/toolkit/components/places/tests/expiration/test_pref_maxpages.js
+++ b/toolkit/components/places/tests/expiration/test_pref_maxpages.js
@@ -116,17 +116,17 @@ function run_test() {
 function run_next_test() {
   if (gTests.length) {
     gCurrentTest = gTests.shift();
     gTestIndex++;
     print("\nTEST " + gTestIndex + ": " + gCurrentTest.desc);
     gCurrentTest.receivedNotifications = 0;
 
     // Setup visits.
-    let now = Date.now() * 1000;
+    let now = getExpirablePRTime();
     for (let i = 0; i < gCurrentTest.addPages; i++) {
       hs.addVisit(uri("http://" + gTestIndex + "." + i + ".mozilla.org/"), now++, null,
                   hs.TRANSITION_TYPED, false, 0);
     }
 
     // Observe history.
     historyObserver = {
       onBeginUpdateBatch: function PEX_onBeginUpdateBatch() {},
--- a/toolkit/components/places/tests/unit/test_history_removeAllPages.js
+++ b/toolkit/components/places/tests/unit/test_history_removeAllPages.js
@@ -85,17 +85,17 @@ let historyObserver = {
         // to -MAX(visit_count, 1), so we will be able to recalculate frecency
         // starting from most frecent bookmarks.
         stmt = mDBConn.createStatement(
           "SELECT h.id FROM moz_places h WHERE h.frecency > 0 ");
         do_check_false(stmt.executeStep());
         stmt.finalize();
 
         stmt = mDBConn.createStatement(
-          "SELECT h.id FROM moz_places h WHERE h.frecency = -2 " +
+          "SELECT h.id FROM moz_places h WHERE h.frecency < 0 " +
             "AND EXISTS (SELECT id FROM moz_bookmarks WHERE fk = h.id) LIMIT 1");
         do_check_true(stmt.executeStep());
         stmt.finalize();
 
         // Check that all visit_counts have been brought to 0
         stmt = mDBConn.createStatement(
           "SELECT id FROM moz_places WHERE visit_count <> 0 LIMIT 1");
         do_check_false(stmt.executeStep());
--- a/toolkit/components/places/tests/unit/test_removeVisitsByTimeframe.js
+++ b/toolkit/components/places/tests/unit/test_removeVisitsByTimeframe.js
@@ -377,18 +377,18 @@ var gTests = [
       print("nsIGlobalHistory2.isVisited should return false.");
       do_check_false(histsvc.QueryInterface(Ci.nsIGlobalHistory2).
                        isVisited(TEST_URI));
 
       print("nsINavBookmarksService.isBookmarked should return true.");
       do_check_true(bmsvc.isBookmarked(TEST_URI));
 
       waitForAsyncUpdates(function () {
-        print("Frecency should be -visit_count.")
-        do_check_eq(frecencyForUrl(TEST_URI), -10);
+        print("Frecency should be negative.")
+        do_check_true(frecencyForUrl(TEST_URI) < 0);
         run_next_test();
       });
     }
   }
 ];
 
 ///////////////////////////////////////////////////////////////////////////////