Bug 669410 - Reduce the SQLite caches' default size in url-classifier. r=dcamp
authorGian-Carlo Pascutto <gpascutto@mozilla.com>
Thu, 08 Sep 2011 22:15:37 +0200
changeset 76777 91ce20b5323f6502420dc3e2029d7e77af2411f1
parent 76776 5e9bb169d5fc5d2a9f54329818b3bc554b3814e9
child 76778 ed57eb1fa5f6f84ee1bab2595de5fc419fba251d
push id3
push userfelipc@gmail.com
push dateFri, 30 Sep 2011 20:09:13 +0000
reviewersdcamp
bugs669410
milestone9.0a1
Bug 669410 - Reduce the SQLite caches' default size in url-classifier. r=dcamp
browser/app/profile/firefox.js
toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp
--- a/browser/app/profile/firefox.js
+++ b/browser/app/profile/firefox.js
@@ -729,29 +729,21 @@ pref("urlclassifier.gethashnoise", 4);
 // The list of tables that use the gethash request to confirm partial results.
 pref("urlclassifier.gethashtables", "goog-phish-shavar,goog-malware-shavar");
 
 // If an urlclassifier table has not been updated in this number of seconds,
 // a gethash request will be forced to check that the result is still in
 // the database.
 pref("urlclassifier.confirm-age", 2700);
 
-#ifdef MOZ_WIDGET_GTK2
-#define RESTRICT_CACHEMAX
-#endif
-#ifdef XP_OS2
-#define RESTRICT_CACHEMAX
-#endif
+// Maximum size of the sqlite3 cache during an update, in bytes
+pref("urlclassifier.updatecachemax", 41943040);
 
-// Maximum size of the sqlite3 cache during an update, in bytes
-#ifdef RESTRICT_CACHEMAX
-pref("urlclassifier.updatecachemax", 104857600);
-#else
-pref("urlclassifier.updatecachemax", -1);
-#endif
+// Maximum size of the sqlite3 cache for lookups, in bytes
+pref("urlclassifier.lookupcachemax", 1048576);
 
 // URL for checking the reason for a malware warning.
 pref("browser.safebrowsing.malware.reportURL", "http://safebrowsing.clients.google.com/safebrowsing/diagnostic?client=%NAME%&hl=%LOCALE%&site=");
 
 #endif
 
 pref("browser.EULA.version", 3);
 pref("browser.rights.version", 3);
--- a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
@@ -157,16 +157,19 @@ static const PRLogModuleInfo *gUrlClassi
 #define GETHASH_TABLES_PREF     "urlclassifier.gethashtables"
 
 #define CONFIRM_AGE_PREF        "urlclassifier.confirm-age"
 #define CONFIRM_AGE_DEFAULT_SEC (45 * 60)
 
 #define UPDATE_CACHE_SIZE_PREF    "urlclassifier.updatecachemax"
 #define UPDATE_CACHE_SIZE_DEFAULT -1
 
+#define LOOKUP_CACHE_SIZE_PREF    "urlclassifier.lookupcachemax"
+#define LOOKUP_CACHE_SIZE_DEFAULT -1
+
 // Amount of time to spend updating before committing and delaying, in
 // seconds.  This is checked after each update stream, so the actual
 // time spent can be higher than this, depending on update stream size.
 #define UPDATE_WORKING_TIME         "urlclassifier.workingtime"
 #define UPDATE_WORKING_TIME_DEFAULT 5
 
 // The amount of time to delay after hitting UPDATE_WORKING_TIME, in
 // seconds.
@@ -186,16 +189,17 @@ nsIThread* nsUrlClassifierDBService::gDb
 
 // Once we've committed to shutting down, don't do work in the background
 // thread.
 static PRBool gShuttingDownThread = PR_FALSE;
 
 static PRInt32 gFreshnessGuarantee = CONFIRM_AGE_DEFAULT_SEC;
 
 static PRInt32 gUpdateCacheSize = UPDATE_CACHE_SIZE_DEFAULT;
+static PRInt32 gLookupCacheSize = LOOKUP_CACHE_SIZE_DEFAULT;
 
 static PRInt32 gWorkingTimeThreshold = UPDATE_WORKING_TIME_DEFAULT;
 static PRInt32 gDelayTime = UPDATE_DELAY_TIME_DEFAULT;
 
 static void
 SplitTables(const nsACString& str, nsTArray<nsCString>& tables)
 {
   tables.Clear();
@@ -1186,16 +1190,20 @@ private:
   // Add entries to the results.
   nsresult AddNoise(PRInt64 nearID,
                     PRInt32 count,
                     nsTArray<nsUrlClassifierLookupResult>& results);
 
   // Construct a Prefix Tree with known prefixes
   nsresult ConstructPrefixTree();
 
+  // Set the SQLite cache size
+  nsresult SetCacheSize(mozIStorageConnection * aConnection,
+                        PRInt32 aCacheSize);
+
   nsCOMPtr<nsIFile> mDBFile;
 
   nsCOMPtr<nsICryptoHash> mCryptoHash;
 
   // Holds a connection to the Db.  We lazily initialize this because it has
   // to be created in the background thread (currently mozStorageConnection
   // isn't thread safe).
   nsCOMPtr<mozIStorageConnection> mConnection;
@@ -3055,43 +3063,56 @@ nsUrlClassifierDBServiceWorker::FinishSt
                                   static_cast<PRUint32>(nextStreamDelay));
 
   ResetStream();
 
   return NS_OK;
 }
 
 nsresult
+nsUrlClassifierDBServiceWorker::SetCacheSize(
+  mozIStorageConnection * aConnection, PRInt32 aCacheSize)
+{
+  mozStorageStatementScoper scoper(mGetPageSizeStatement);
+  PRBool hasResult;
+  nsresult rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
+  PRUint32 pageSize = mGetPageSizeStatement->AsInt32(0);
+  PRUint32 cachePages = aCacheSize / pageSize;
+  nsCAutoString cacheSizePragma("PRAGMA cache_size=");
+  cacheSizePragma.AppendInt(cachePages);
+  rv = aConnection->ExecuteSimpleSQL(cacheSizePragma);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return NS_OK;
+}
+
+nsresult
 nsUrlClassifierDBServiceWorker::SetupUpdate()
 {
   LOG(("nsUrlClassifierDBServiceWorker::SetupUpdate"));
   PRBool inProgress;
   nsresult rv = mConnection->GetTransactionInProgress(&inProgress);
   if (inProgress) {
     return NS_OK;
   }
 
   mUpdateStartTime = PR_IntervalNow();
 
   rv = mConnection->BeginTransaction();
   NS_ENSURE_SUCCESS(rv, rv);
 
   if (gUpdateCacheSize > 0) {
-    PRBool hasResult;
-    rv = mGetPageSizeStatement->ExecuteStep(&hasResult);
+    rv = SetCacheSize(mConnection, gUpdateCacheSize);
     NS_ENSURE_SUCCESS(rv, rv);
-
-    NS_ASSERTION(hasResult, "Should always be able to get page size from sqlite");
-    PRUint32 pageSize = mGetPageSizeStatement->AsInt32(0);
-    PRUint32 cachePages = gUpdateCacheSize / pageSize;
-    nsCAutoString cacheSizePragma("PRAGMA cache_size=");
-    cacheSizePragma.AppendInt(cachePages);
-    rv = mConnection->ExecuteSimpleSQL(cacheSizePragma);
-    NS_ENSURE_SUCCESS(rv, rv);
-    mGrewCache = PR_TRUE;
+    if (gUpdateCacheSize != gLookupCacheSize) {
+      mGrewCache = PR_TRUE;
+    }
   }
 
   return NS_OK;
 }
 
 nsresult
 nsUrlClassifierDBServiceWorker::ApplyUpdate()
 {
@@ -3332,16 +3353,24 @@ nsUrlClassifierDBServiceWorker::OpenDb()
       NS_ENSURE_SUCCESS(rv, rv);
     }
   }
 
   connection->SetGrowthIncrement(5 * 1024 * 1024, EmptyCString());
   rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF"));
   NS_ENSURE_SUCCESS(rv, rv);
 
+  rv = connection->CreateStatement
+    (NS_LITERAL_CSTRING("PRAGMA page_size"),
+     getter_AddRefs(mGetPageSizeStatement));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  rv = SetCacheSize(connection, gLookupCacheSize);
+  NS_ENSURE_SUCCESS(rv, rv);
+
   if (newDB) {
     rv = connection->SetSchemaVersion(IMPLEMENTATION_VERSION);
     NS_ENSURE_SUCCESS(rv, rv);
   }
 
   // Create the table
   rv = MaybeCreateTables(connection);
   NS_ENSURE_SUCCESS(rv, rv);
@@ -3386,21 +3415,16 @@ nsUrlClassifierDBServiceWorker::OpenDb()
   NS_ENSURE_SUCCESS(rv, rv);
 
   rv = connection->CreateStatement
     (NS_LITERAL_CSTRING("INSERT INTO moz_tables(id, name, add_chunks, sub_chunks)"
                         " VALUES (null, ?1, null, null)"),
      getter_AddRefs(mInsertTableIdStatement));
   NS_ENSURE_SUCCESS(rv, rv);
 
-  rv = connection->CreateStatement
-    (NS_LITERAL_CSTRING("PRAGMA page_size"),
-     getter_AddRefs(mGetPageSizeStatement));
-  NS_ENSURE_SUCCESS(rv, rv);
-
   mConnection = connection;
 
   mCryptoHash = do_CreateInstance(NS_CRYPTO_HASH_CONTRACTID, &rv);
   NS_ENSURE_SUCCESS(rv, rv);
 
   LOG(("SB construcing Prefix Tree\n"));
 
   rv = ConstructPrefixTree();
@@ -3923,16 +3947,19 @@ nsUrlClassifierDBService::Init()
     rv = prefs->GetIntPref(CONFIRM_AGE_PREF, &tmpint);
     PR_ATOMIC_SET(&gFreshnessGuarantee, NS_SUCCEEDED(rv) ? tmpint : CONFIRM_AGE_DEFAULT_SEC);
 
     prefs->AddObserver(CONFIRM_AGE_PREF, this, PR_FALSE);
 
     rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
     PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
 
+    rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
+    PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
+
     rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);
     PR_ATOMIC_SET(&gWorkingTimeThreshold,
                   NS_SUCCEEDED(rv) ? tmpint : UPDATE_WORKING_TIME_DEFAULT);
 
     rv = prefs->GetIntPref(UPDATE_DELAY_TIME, &tmpint);
     PR_ATOMIC_SET(&gDelayTime,
                   NS_SUCCEEDED(rv) ? tmpint : UPDATE_DELAY_TIME_DEFAULT);
   }
@@ -4229,16 +4256,20 @@ nsUrlClassifierDBService::Observe(nsISup
     } else if (NS_LITERAL_STRING(CONFIRM_AGE_PREF).Equals(aData)) {
       PRInt32 tmpint;
       rv = prefs->GetIntPref(CONFIRM_AGE_PREF, &tmpint);
       PR_ATOMIC_SET(&gFreshnessGuarantee, NS_SUCCEEDED(rv) ? tmpint : CONFIRM_AGE_DEFAULT_SEC);
     } else if (NS_LITERAL_STRING(UPDATE_CACHE_SIZE_PREF).Equals(aData)) {
       PRInt32 tmpint;
       rv = prefs->GetIntPref(UPDATE_CACHE_SIZE_PREF, &tmpint);
       PR_ATOMIC_SET(&gUpdateCacheSize, NS_SUCCEEDED(rv) ? tmpint : UPDATE_CACHE_SIZE_DEFAULT);
+    } else if (NS_LITERAL_STRING(LOOKUP_CACHE_SIZE_PREF).Equals(aData)) {
+      PRInt32 tmpint;
+      rv = prefs->GetIntPref(LOOKUP_CACHE_SIZE_PREF, &tmpint);
+      PR_ATOMIC_SET(&gLookupCacheSize, NS_SUCCEEDED(rv) ? tmpint : LOOKUP_CACHE_SIZE_DEFAULT);
     } else if (NS_LITERAL_STRING(UPDATE_WORKING_TIME).Equals(aData)) {
       PRInt32 tmpint;
       rv = prefs->GetIntPref(UPDATE_WORKING_TIME, &tmpint);
       PR_ATOMIC_SET(&gWorkingTimeThreshold,
                     NS_SUCCEEDED(rv) ? tmpint : UPDATE_WORKING_TIME_DEFAULT);
     } else if (NS_LITERAL_STRING(UPDATE_DELAY_TIME).Equals(aData)) {
       PRInt32 tmpint;
       rv = prefs->GetIntPref(UPDATE_DELAY_TIME, &tmpint);
--- a/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierPrefixSet.cpp
@@ -101,57 +101,66 @@ nsUrlClassifierPrefixSet::SetPrefixes(co
 
 NS_IMETHODIMP
 nsUrlClassifierPrefixSet::AddPrefixes(const PRUint32 * prefixes, PRUint32 aLength)
 {
   if (aLength == 0) {
     return NS_OK;
   }
 
-  MutexAutoLock lock(mPrefixTreeLock);
+  nsTArray<PRUint32> mNewIndexPrefixes(mIndexPrefixes);
+  nsTArray<PRUint32> mNewIndexStarts(mIndexStarts);
+  nsTArray<PRUint16> mNewDeltas(mDeltas);
 
-  mIndexPrefixes.AppendElement(prefixes[0]);
-  mIndexStarts.AppendElement(mDeltas.Length());
+  mNewIndexPrefixes.AppendElement(prefixes[0]);
+  mNewIndexStarts.AppendElement(mNewDeltas.Length());
 
   PRUint32 numOfDeltas = 0;
   PRUint32 currentItem = prefixes[0];
   for (PRUint32 i = 1; i < aLength; i++) {
     if ((numOfDeltas >= DELTAS_LIMIT) ||
           (prefixes[i] - currentItem >= MAX_INDEX_DIFF)) {
-      mIndexStarts.AppendElement(mDeltas.Length());
-      mIndexPrefixes.AppendElement(prefixes[i]);
+      mNewIndexStarts.AppendElement(mNewDeltas.Length());
+      mNewIndexPrefixes.AppendElement(prefixes[i]);
       numOfDeltas = 0;
     } else {
       PRUint16 delta = prefixes[i] - currentItem;
-      mDeltas.AppendElement(delta);
+      mNewDeltas.AppendElement(delta);
       numOfDeltas++;
     }
     currentItem = prefixes[i];
   }
 
-  mIndexPrefixes.Compact();
-  mIndexStarts.Compact();
-  mDeltas.Compact();
+  mNewIndexPrefixes.Compact();
+  mNewIndexStarts.Compact();
+  mNewDeltas.Compact();
+
+  LOG(("Total number of indices: %d", mNewIndexPrefixes.Length()));
+  LOG(("Total number of deltas: %d", mNewDeltas.Length()));
 
-  LOG(("Total number of indices: %d", mIndexPrefixes.Length()));
-  LOG(("Total number of deltas: %d", mDeltas.Length()));
+  MutexAutoLock lock(mPrefixTreeLock);
+
+  // This just swaps some pointers
+  mIndexPrefixes.SwapElements(mNewIndexPrefixes);
+  mIndexStarts.SwapElements(mNewIndexStarts);
+  mDeltas.SwapElements(mNewDeltas);
 
   mHasPrefixes = PR_TRUE;
   mTreeIsReady.NotifyAll();
 
   return NS_OK;
 }
 
 PRUint32 nsUrlClassifierPrefixSet::BinSearch(PRUint32 start,
-                                              PRUint32 end,
-                                              PRUint32 target)
+                                             PRUint32 end,
+                                             PRUint32 target)
 {
   while (start != end && end >= start) {
-    int i = start + ((end - start) >> 1);
-    int value = mIndexPrefixes[i];
+    PRUint32 i = start + ((end - start) >> 1);
+    PRUint32 value = mIndexPrefixes[i];
     if (value < target) {
       start = i + 1;
     } else if (value > target) {
       end = i - 1;
     } else {
       return i;
     }
   }