Bug 581606 - Avoid sqlite fragmentation via SQLITE_FCNTL_CHUNK_SIZE r=asuth sr=shaver a=b6
authorTaras Glek <tglek@mozilla.com>
Wed, 01 Sep 2010 20:35:46 -0500
changeset 51868 1065142ddb2c010ceb6b4f7eefb30101ed785a6c
parent 51867 45d8f81c82cf5365f70d61d67b1cace17b15b45b
child 51869 5a051c65160e8c6a00d30d11f174d27a7cbba97d
push id1
push userroot
push dateTue, 26 Apr 2011 22:38:44 +0000
treeherdermozilla-beta@bfdb6e623a36 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersasuth, shaver, b6
bugs581606
milestone2.0b6pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 581606 - Avoid sqlite fragmentation via SQLITE_FCNTL_CHUNK_SIZE r=asuth sr=shaver a=b6
netwerk/cookie/nsCookieService.cpp
storage/public/mozIStorageConnection.idl
storage/src/mozStorageConnection.cpp
storage/test/unit/test_chunk_growth.js
toolkit/components/places/src/nsNavHistory.cpp
toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp
--- a/netwerk/cookie/nsCookieService.cpp
+++ b/netwerk/cookie/nsCookieService.cpp
@@ -730,16 +730,19 @@ nsCookieService::TryInitDB(PRBool aDelet
   }
 
   // open a connection to the cookie database, and only cache our connection
   // and statements upon success. The connection is opened shared such that
   // the main and background threads can operate on the db concurrently.
   rv = mStorageService->OpenDatabase(cookieFile, getter_AddRefs(mDBState->dbConn));
   NS_ENSURE_SUCCESS(rv, rv);
 
+  // Grow cookie db in 512KB increments
+  mDBState->dbConn->SetGrowthIncrement(512 * 1024, EmptyCString());
+
   PRBool tableExists = PR_FALSE;
   mDBState->dbConn->TableExists(NS_LITERAL_CSTRING("moz_cookies"), &tableExists);
   if (!tableExists) {
       rv = CreateTable();
       NS_ENSURE_SUCCESS(rv, rv);
 
   } else {
     // table already exists; check the schema version before reading
--- a/storage/public/mozIStorageConnection.idl
+++ b/storage/public/mozIStorageConnection.idl
@@ -56,17 +56,17 @@ interface nsIFile;
  * mozIStorageConnection represents a database connection attached to
  * a specific file or to the in-memory data storage.  It is the
  * primary interface for interacting with a database, including
  * creating prepared statements, executing SQL, and examining database
  * errors.
  *
  * @threadsafe
  */
-[scriptable, uuid(7b5ed315-58b3-41fd-8257-d5768943021d)]
+[scriptable, uuid(ad035628-4ffb-42ff-a256-0ed9e410b859)]
 interface mozIStorageConnection : nsISupports {
   /**
    * Closes a database connection.  Callers must finalize all statements created
    * for this connection prior to calling this method.  It is illegal to use
    * call this method if any asynchronous statements have been executed on this
    * connection.
    *
    * @throws NS_ERROR_UNEXPECTED
@@ -345,9 +345,21 @@ interface mozIStorageConnection : nsISup
                                                 in mozIStorageProgressHandler aHandler);
 
   /**
    * Remove a progress handler.
    *
    * @return previous registered handler.
    */
   mozIStorageProgressHandler removeProgressHandler();
+
+  /**
+   * Controls SQLITE_FCNTL_CHUNK_SIZE setting in sqlite. This helps avoid fragmentation
+   * by growing/shrinking the database file in SQLITE_FCNTL_CHUNK_SIZE increments.
+   *
+   * @param aIncrement
+   *        The database file will grow in multiples of chunkSize.
+   * @param aDatabaseName
+   *        Sqlite database name. "" means pass NULL for zDbName to sqlite3_file_control.
+   *        See http://sqlite.org/c3ref/file_control.html for more details.
+   */
+  void setGrowthIncrement(in PRInt32 aIncrement, in AUTF8String aDatabaseName);
 };
--- a/storage/src/mozStorageConnection.cpp
+++ b/storage/src/mozStorageConnection.cpp
@@ -1100,10 +1100,20 @@ Connection::RemoveProgressHandler(mozISt
   NS_IF_ADDREF(*_oldHandler = mProgressHandler);
 
   mProgressHandler = nsnull;
   ::sqlite3_progress_handler(mDBConn, 0, NULL, NULL);
 
   return NS_OK;
 }
 
+NS_IMETHODIMP
+Connection::SetGrowthIncrement(PRInt32 aChunkSize, const nsACString &aDatabaseName)
+{
+  (void)::sqlite3_file_control(mDBConn,
+                               aDatabaseName.Length() ? nsPromiseFlatCString(aDatabaseName).get() : NULL,
+                               SQLITE_FCNTL_CHUNK_SIZE,
+                               &aChunkSize);
+  return NS_OK;
+}
+
 } // namespace storage
 } // namespace mozilla
new file mode 100644
--- /dev/null
+++ b/storage/test/unit/test_chunk_growth.js
@@ -0,0 +1,50 @@
+/* Any copyright is dedicated to the Public Domain.
+ * http://creativecommons.org/publicdomain/zero/1.0/
+ */
+
+// This file tests SQLITE_FCNTL_CHUNK_SIZE behaves as expected
+
+function run_sql(d, sql) {
+  var stmt = d.createStatement(sql)
+  stmt.execute()
+  stmt.finalize();
+}
+
+function new_file(name)
+{
+  var file = dirSvc.get("ProfD", Ci.nsIFile);
+  file.append(name);
+  return file;
+}
+
+function get_size(name) {
+  return new_file(name).fileSize
+}
+
+function run_test()
+{
+  const filename = "chunked.sqlite";
+  const CHUNK_SIZE = 512 * 1024;
+  var d = getDatabase(new_file(filename));
+  d.setGrowthIncrement(CHUNK_SIZE, "");
+  run_sql(d, "CREATE TABLE bloat(data varchar)");
+
+  var orig_size = get_size(filename);
+  /* Dump in at least 32K worth of data.
+   * While writing ensure that the file size growth in chunksize set above.
+   */
+  const str1024 = new Array(1024).join("T");
+  for(var i = 0; i < 32; i++) {
+    run_sql(d, "INSERT INTO bloat VALUES('" + str1024 + "')");
+    var size = get_size(filename)
+    // Must not grow in small increments.
+    do_check_true(size == orig_size || size >= CHUNK_SIZE);
+  }
+  /* In addition to growing in chunk-size increments, the db
+   * should shrink in chunk-size increments too.
+   */
+  run_sql(d, "DELETE FROM bloat")
+  run_sql(d, "VACUUM")
+  do_check_true(get_size(filename) >= CHUNK_SIZE)
+}
+
--- a/toolkit/components/places/src/nsNavHistory.cpp
+++ b/toolkit/components/places/src/nsNavHistory.cpp
@@ -704,16 +704,19 @@ nsNavHistory::InitDB()
   rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
       "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE));
   NS_ENSURE_SUCCESS(rv, rv);
 
   // We are going to initialize tables, so everything from now on should be in
   // a transaction for performances.
   mozStorageTransaction transaction(mDBConn, PR_FALSE);
 
+  // Grow places in 10MB increments
+  mDBConn->SetGrowthIncrement(10 * 1024 * 1024, EmptyCString());
+
   // Initialize the other Places services' database tables before creating our
   // statements. Some of our statements depend on these external tables, such as
   // the bookmarks or favicon tables.
   rv = nsNavBookmarks::InitTables(mDBConn);
   NS_ENSURE_SUCCESS(rv, rv);
   rv = nsFaviconService::InitTables(mDBConn);
   NS_ENSURE_SUCCESS(rv, rv);
   rv = nsAnnotationService::InitTables(mDBConn);
--- a/toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp
+++ b/toolkit/components/url-classifier/src/nsUrlClassifierDBService.cpp
@@ -3413,16 +3413,17 @@ nsUrlClassifierDBServiceWorker::OpenDb()
 
       newDB = PR_TRUE;
 
       rv = storageService->OpenDatabase(mDBFile, getter_AddRefs(connection));
       NS_ENSURE_SUCCESS(rv, rv);
     }
   }
 
+  connection->SetGrowthIncrement(5 * 1024 * 1024, EmptyCString());
   rv = connection->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous=OFF"));
   NS_ENSURE_SUCCESS(rv, rv);
 
   if (newDB) {
     rv = connection->SetSchemaVersion(IMPLEMENTATION_VERSION);
     NS_ENSURE_SUCCESS(rv, rv);
   }