Bug 857376 - 'Use 2k page_size for databases on B2G'. r=mak, sr=mossop.
authorBen Turner <bent.mozilla@gmail.com>
Thu, 04 Apr 2013 17:14:46 -0700
changeset 138666 d4130427c3130ec27434c0cadb29b8cf12fe0aaa
parent 138665 ba7609e950386e7260b86b93792163f4ef73fae3
child 138667 3e311d250b6eae1a2ca801c17638e625fb0fe565
push id2579
push userakeybl@mozilla.com
push dateMon, 24 Jun 2013 18:52:47 +0000
treeherdermozilla-beta@b69b7de8a05a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmak, mossop
bugs857376
milestone23.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 857376 - 'Use 2k page_size for databases on B2G'. r=mak, sr=mossop.
b2g/app/b2g.js
db/sqlite3/src/Makefile.in
storage/public/mozIStorageConnection.idl
storage/public/mozIStorageVacuumParticipant.idl
storage/src/VacuumManager.cpp
storage/src/mozStorageConnection.cpp
storage/src/mozStorageService.cpp
storage/src/mozStorageService.h
storage/test/unit/test_vacuum.js
storage/test/unit/vacuumParticipant.js
toolkit/components/places/nsNavHistory.cpp
--- a/b2g/app/b2g.js
+++ b/b2g/app/b2g.js
@@ -664,8 +664,13 @@ pref("memory_info_dumper.watch_fifo.enab
 pref("memory_info_dumper.watch_fifo.directory", "/data/local");
 
 pref("general.useragent.enable_overrides", true);
 
 pref("b2g.version", @MOZ_B2G_VERSION@);
 
 // Disable console buffering to save memory.
 pref("consoleservice.buffered", false);
+
+#ifdef MOZ_WIDGET_GONK
+// Performance testing suggests 2k is a better page size for SQLite.
+pref("toolkit.storage.pageSize", 2048);
+#endif
\ No newline at end of file
--- a/db/sqlite3/src/Makefile.in
+++ b/db/sqlite3/src/Makefile.in
@@ -69,18 +69,21 @@ CSRCS = \
   sqlite3.c \
   $(NULL)
 
 # -DSQLITE_SECURE_DELETE=1 will cause SQLITE to 0-fill delete data so we
 # don't have to vacuum to make sure the data is not visible in the file.
 # -DSQLITE_ENABLE_FTS3=1 enables the full-text index module.
 # -DSQLITE_CORE=1 statically links that module into the SQLite library.
 # -DSQLITE_DEFAULT_PAGE_SIZE=32768 and SQLITE_MAX_DEFAULT_PAGE_SIZE=32768
-# increases the page size from 1k, see bug 416330.  The value must stay in sync
-# with mozIStorageConnection::DEFAULT_PAGE_SIZE.
+# increases the page size from 1k, see bug 416330.  It must be kept in sync with
+# the value of PREF_TS_PAGESIZE_DEFAULT in mozStorageService.cpp.  The value can
+# be overridden on a per-platform basis through the use of the PREF_TS_PAGESIZE
+# hidden preference.  If that preference is missing or invalid then this value
+# will be used.
 # -DSQLITE_MAX_SCHEMA_RETRY increases the times SQLite may try to reparse
 # statements when the schema changes. This is important when supporting lots of
 # concurrent connections, especially when they use shared cache.
 # Note: Be sure to update the configure.in checks when these change!
 DEFINES = \
   -DSQLITE_SECURE_DELETE=1 \
   -DSQLITE_THREADSAFE=1 \
   -DSQLITE_CORE=1 \
--- a/storage/public/mozIStorageConnection.idl
+++ b/storage/public/mozIStorageConnection.idl
@@ -20,27 +20,19 @@ interface nsIFile;
  * mozIStorageConnection represents a database connection attached to
  * a specific file or to the in-memory data storage.  It is the
  * primary interface for interacting with a database, including
  * creating prepared statements, executing SQL, and examining database
  * errors.
  *
  * @threadsafe
  */
-[scriptable, uuid(b2a4b534-f92e-4387-9bd9-d10408173925)]
+[scriptable, uuid(c8646e4b-3e2d-4df3-98a9-e2bbf74f279c)]
 interface mozIStorageConnection : nsISupports {
   /**
-   * The default size for SQLite database pages used by mozStorage for new
-   * databases.
-   * This value must stay in sync with the SQLITE_DEFAULT_PAGE_SIZE define in
-   * /db/sqlite3/src/Makefile.in
-   */
-  const long DEFAULT_PAGE_SIZE = 32768;
-
-  /**
    * Closes a database connection.  Callers must finalize all statements created
    * for this connection prior to calling this method.  It is illegal to use
    * call this method if any asynchronous statements have been executed on this
    * connection.
    *
    * @throws NS_ERROR_UNEXPECTED
    *         If any statement has been executed asynchronously on this object.
    * @throws NS_ERROR_UNEXPECTED
@@ -85,16 +77,22 @@ interface mozIStorageConnection : nsISup
    * @param aReadOnly
    *        If true, the returned database should be put into read-only mode.
    *        Defaults to false.
    * @return the cloned database connection.
    */
   mozIStorageConnection clone([optional] in boolean aReadOnly);
 
   /**
+   * The default size for SQLite database pages used by mozStorage for new
+   * databases.
+   */
+  readonly attribute long defaultPageSize;
+
+  /**
    * Indicates if the connection is open and ready to use.  This will be false
    * if the connection failed to open, or it has been closed.
    */
   readonly attribute boolean connectionReady;
 
   /**
    * The current database nsIFile.  Null if the database
    * connection refers to an in-memory database.
--- a/storage/public/mozIStorageVacuumParticipant.idl
+++ b/storage/public/mozIStorageVacuumParticipant.idl
@@ -18,18 +18,18 @@ interface mozIStorageConnection;
 [scriptable, uuid(8f367508-1d9a-4d3f-be0c-ac11b6dd7dbf)]
 interface mozIStorageVacuumParticipant : nsISupports {
   /**
    * The expected page size in bytes for the database.  The vacuum manager will
    * try to correct the page size during idle based on this value.
    *
    * @note If the database is using the WAL journal mode, the page size won't
   *        be changed to the requested value.  See bug 634374.
-   * @note Valid page size values are from 512 to 65536.
-   *       The suggested value is mozIStorageConnection::DEFAULT_PAGE_SIZE.
+   * @note Valid page size values are powers of 2 between 512 and 65536.
+   *       The suggested value is mozIStorageConnection::defaultPageSize.
    */
   readonly attribute long expectedDatabasePageSize;
 
   /**
    * Connection to the database file to be vacuumed.
    */
   readonly attribute mozIStorageConnection databaseConnection;
 
--- a/storage/src/VacuumManager.cpp
+++ b/storage/src/VacuumManager.cpp
@@ -140,20 +140,20 @@ Vacuumer::execute()
     return false;
   }
 
   // Ask for the expected page size.  Vacuum can change the page size, unless
   // the database is using WAL journaling.
   // TODO Bug 634374: figure out a strategy to fix page size with WAL.
   int32_t expectedPageSize = 0;
   rv = mParticipant->GetExpectedDatabasePageSize(&expectedPageSize);
-  if (NS_FAILED(rv) || expectedPageSize < 512 || expectedPageSize > 65536) {
+  if (NS_FAILED(rv) || !Service::pageSizeIsValid(expectedPageSize)) {
     NS_WARNING("Invalid page size requested for database, will use default ");
     NS_WARNING(mDBFilename.get());
-    expectedPageSize = mozIStorageConnection::DEFAULT_PAGE_SIZE;
+    expectedPageSize = Service::getDefaultPageSize();
   }
 
   // Get the database filename.  Last vacuum time is stored under this name
   // in PREF_VACUUM_BRANCH.
   nsCOMPtr<nsIFile> databaseFile;
   mDBConn->GetDatabaseFile(getter_AddRefs(databaseFile));
   if (!databaseFile) {
     NS_WARNING("Trying to vacuum a in-memory database!");
--- a/storage/src/mozStorageConnection.cpp
+++ b/storage/src/mozStorageConnection.cpp
@@ -557,20 +557,21 @@ Connection::initializeInternal(nsIFile* 
 
   nsAutoCString leafName(":memory");
   if (aDatabaseFile)
     (void)aDatabaseFile->GetNativeLeafName(leafName);
   PR_LOG(gStorageLog, PR_LOG_NOTICE, ("Opening connection to '%s' (%p)",
                                       leafName.get(), this));
 #endif
 
+  int64_t pageSize = Service::getDefaultPageSize();
+
   // Set page_size to the preferred default value.  This is effective only if
   // the database has just been created, otherwise, if the database does not
   // use WAL journal mode, a VACUUM operation will updated its page_size.
-  int64_t pageSize = DEFAULT_PAGE_SIZE;
   nsAutoCString pageSizeQuery(MOZ_STORAGE_UNIQUIFY_QUERY_STR
                               "PRAGMA page_size = ");
   pageSizeQuery.AppendInt(pageSize);
   nsresult rv = ExecuteSimpleSQL(pageSizeQuery);
   NS_ENSURE_SUCCESS(rv, rv);
 
   // Setting the cache_size forces the database open, verifying if it is valid
   // or corrupt.  So this is executed regardless it being actually needed.
@@ -1040,16 +1041,23 @@ Connection::Clone(bool aReadOnly,
   SQLiteMutexAutoLock lockedScope(sharedDBMutex);
   (void)mFunctions.EnumerateRead(copyFunctionEnumerator, clone);
 
   NS_ADDREF(*_connection = clone);
   return NS_OK;
 }
 
 NS_IMETHODIMP
+Connection::GetDefaultPageSize(int32_t *_defaultPageSize)
+{
+  *_defaultPageSize = Service::getDefaultPageSize();
+  return NS_OK;
+}
+
+NS_IMETHODIMP
 Connection::GetConnectionReady(bool *_ready)
 {
   *_ready = ConnectionReady();
   return NS_OK;
 }
 
 NS_IMETHODIMP
 Connection::GetDatabaseFile(nsIFile **_dbFile)
--- a/storage/src/mozStorageService.cpp
+++ b/storage/src/mozStorageService.cpp
@@ -35,16 +35,22 @@
 #include "nsIMemoryReporter.h"
 
 ////////////////////////////////////////////////////////////////////////////////
 //// Defines
 
 #define PREF_TS_SYNCHRONOUS "toolkit.storage.synchronous"
 #define PREF_TS_SYNCHRONOUS_DEFAULT 1
 
+#define PREF_TS_PAGESIZE "toolkit.storage.pageSize"
+
+// This value must be kept in sync with the value of SQLITE_DEFAULT_PAGE_SIZE in
+// db/sqlite3/src/Makefile.in.
+#define PREF_TS_PAGESIZE_DEFAULT 32768
+
 namespace mozilla {
 namespace storage {
 
 ////////////////////////////////////////////////////////////////////////////////
 //// Memory Reporting
 
 static int64_t
 GetStorageSQLiteMemoryUsed()
@@ -225,27 +231,31 @@ NS_IMPL_THREADSAFE_ISUPPORTS1(
 //// Helpers
 
 class ServiceMainThreadInitializer : public nsRunnable
 {
 public:
   ServiceMainThreadInitializer(Service *aService,
                                nsIObserver *aObserver,
                                nsIXPConnect **aXPConnectPtr,
-                               int32_t *aSynchronousPrefValPtr)
+                               int32_t *aSynchronousPrefValPtr,
+                               int32_t *aPageSizePtr)
   : mService(aService)
   , mObserver(aObserver)
   , mXPConnectPtr(aXPConnectPtr)
   , mSynchronousPrefValPtr(aSynchronousPrefValPtr)
+  , mPageSizePtr(aPageSizePtr)
   {
   }
 
   NS_IMETHOD Run()
   {
     NS_PRECONDITION(NS_IsMainThread(), "Must be running on the main thread!");
+    NS_PRECONDITION(*mPageSizePtr == PREF_TS_PAGESIZE_DEFAULT,
+                    "Must be set to the default value here!");
 
     // NOTE:  All code that can only run on the main thread and needs to be run
     //        during initialization should be placed here.  During the off-
     //        chance that storage is initialized on a background thread, this
     //        will ensure everything that isn't threadsafe is initialized in
     //        the right place.
 
     // Register for xpcom-shutdown so we can cleanup after ourselves.  The
@@ -264,31 +274,42 @@ public:
 
     // We need to obtain the toolkit.storage.synchronous preferences on the main
     // thread because the preference service can only be accessed there.  This
     // is cached in the service for all future Open[Unshared]Database calls.
     int32_t synchronous =
       Preferences::GetInt(PREF_TS_SYNCHRONOUS, PREF_TS_SYNCHRONOUS_DEFAULT);
     ::PR_ATOMIC_SET(mSynchronousPrefValPtr, synchronous);
 
+    // We need to obtain the toolkit.storage.pageSize preferences on the main
+    // thread because the preference service can only be accessed there.  This
+    // is cached in the service for all future Open[Unshared]Database calls.
+    int32_t pageSize =
+      Preferences::GetInt(PREF_TS_PAGESIZE, PREF_TS_PAGESIZE_DEFAULT);
+    if (Service::pageSizeIsValid(pageSize) &&
+        PREF_TS_PAGESIZE_DEFAULT != pageSize) {
+      ::PR_ATOMIC_SET(mPageSizePtr, pageSize);
+    }
+
     // Create and register our SQLite memory reporters.  Registration can only
     // happen on the main thread (otherwise you'll get cryptic crashes).
     mService->mStorageSQLiteReporter = new NS_MEMORY_REPORTER_NAME(StorageSQLite);
     mService->mStorageSQLiteMultiReporter = new StorageSQLiteMultiReporter(mService);
     (void)::NS_RegisterMemoryReporter(mService->mStorageSQLiteReporter);
     (void)::NS_RegisterMemoryMultiReporter(mService->mStorageSQLiteMultiReporter);
 
     return NS_OK;
   }
 
 private:
   Service *mService;
   nsIObserver *mObserver;
   nsIXPConnect **mXPConnectPtr;
   int32_t *mSynchronousPrefValPtr;
+  int32_t *mPageSizePtr;
 };
 
 ////////////////////////////////////////////////////////////////////////////////
 //// Service
 
 NS_IMPL_THREADSAFE_ISUPPORTS2(
   Service,
   mozIStorageService,
@@ -355,16 +376,18 @@ int32_t Service::sSynchronousPref;
 
 // static
 int32_t
 Service::getSynchronousPref()
 {
   return sSynchronousPref;
 }
 
+int32_t Service::sDefaultPageSize = PREF_TS_PAGESIZE_DEFAULT;
+
 Service::Service()
 : mMutex("Service::mMutex")
 , mSqliteVFS(nullptr)
 , mRegistrationMutex("Service::mRegistrationMutex")
 , mConnections()
 , mStorageSQLiteReporter(nullptr)
 , mStorageSQLiteMultiReporter(nullptr)
 {
@@ -577,17 +600,18 @@ Service::initialize()
   }
 
   // Set the default value for the toolkit.storage.synchronous pref.  It will be
   // updated with the user preference on the main thread.
   sSynchronousPref = PREF_TS_SYNCHRONOUS_DEFAULT;
 
   // Run the things that need to run on the main thread there.
   nsCOMPtr<nsIRunnable> event =
-    new ServiceMainThreadInitializer(this, this, &sXPConnect, &sSynchronousPref);
+    new ServiceMainThreadInitializer(this, this, &sXPConnect,
+                                     &sSynchronousPref, &sDefaultPageSize);
   if (event && ::NS_IsMainThread()) {
     (void)event->Run();
   }
   else {
     (void)::NS_DispatchToMainThread(event);
   }
 
   return NS_OK;
--- a/storage/src/mozStorageService.h
+++ b/storage/src/mozStorageService.h
@@ -64,16 +64,37 @@ public:
   static already_AddRefed<nsIXPConnect> getXPConnect();
 
   /**
    * Obtains the cached data for the toolkit.storage.synchronous preference.
    */
   static int32_t getSynchronousPref();
 
   /**
+   * Obtains the default page size for this platform. The default value is
+   * specified in the SQLite makefile (SQLITE_DEFAULT_PAGE_SIZE) but it may be
+   * overriden with the PREF_TS_PAGESIZE hidden preference.
+   */
+  static int32_t getDefaultPageSize()
+  {
+    return sDefaultPageSize;
+  }
+
+  /**
+   * Returns a boolean value indicating whether or not the given page size is
+   * valid (currently understood as a power of 2 between 512 and 65536).
+   */
+  static bool pageSizeIsValid(int32_t aPageSize)
+  {
+    return aPageSize == 512 || aPageSize == 1024 || aPageSize == 2048 ||
+           aPageSize == 4096 || aPageSize == 8192 || aPageSize == 16384 ||
+           aPageSize == 32768 || aPageSize == 65536;
+  }
+
+  /**
    * Registers the connection with the storage service.  Connections are
    * registered so they can be iterated over.
    *
    * @pre mRegistrationMutex is not held
    *
    * @param  aConnection
    *         The connection to register.
    */
@@ -155,16 +176,17 @@ private:
   nsCOMPtr<nsIMemoryReporter> mStorageSQLiteReporter;
   nsCOMPtr<nsIMemoryMultiReporter> mStorageSQLiteMultiReporter;
 
   static Service *gService;
 
   static nsIXPConnect *sXPConnect;
 
   static int32_t sSynchronousPref;
+  static int32_t sDefaultPageSize;
 
   friend class ServiceMainThreadInitializer;
 };
 
 } // namespace storage
 } // namespace mozilla
 
 #endif /* MOZSTORAGESERVICE_H */
--- a/storage/test/unit/test_vacuum.js
+++ b/storage/test/unit/test_vacuum.js
@@ -177,17 +177,17 @@ function test_page_size_change()
 
   // We did setup the database with a small page size, the previous vacuum
   // should have updated it.
   print("Check that page size was updated.");
   let conn = getDatabase(new_db_file("testVacuum"));
   let stmt = conn.createStatement("PRAGMA page_size");
   try {
     while (stmt.executeStep()) {
-      do_check_eq(stmt.row.page_size,  Ci.mozIStorageConnection.DEFAULT_PAGE_SIZE);
+      do_check_eq(stmt.row.page_size,  conn.defaultPageSize);
     }
   }
   finally {
     stmt.finalize();
   }
 
   run_next_test();
 },
--- a/storage/test/unit/vacuumParticipant.js
+++ b/storage/test/unit/vacuumParticipant.js
@@ -38,17 +38,17 @@ function vacuumParticipant()
 }
 
 vacuumParticipant.prototype =
 {
   classDescription: "vacuumParticipant",
   classID: Components.ID("{52aa0b22-b82f-4e38-992a-c3675a3355d2}"),
   contractID: "@unit.test.com/test-vacuum-participant;1",
 
-  get expectedDatabasePageSize() Ci.mozIStorageConnection.DEFAULT_PAGE_SIZE,
+  get expectedDatabasePageSize() this._dbConn.defaultPageSize,
   get databaseConnection() this._dbConn,
 
   _grant: true,
   onBeginVacuum: function TVP_onBeginVacuum()
   {
     if (!this._grant) {
       this._grant = true;
       return false;
--- a/toolkit/components/places/nsNavHistory.cpp
+++ b/toolkit/components/places/nsNavHistory.cpp
@@ -2868,18 +2868,19 @@ nsNavHistory::GetDatabaseConnection(mozI
 {
   return GetDBConnection(_DBConnection);
 }
 
 
 NS_IMETHODIMP
 nsNavHistory::GetExpectedDatabasePageSize(int32_t* _expectedPageSize)
 {
-  *_expectedPageSize = mozIStorageConnection::DEFAULT_PAGE_SIZE;
-  return NS_OK;
+  NS_ENSURE_STATE(mDB);
+  NS_ENSURE_STATE(mDB->MainConn());
+  return mDB->MainConn()->GetDefaultPageSize(_expectedPageSize);
 }
 
 
 NS_IMETHODIMP
 nsNavHistory::OnBeginVacuum(bool* _vacuumGranted)
 {
   // TODO: Check if we have to deny the vacuum in some heavy-load case.
   // We could maybe want to do that during batches?