Bug 1356220 - Set a journal_size_limit on favicons.sqlite and improve the wal autocheckpoint value. r=adw
authorMarco Bonardo <mbonardo@mozilla.com>
Fri, 21 Apr 2017 18:43:29 +0200
changeset 354457 4719f83be95240031a6e748eecab2934e8fccae2
parent 354456 f7ea426c7ad1be745db6e5ec1cc82f7fc5e9a271
child 354458 58e7ff564f30818956c95995ddbe3eddf1e7bb54
push id31701
push userihsiao@mozilla.com
push dateMon, 24 Apr 2017 03:33:36 +0000
treeherdermozilla-central@933ced9c93e9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersadw
bugs1356220
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1356220 - Set a journal_size_limit on favicons.sqlite and improve the wal autocheckpoint value. r=adw journal_size_limit doesn't apply to attached databases, so it must be set apart. Additionally, the current journal_size_limit setting is wrong it should be bytes. Finally, increase the wal_autocheckpoint value for performance reasons. Sqlite by default uses a 4MiB autocheckpoint, but we use synchronous=NORMAL, that is a bit more dataloss risky. For that reason just use half of the default value. journal_size_limit is set higher than the autocheckpoint value, so that we won't truncate at every checkpoint. MozReview-Commit-ID: 2ZUy2Iwkjjc
toolkit/components/places/Database.cpp
--- a/toolkit/components/places/Database.cpp
+++ b/toolkit/components/places/Database.cpp
@@ -69,20 +69,29 @@
 // * IE didn't support urls longer than 2083 chars
 // * Sitemaps protocol used to support a maximum of 2048 chars
 // * Various SEO guides suggest to not go over 2000 chars
 // * Various apps/services are known to have issues over 2000 chars
 // * RFC 2616 - HTTP/1.1 suggests being cautious about depending
 //   on URI lengths above 255 bytes
 #define PREF_HISTORY_MAXURLLEN_DEFAULT 2000
 
-// Maximum size for the WAL file.  It should be small enough since in case of
-// crashes we could lose all the transactions in the file.  But a too small
-// file could hurt performance.
-#define DATABASE_MAX_WAL_SIZE_IN_KIBIBYTES 512
+// Maximum size for the WAL file.
+// For performance reasons this should be as large as possible, so that more
+// transactions can fit into it, and the checkpoint cost is paid less often.
+// At the same time, since we use synchronous = NORMAL, an fsync happens only
+// at checkpoint time, so we don't want the WAL to grow too much and risk to
+// lose all the contained transactions on a crash.
+#define DATABASE_MAX_WAL_BYTES 2048000
+
+// Since exceeding the journal limit will cause a truncate, we allow a slightly
+// larger limit than DATABASE_MAX_WAL_BYTES to reduce the number of truncates.
+// This is the number of bytes the journal can grow over the maximum wal size
+// before being truncated.
+#define DATABASE_JOURNAL_OVERHEAD_BYTES 2048000
 
 #define BYTES_PER_KIBIBYTE 1024
 
 // How much time Sqlite can wait before returning a SQLITE_BUSY error.
 #define DATABASE_BUSY_TIMEOUT_MS 100
 
 // Old Sync GUID annotation.
 #define SYNCGUID_ANNO NS_LITERAL_CSTRING("sync/guid")
@@ -323,22 +332,20 @@ SetupDurability(nsCOMPtr<mozIStorageConn
     SetJournalMode(aDBConn, JOURNAL_MEMORY);
     rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
       "PRAGMA synchronous = OFF"));
     NS_ENSURE_SUCCESS(rv, rv);
   } else {
     // Be sure to set journal mode after page_size.  WAL would prevent the change
     // otherwise.
     if (JOURNAL_WAL == SetJournalMode(aDBConn, JOURNAL_WAL)) {
-      // Set the WAL journal size limit.  We want it to be small, since in
-      // synchronous = NORMAL mode a crash could cause loss of all the
-      // transactions in the journal.  For added safety we will also force
-      // checkpointing at strategic moments.
+      // Set the WAL journal size limit.
+      // For added safety we will also force checkpointing at strategic moments.
       int32_t checkpointPages =
-        static_cast<int32_t>(DATABASE_MAX_WAL_SIZE_IN_KIBIBYTES * 1024 / aDBPageSize);
+        static_cast<int32_t>(DATABASE_MAX_WAL_BYTES / aDBPageSize);
       nsAutoCString checkpointPragma("PRAGMA wal_autocheckpoint = ");
       checkpointPragma.AppendInt(checkpointPages);
       rv = aDBConn->ExecuteSimpleSQL(checkpointPragma);
       NS_ENSURE_SUCCESS(rv, rv);
     }
     else {
       // Ignore errors, if we fail here the database could be considered corrupt
       // and we won't be able to go on, even if it's just matter of a bogus file
@@ -349,23 +356,19 @@ SetupDurability(nsCOMPtr<mozIStorageConn
       // case of crashes or unclean shutdowns.
       rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
           "PRAGMA synchronous = FULL"));
       NS_ENSURE_SUCCESS(rv, rv);
     }
   }
 
   // The journal is usually free to grow for performance reasons, but it never
-  // shrinks back.  Since the space taken may be problematic, especially on
-  // mobile devices, limit its size.
-  // Since exceeding the limit will cause a truncate, allow a slightly
-  // larger limit than DATABASE_MAX_WAL_SIZE_IN_KIBIBYTES to reduce the number
-  // of times it is needed.
+  // shrinks back.  Since the space taken may be problematic, limit its size.
   nsAutoCString journalSizePragma("PRAGMA journal_size_limit = ");
-  journalSizePragma.AppendInt(DATABASE_MAX_WAL_SIZE_IN_KIBIBYTES * 3);
+  journalSizePragma.AppendInt(DATABASE_MAX_WAL_BYTES + DATABASE_JOURNAL_OVERHEAD_BYTES);
   (void)aDBConn->ExecuteSimpleSQL(journalSizePragma);
 
   // Grow places in |growthIncrementKiB| increments to limit fragmentation on disk.
   // By default, it's 5 MB.
   int32_t growthIncrementKiB =
     Preferences::GetInt(PREF_GROWTH_INCREMENT_KIB, 5 * BYTES_PER_KIBIBYTE);
   if (growthIncrementKiB > 0) {
     (void)aDBConn->SetGrowthIncrement(growthIncrementKiB * BYTES_PER_KIBIBYTE, EmptyCString());
@@ -387,16 +390,21 @@ AttachFaviconsDatabase(nsCOMPtr<mozIStor
 
   rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("ATTACH DATABASE '") +
     NS_ConvertUTF16toUTF8(path) + NS_LITERAL_CSTRING("' AS favicons"));
   NS_ENSURE_SUCCESS(rv, rv);
 
   rv = aDBConn->ExecuteSimpleSQL(CREATE_ICONS_AFTERINSERT_TRIGGER);
   NS_ENSURE_SUCCESS(rv, rv);
 
+  // The journal limit must be set apart for each database.
+  nsAutoCString journalSizePragma("PRAGMA favicons.journal_size_limit = ");
+  journalSizePragma.AppendInt(DATABASE_MAX_WAL_BYTES + DATABASE_JOURNAL_OVERHEAD_BYTES);
+  Unused << aDBConn->ExecuteSimpleSQL(journalSizePragma);
+
   return NS_OK;
 }
 
 } // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
 //// Database