Bug 1356220 - Set a journal_size_limit on favicons.sqlite and improve the wal autocheckpoint value. r=adw draft
authorMarco Bonardo <mbonardo@mozilla.com>
Fri, 21 Apr 2017 18:43:29 +0200
changeset 566469 dd3723d948f3cdec31109292e10ff0205d719aad
parent 566156 8b854986038cf3f3f240697e27ef48ea65914c13
child 625327 6072fd908060172d5f20c80434da225d9822fb93
push id55228
push usermak77@bonardo.net
push dateFri, 21 Apr 2017 16:49:11 +0000
reviewersadw
bugs1356220
milestone55.0a1
Bug 1356220 - Set a journal_size_limit on favicons.sqlite and improve the wal autocheckpoint value. r=adw journal_size_limit doesn't apply to attached databases, so it must be set apart. Additionally, the current journal_size_limit setting is wrong it should be bytes. Finally, increase the wal_autocheckpoint value for performance reasons. Sqlite by default uses a 4MiB autocheckpoint, but we use synchronous=NORMAL, that is a bit more dataloss risky. For that reason just use half of the default value. journal_size_limit is set higher than the autocheckpoint value, so that we won't truncate at every checkpoint. MozReview-Commit-ID: 2ZUy2Iwkjjc
toolkit/components/places/Database.cpp
--- a/toolkit/components/places/Database.cpp
+++ b/toolkit/components/places/Database.cpp
@@ -69,20 +69,29 @@
 // * IE didn't support urls longer than 2083 chars
 // * Sitemaps protocol used to support a maximum of 2048 chars
 // * Various SEO guides suggest to not go over 2000 chars
 // * Various apps/services are known to have issues over 2000 chars
 // * RFC 2616 - HTTP/1.1 suggests being cautious about depending
 //   on URI lengths above 255 bytes
 #define PREF_HISTORY_MAXURLLEN_DEFAULT 2000
 
-// Maximum size for the WAL file.  It should be small enough since in case of
-// crashes we could lose all the transactions in the file.  But a too small
-// file could hurt performance.
-#define DATABASE_MAX_WAL_SIZE_IN_KIBIBYTES 512
+// Maximum size for the WAL file.
+// For performance reasons this should be as large as possible, so that more
+// transactions can fit into it, and the checkpoint cost is paid less often.
+// At the same time, since we use synchronous = NORMAL, an fsync happens only
+// at checkpoint time, so we don't want the WAL to grow too much and risk to
+// lose all the contained transactions on a crash.
+#define DATABASE_MAX_WAL_BYTES 2048000
+
+// Since exceeding the journal limit will cause a truncate, we allow a slightly
+// larger limit than DATABASE_MAX_WAL_BYTES to reduce the number of truncates.
+// This is the number of bytes the journal can grow over the maximum wal size
+// before being truncated.
+#define DATABASE_JOURNAL_OVERHEAD_BYTES 2048000
 
 #define BYTES_PER_KIBIBYTE 1024
 
 // How much time Sqlite can wait before returning a SQLITE_BUSY error.
 #define DATABASE_BUSY_TIMEOUT_MS 100
 
 // Old Sync GUID annotation.
 #define SYNCGUID_ANNO NS_LITERAL_CSTRING("sync/guid")
@@ -323,22 +332,20 @@ SetupDurability(nsCOMPtr<mozIStorageConn
     SetJournalMode(aDBConn, JOURNAL_MEMORY);
     rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
       "PRAGMA synchronous = OFF"));
     NS_ENSURE_SUCCESS(rv, rv);
   } else {
     // Be sure to set journal mode after page_size.  WAL would prevent the change
     // otherwise.
     if (JOURNAL_WAL == SetJournalMode(aDBConn, JOURNAL_WAL)) {
-      // Set the WAL journal size limit.  We want it to be small, since in
-      // synchronous = NORMAL mode a crash could cause loss of all the
-      // transactions in the journal.  For added safety we will also force
-      // checkpointing at strategic moments.
+      // Set the WAL journal size limit.
+      // For added safety we will also force checkpointing at strategic moments.
       int32_t checkpointPages =
-        static_cast<int32_t>(DATABASE_MAX_WAL_SIZE_IN_KIBIBYTES * 1024 / aDBPageSize);
+        static_cast<int32_t>(DATABASE_MAX_WAL_BYTES / aDBPageSize);
       nsAutoCString checkpointPragma("PRAGMA wal_autocheckpoint = ");
       checkpointPragma.AppendInt(checkpointPages);
       rv = aDBConn->ExecuteSimpleSQL(checkpointPragma);
       NS_ENSURE_SUCCESS(rv, rv);
     }
     else {
       // Ignore errors, if we fail here the database could be considered corrupt
       // and we won't be able to go on, even if it's just matter of a bogus file
@@ -349,23 +356,19 @@ SetupDurability(nsCOMPtr<mozIStorageConn
       // case of crashes or unclean shutdowns.
       rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING(
           "PRAGMA synchronous = FULL"));
       NS_ENSURE_SUCCESS(rv, rv);
     }
   }
 
   // The journal is usually free to grow for performance reasons, but it never
-  // shrinks back.  Since the space taken may be problematic, especially on
-  // mobile devices, limit its size.
-  // Since exceeding the limit will cause a truncate, allow a slightly
-  // larger limit than DATABASE_MAX_WAL_SIZE_IN_KIBIBYTES to reduce the number
-  // of times it is needed.
+  // shrinks back.  Since the space taken may be problematic, limit its size.
   nsAutoCString journalSizePragma("PRAGMA journal_size_limit = ");
-  journalSizePragma.AppendInt(DATABASE_MAX_WAL_SIZE_IN_KIBIBYTES * 3);
+  journalSizePragma.AppendInt(DATABASE_MAX_WAL_BYTES + DATABASE_JOURNAL_OVERHEAD_BYTES);
   (void)aDBConn->ExecuteSimpleSQL(journalSizePragma);
 
   // Grow places in |growthIncrementKiB| increments to limit fragmentation on disk.
   // By default, it's 5 MB.
   int32_t growthIncrementKiB =
     Preferences::GetInt(PREF_GROWTH_INCREMENT_KIB, 5 * BYTES_PER_KIBIBYTE);
   if (growthIncrementKiB > 0) {
     (void)aDBConn->SetGrowthIncrement(growthIncrementKiB * BYTES_PER_KIBIBYTE, EmptyCString());
@@ -387,16 +390,21 @@ AttachFaviconsDatabase(nsCOMPtr<mozIStor
 
   rv = aDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING("ATTACH DATABASE '") +
     NS_ConvertUTF16toUTF8(path) + NS_LITERAL_CSTRING("' AS favicons"));
   NS_ENSURE_SUCCESS(rv, rv);
 
   rv = aDBConn->ExecuteSimpleSQL(CREATE_ICONS_AFTERINSERT_TRIGGER);
   NS_ENSURE_SUCCESS(rv, rv);
 
+  // The journal limit must be set apart for each database.
+  nsAutoCString journalSizePragma("PRAGMA favicons.journal_size_limit = ");
+  journalSizePragma.AppendInt(DATABASE_MAX_WAL_BYTES + DATABASE_JOURNAL_OVERHEAD_BYTES);
+  Unused << aDBConn->ExecuteSimpleSQL(journalSizePragma);
+
   return NS_OK;
 }
 
 } // namespace
 
 ////////////////////////////////////////////////////////////////////////////////
 //// Database