author | Marco Bonardo <mbonardo@mozilla.com> |
Sat, 14 Aug 2010 09:57:09 +0200 | |
changeset 50588 | 41d486ae06b26aa29d5bf84f36751e774a4b2281 |
parent 50586 | df37bb06494a08bcac6950a9421b16ec31f611b6 (current diff) |
parent 50587 | 2fa5b630f3bc5e0d4ac7f56606b8e51331763cc2 (diff) |
child 50589 | 802b081c965892281fdd22507aac60be24afcbbb |
push id | 1 |
push user | root |
push date | Tue, 26 Apr 2011 22:38:44 +0000 |
treeherder | mozilla-beta@bfdb6e623a36 [default view] [failures only] |
perfherder | [talos] [build metrics] [platform microbench] (compared to previous push) |
milestone | 2.0b4pre |
first release with | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
last release without | nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
|
--- a/toolkit/components/places/src/Makefile.in +++ b/toolkit/components/places/src/Makefile.in @@ -77,19 +77,16 @@ CPPSRCS = \ EXTRA_DSO_LDOPTS += \ $(DEPTH)/db/morkreader/$(LIB_PREFIX)morkreader_s.$(LIB_SUFFIX) \ $(MOZ_UNICHARUTIL_LIBS) \ $(MOZ_COMPONENT_LIBS) \ $(NULL) LOCAL_INCLUDES += -I$(srcdir)/../../build -# This is the default value. Must be in sync with the one defined in SQLite. -DEFINES += -DSQLITE_DEFAULT_PAGE_SIZE=32768 - EXTRA_COMPONENTS = \ toolkitplaces.manifest \ nsLivemarkService.js \ nsTaggingService.js \ nsPlacesDBFlush.js \ nsPlacesExpiration.js \ nsMicrosummaryService.js \ $(NULL)
--- a/toolkit/components/places/src/nsNavHistory.cpp +++ b/toolkit/components/places/src/nsNavHistory.cpp @@ -60,17 +60,16 @@ #include "prsystem.h" #include "prtime.h" #include "nsEscape.h" #include "nsIEffectiveTLDService.h" #include "nsIClassInfoImpl.h" #include "nsThreadUtils.h" #include "nsAppDirectoryServiceDefs.h" #include "nsMathUtils.h" -#include "mozIStorageAsyncStatement.h" #include "nsNavBookmarks.h" #include "nsAnnotationService.h" #include "nsILivemarkService.h" #include "nsFaviconService.h" #include "nsPlacesTables.h" #include "nsPlacesIndexes.h" @@ -137,16 +136,20 @@ using namespace mozilla::places; #define DATABASE_SCHEMA_VERSION 10 // Filename of the database. #define DATABASE_FILENAME NS_LITERAL_STRING("places.sqlite") // Filename used to backup corrupt databases. #define DATABASE_CORRUPT_FILENAME NS_LITERAL_STRING("places.sqlite.corrupt") +// We use the TRUNCATE journal mode to reduce the number of fsyncs. Without +// this setting we had a Ts hit on Linux. See bug 460315 for details. +#define DATABASE_JOURNAL_MODE "TRUNCATE" + // Fraction of free pages in the database to force a vacuum between // DATABASE_MAX_TIME_BEFORE_VACUUM and DATABASE_MIN_TIME_BEFORE_VACUUM. #define DATABASE_VACUUM_FREEPAGES_THRESHOLD 0.1 // This is the maximum time (in microseconds) that can pass between 2 VACUUM // operations. #define DATABASE_MAX_TIME_BEFORE_VACUUM (PRInt64)60 * 24 * 60 * 60 * 1000 * 1000 // This is the minimum time (in microseconds) that should pass between 2 VACUUM // operations. @@ -369,18 +372,16 @@ const PRInt32 nsNavHistory::kGetInfoInde PLACES_FACTORY_SINGLETON_IMPLEMENTATION(nsNavHistory, gHistoryService) nsNavHistory::nsNavHistory() : mBatchLevel(0) , mBatchDBTransaction(nsnull) -, mDBPageSize(0) -, mCurrentJournalMode(JOURNAL_DELETE) , mCachedNow(0) , mExpireNowTimer(nsnull) , mLastSessionID(0) , mHistoryEnabled(PR_TRUE) , mNumVisitsForFrecency(10) , mTagsFolder(-1) , mInPrivateBrowsing(PRIVATEBROWSING_NOTINITED) , mDatabaseStatus(DATABASE_STATUS_OK) @@ -615,92 +616,35 @@ nsNavHistory::InitDBFile(PRBool aForceIn } NS_ENSURE_SUCCESS(rv, rv); return NS_OK; } nsresult -nsNavHistory::SetJournalMode(enum JournalMode aJournalMode) { - nsCAutoString journalMode; - switch (aJournalMode) { - case JOURNAL_DELETE: - journalMode.AssignLiteral("delete"); - break; - case JOURNAL_TRUNCATE: - journalMode.AssignLiteral("truncate"); - break; - case JOURNAL_MEMORY: - journalMode.AssignLiteral("memory"); - break; - case JOURNAL_WAL: - journalMode.AssignLiteral("wal"); - break; - default: - NS_ABORT_IF_FALSE(false, "Trying to set an unknown journal mode."); - } - - nsCOMPtr<mozIStorageStatement> statement; - nsresult rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( - "PRAGMA journal_mode = ") + journalMode, - getter_AddRefs(statement)); - NS_ENSURE_SUCCESS(rv, rv); - - mozStorageStatementScoper scoper(statement); - PRBool hasResult; - rv = statement->ExecuteStep(&hasResult); - NS_ENSURE_SUCCESS(rv, rv); - NS_ENSURE_TRUE(hasResult, NS_ERROR_FAILURE); - - nsCAutoString currentJournalMode; - rv = statement->GetUTF8String(0, currentJournalMode); - NS_ENSURE_SUCCESS(rv, rv); - bool succeeded = currentJournalMode.Equals(journalMode); - NS_WARN_IF_FALSE(succeeded, - nsPrintfCString(128, "Setting journal mode failed: %s", - PromiseFlatCString(journalMode).get()).get()); - if (succeeded) { - mCurrentJournalMode = aJournalMode; - } - else { - return NS_ERROR_FAILURE; - } - - return NS_OK; -} - - -nsresult nsNavHistory::InitDB() { - // WARNING: - // Any unfinalized statement will cause failure on setting WAL journal mode. - // Be sure to avoid them till journal mode has been set. - // Get the database schema version. PRInt32 currentSchemaVersion = 0; nsresult rv = mDBConn->GetSchemaVersion(¤tSchemaVersion); NS_ENSURE_SUCCESS(rv, rv); - { - // Get the page size. This may be different than the default if the - // database file already existed with a different page size. - nsCOMPtr<mozIStorageStatement> statement; - rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"), - getter_AddRefs(statement)); - NS_ENSURE_SUCCESS(rv, rv); - - PRBool hasResult; - mozStorageStatementScoper scoper(statement); - rv = statement->ExecuteStep(&hasResult); - NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE); - rv = statement->GetInt32(0, &mDBPageSize); - NS_ENSURE_SUCCESS(rv, rv); - NS_ENSURE_TRUE(mDBPageSize > 0, NS_ERROR_UNEXPECTED); - } + + // Get the page size. This may be different than the default if the + // database file already existed with a different page size. + nsCOMPtr<mozIStorageStatement> statement; + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("PRAGMA page_size"), + getter_AddRefs(statement)); + NS_ENSURE_SUCCESS(rv, rv); + + PRBool hasResult; + mozStorageStatementScoper scoper(statement); + rv = statement->ExecuteStep(&hasResult); + NS_ENSURE_TRUE(NS_SUCCEEDED(rv) && hasResult, NS_ERROR_FAILURE); + PRInt32 pageSize = statement->AsInt32(0); // Ensure that temp tables are held in memory, not on disk. We use temp // tables mainly for fsync and I/O reduction. rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( "PRAGMA temp_store = MEMORY")); NS_ENSURE_SUCCESS(rv, rv); // Set pragma synchronous to FULL to ensure maximum data integrity, even in @@ -722,36 +666,31 @@ nsNavHistory::InitDB() cachePercentage = 50; if (cachePercentage < 0) cachePercentage = 0; static PRInt64 physMem = PR_GetPhysicalMemorySize(); PRInt64 cacheSize = physMem * cachePercentage / 100; // Compute number of cached pages, this will be our cache size. - PRInt64 cachePages = cacheSize / mDBPageSize; - nsCAutoString cacheSizePragma("PRAGMA cache_size = "); - cacheSizePragma.AppendInt(cachePages); - rv = mDBConn->ExecuteSimpleSQL(cacheSizePragma); + PRInt64 cachePages = cacheSize / pageSize; + nsCAutoString pageSizePragma("PRAGMA cache_size = "); + pageSizePragma.AppendInt(cachePages); + rv = mDBConn->ExecuteSimpleSQL(pageSizePragma); NS_ENSURE_SUCCESS(rv, rv); // Lock the database file. This is done partly to avoid third party // applications to access it while it's in use, partly for performance. rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( "PRAGMA locking_mode = EXCLUSIVE")); NS_ENSURE_SUCCESS(rv, rv); - // Be sure to set journal mode after page_size, WAL would prevent the change - // otherwise. - if (NS_FAILED(SetJournalMode(JOURNAL_WAL))) { - // Ignore errors, if we fail here the database could be considered corrupt - // and we won't be able to go on, even if it's just matter of a bogus - // filesystem. The default mode (DELETE) will be fine in such a case. - (void)SetJournalMode(JOURNAL_TRUNCATE); - } + rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE)); + NS_ENSURE_SUCCESS(rv, rv); // We are going to initialize tables, so everything from now on should be in // a transaction for performances. mozStorageTransaction transaction(mDBConn, PR_FALSE); // Initialize the other Places services' database tables before creating our // statements. Some of our statements depend on these external tables, such as // the bookmarks or favicon tables. @@ -1836,29 +1775,31 @@ nsNavHistory::MigrateV9Up(mozIStorageCon // Now let's sync the column contents with real visit dates. // This query can be really slow due to disk access, since it will basically // dupe the table contents in the journal file, and then write them down // in the database. // We will temporary use a memory journal file, this has the advantage of // reducing write times by a half, but will temporary consume more memory // and increase risks of corruption if we should crash in the middle of this // update. - (void)SetJournalMode(JOURNAL_MEMORY); + rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "PRAGMA journal_mode = MEMORY")); + NS_ENSURE_SUCCESS(rv, rv); rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( "UPDATE moz_places SET last_visit_date = " "(SELECT MAX(visit_date) " "FROM moz_historyvisits " "WHERE place_id = moz_places.id)")); NS_ENSURE_SUCCESS(rv, rv); // Restore the default journal mode. - if (NS_FAILED(SetJournalMode(JOURNAL_WAL))) { - (void)SetJournalMode(JOURNAL_TRUNCATE); - } + rv = mDBConn->ExecuteSimpleSQL(NS_LITERAL_CSTRING( + "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE)); + NS_ENSURE_SUCCESS(rv, rv); } return transaction.Commit(); } nsresult nsNavHistory::MigrateV10Up(mozIStorageConnection *aDBConn) @@ -5926,35 +5867,48 @@ nsNavHistory::VacuumDatabase() nsCOMPtr<nsIObserverService> observerService = do_GetService(NS_OBSERVERSERVICE_CONTRACTID); if (observerService) { (void)observerService->NotifyObservers(nsnull, TOPIC_DATABASE_VACUUM_STARTING, nsnull); } - // If journal mode is WAL, a VACUUM cannot upgrade page_size value. - // If current page_size is not the expected one, journal mode must be - // changed to a rollback one. Once done we won't be able to go back to WAL - // mode though, since unfinalized statements exist. Just keep using - // compatible mode till next restart. - // See http://www.sqlite.org/wal.html - if (mCurrentJournalMode == JOURNAL_WAL && - mDBPageSize != SQLITE_DEFAULT_PAGE_SIZE) { - (void)SetJournalMode(JOURNAL_TRUNCATE); - } - - nsCOMPtr<mozIStorageAsyncStatement> vacuum; - rv = mDBConn->CreateAsyncStatement(NS_LITERAL_CSTRING("VACUUM"), - getter_AddRefs(vacuum)); - NS_ENSURE_SUCCESS(rv, rv); + // Actually vacuuming a database is a slow operation, since it could take + // seconds. Part of the time is spent in updating the journal file on disk + // and this is particularly bad on devices with slow I/O. Temporary + // moving the journal to memory could increase a bit the possibility of + // corruption if we crash during this time, but makes the process really + // faster. + nsCOMPtr<mozIStorageStatement> journalToMemory; + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA journal_mode = MEMORY"), + getter_AddRefs(journalToMemory)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<mozIStorageStatement> vacuum; + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING("VACUUM"), + getter_AddRefs(vacuum)); + NS_ENSURE_SUCCESS(rv, rv); + + nsCOMPtr<mozIStorageStatement> journalToDefault; + rv = mDBConn->CreateStatement(NS_LITERAL_CSTRING( + "PRAGMA journal_mode = " DATABASE_JOURNAL_MODE), + getter_AddRefs(journalToDefault)); + NS_ENSURE_SUCCESS(rv, rv); + + mozIStorageBaseStatement *stmts[] = { + journalToMemory, + vacuum, + journalToDefault + }; nsCOMPtr<mozIStoragePendingStatement> ps; - rv = vacuum->ExecuteAsync(nsnull, getter_AddRefs(ps)); - NS_ENSURE_SUCCESS(rv, rv); - vacuum->Finalize(); + rv = mDBConn->ExecuteAsync(stmts, NS_ARRAY_LENGTH(stmts), nsnull, + getter_AddRefs(ps)); + NS_ENSURE_SUCCESS(rv, rv); if (mPrefBranch) { (void)mPrefBranch->SetIntPref(PREF_LAST_VACUUM, (PRInt32)(PR_Now() / PR_USEC_PER_SEC)); } } return NS_OK;
--- a/toolkit/components/places/src/nsNavHistory.h +++ b/toolkit/components/places/src/nsNavHistory.h @@ -117,28 +117,16 @@ namespace places { , DB_RECENT_VISIT_OF_URL = 4 , DB_GET_PAGE_VISIT_STATS = 5 , DB_UPDATE_PAGE_VISIT_STATS = 6 , DB_ADD_NEW_PAGE = 7 , DB_GET_URL_PAGE_INFO = 8 , DB_SET_PLACE_TITLE = 9 }; - enum JournalMode { - // Default SQLite jousrnal mode. - JOURNAL_DELETE = 0 - // Can reduce fsyncs on Linux when journal is deleted (See bug 460315). - // We fallback to this mode when WAL is unavailable. - , JOURNAL_TRUNCATE - // Unsafe in case of crashes on database swap or low memory. - , JOURNAL_MEMORY - // Can reduce number of fsyncs. We try to use this mode by default. - , JOURNAL_WAL - }; - } // namespace places } // namespace mozilla class mozIAnnotationService; class nsNavHistory; class nsNavBookmarks; class QueryKeyValuePair; @@ -476,17 +464,16 @@ protected: nsDataHashtable<nsStringHashKey, int> gExpandedItems; // // Database stuff // nsCOMPtr<mozIStorageService> mDBService; nsCOMPtr<mozIStorageConnection> mDBConn; nsCOMPtr<nsIFile> mDBFile; - PRInt32 mDBPageSize; nsCOMPtr<mozIStorageStatement> mDBGetURLPageInfo; // kGetInfoIndex_* results nsCOMPtr<mozIStorageStatement> mDBGetIdPageInfo; // kGetInfoIndex_* results nsCOMPtr<mozIStorageStatement> mDBRecentVisitOfURL; // converts URL into most recent visit ID/session ID nsCOMPtr<mozIStorageStatement> mDBRecentVisitOfPlace; // converts placeID into most recent visit ID/session ID nsCOMPtr<mozIStorageStatement> mDBInsertVisit; // used by AddVisit nsCOMPtr<mozIStorageStatement> mDBGetPageVisitStats; // used by AddVisit @@ -545,22 +532,16 @@ protected: * * @param aForceInit * Indicates if we should close an open database connection or not. * Note: A valid database connection must be opened if this is true. */ nsresult InitDBFile(PRBool aForceInit); /** - * Set journal mode on the database. - */ - nsresult SetJournalMode(enum mozilla::places::JournalMode aJournalMode); - enum mozilla::places::JournalMode mCurrentJournalMode; - - /** * Initializes the database. This performs any necessary migrations for the * database. All migration is done inside a transaction that is rolled back * if any error occurs. Upon initialization, history is imported, and some * preferences that are used are set. */ nsresult InitDB(); /**
--- a/toolkit/components/places/src/nsPlacesDBFlush.js +++ b/toolkit/components/places/src/nsPlacesDBFlush.js @@ -133,28 +133,16 @@ nsPlacesDBFlush.prototype = { _self: this, run: function() { // Flush any remaining change to disk tables. this._self._flushWithQueries([kQuerySyncPlacesId, kQuerySyncHistoryVisitsId]); // Close the database connection, this was the last sync and we can't // ensure database coherence from now on. this._self._finalizeInternalStatements(); - - // Before closing the connection we have to set back journal mode to - // a backwards compatible value. Newer journal modes like WAL make - // the database incompatible with old versions of the browser, setting - // an old mode restores database version. - // See http://www.sqlite.org/draft/wal.html - let journalStmt = this._self._db.createAsyncStatement( - "PRAGMA journal_mode = truncate" - ); - journalStmt.executeAsync(); - journalStmt.finalize(); - this._self._db.asyncClose(); } }, Ci.nsIThread.DISPATCH_NORMAL); } else if (aTopic == "nsPref:changed" && aData == kSyncPrefName) { // Get the new pref value, and then update our timer this._syncInterval = Services.prefs.getIntPref(kSyncPrefName); if (this._syncInterval <= 0)