Bug 1162176, Part 1. r=mak.
authorBen Turner <bent.mozilla@gmail.com>
Tue, 19 May 2015 09:17:10 -0700
changeset 244693 280dae3918d936725f24ef4036a97ac1f1a18e3d
parent 244692 446ce38d00050fca97b8b10a6073a01d751de346
child 244694 fba019b3618f38fd1116ed2b53186b3616871c3d
push id28786
push usercbook@mozilla.com
push dateWed, 20 May 2015 13:54:15 +0000
treeherdermozilla-central@8d8df22fe72d [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmak
bugs1162176
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1162176, Part 1. r=mak.
db/sqlite3/src/sqlite.def
storage/mozStorageConnection.cpp
storage/mozStorageConnection.h
toolkit/components/telemetry/Telemetry.cpp
--- a/db/sqlite3/src/sqlite.def
+++ b/db/sqlite3/src/sqlite.def
@@ -46,16 +46,17 @@ EXPORTS
         sqlite3_complete16
         sqlite3_config
         sqlite3_create_collation
         sqlite3_create_collation16
         sqlite3_create_function
         sqlite3_create_function16
         sqlite3_create_module
         sqlite3_data_count
+        sqlite3_db_filename
         sqlite3_db_handle
         sqlite3_db_mutex
         sqlite3_db_status
         sqlite3_declare_vtab
         sqlite3_enable_load_extension
         sqlite3_enable_shared_cache
         sqlite3_errcode
         sqlite3_errmsg
--- a/storage/mozStorageConnection.cpp
+++ b/storage/mozStorageConnection.cpp
@@ -585,17 +585,23 @@ Connection::initialize()
 
   // in memory database requested, sqlite uses a magic file name
   int srv = ::sqlite3_open_v2(":memory:", &mDBConn, mFlags, nullptr);
   if (srv != SQLITE_OK) {
     mDBConn = nullptr;
     return convertResultCode(srv);
   }
 
-  return initializeInternal(nullptr);
+  // Do not set mDatabaseFile or mFileURL here since this is a "memory"
+  // database.
+
+  nsresult rv = initializeInternal();
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return NS_OK;
 }
 
 nsresult
 Connection::initialize(nsIFile *aDatabaseFile)
 {
   NS_ASSERTION (aDatabaseFile, "Passed null file!");
   NS_ASSERTION (!mDBConn, "Initialize called on already opened database!");
   PROFILER_LABEL("mozStorageConnection", "initialize",
@@ -609,21 +615,23 @@ Connection::initialize(nsIFile *aDatabas
 
   int srv = ::sqlite3_open_v2(NS_ConvertUTF16toUTF8(path).get(), &mDBConn,
                               mFlags, nullptr);
   if (srv != SQLITE_OK) {
     mDBConn = nullptr;
     return convertResultCode(srv);
   }
 
-  rv = initializeInternal(aDatabaseFile);
+  // Do not set mFileURL here since this is database does not have an associated
+  // URL.
+  mDatabaseFile = aDatabaseFile;
+
+  rv = initializeInternal();
   NS_ENSURE_SUCCESS(rv, rv);
 
-  mDatabaseFile = aDatabaseFile;
-
   return NS_OK;
 }
 
 nsresult
 Connection::initialize(nsIFileURL *aFileURL)
 {
   NS_ASSERTION (aFileURL, "Passed null file URL!");
   NS_ASSERTION (!mDBConn, "Initialize called on already opened database!");
@@ -639,45 +647,63 @@ Connection::initialize(nsIFileURL *aFile
   NS_ENSURE_SUCCESS(rv, rv);
 
   int srv = ::sqlite3_open_v2(spec.get(), &mDBConn, mFlags, nullptr);
   if (srv != SQLITE_OK) {
     mDBConn = nullptr;
     return convertResultCode(srv);
   }
 
-  rv = initializeInternal(databaseFile);
-  NS_ENSURE_SUCCESS(rv, rv);
-
+  // Set both mDatabaseFile and mFileURL here.
   mFileURL = aFileURL;
   mDatabaseFile = databaseFile;
 
+  rv = initializeInternal();
+  NS_ENSURE_SUCCESS(rv, rv);
+
   return NS_OK;
 }
 
+nsresult
+Connection::initializeInternal()
+{
+  MOZ_ASSERT(mDBConn);
 
-nsresult
-Connection::initializeInternal(nsIFile* aDatabaseFile)
-{
+  if (mFileURL) {
+    const char* dbPath = ::sqlite3_db_filename(mDBConn, "main");
+    MOZ_ASSERT(dbPath);
+
+    const char* telemetryFilename =
+      ::sqlite3_uri_parameter(dbPath, "telemetryFilename");
+    if (telemetryFilename) {
+      if (NS_WARN_IF(*telemetryFilename == '\0')) {
+        return NS_ERROR_INVALID_ARG;
+      }
+      mTelemetryFilename = telemetryFilename;
+    }
+  }
+
+  if (mTelemetryFilename.IsEmpty()) {
+    mTelemetryFilename = getFilename();
+    MOZ_ASSERT(!mTelemetryFilename.IsEmpty());
+  }
+
   // Properly wrap the database handle's mutex.
   sharedDBMutex.initWithMutex(sqlite3_db_mutex(mDBConn));
 
   if (!gStorageLog)
     gStorageLog = ::PR_NewLogModule("mozStorage");
 
   // SQLite tracing can slow down queries (especially long queries)
   // significantly. Don't trace unless the user is actively monitoring SQLite.
   if (PR_LOG_TEST(gStorageLog, PR_LOG_DEBUG)) {
     ::sqlite3_trace(mDBConn, tracefunc, this);
 
-    nsAutoCString leafName(":memory");
-    if (aDatabaseFile)
-      (void)aDatabaseFile->GetNativeLeafName(leafName);
     PR_LOG(gStorageLog, PR_LOG_NOTICE, ("Opening connection to '%s' (%p)",
-                                        leafName.get(), this));
+                                        mTelemetryFilename.get(), this));
   }
 
   int64_t pageSize = Service::getDefaultPageSize();
 
   // Set page_size to the preferred default value.  This is effective only if
   // the database has just been created, otherwise, if the database does not
   // use WAL journal mode, a VACUUM operation will updated its page_size.
   nsAutoCString pageSizeQuery(MOZ_STORAGE_UNIQUIFY_QUERY_STR
@@ -1020,17 +1046,17 @@ Connection::stepStatement(sqlite3 *aNati
 
   // Report very slow SQL statements to Telemetry
   TimeDuration duration = TimeStamp::Now() - startTime;
   const uint32_t threshold =
     NS_IsMainThread() ? Telemetry::kSlowSQLThresholdForMainThread
                       : Telemetry::kSlowSQLThresholdForHelperThreads;
   if (duration.ToMilliseconds() >= threshold) {
     nsDependentCString statementString(::sqlite3_sql(aStatement));
-    Telemetry::RecordSlowSQLStatement(statementString, getFilename(),
+    Telemetry::RecordSlowSQLStatement(statementString, mTelemetryFilename,
                                       duration.ToMilliseconds());
   }
 
   (void)::sqlite3_extended_result_codes(aNativeConnection, 0);
   // Drop off the extended result bits of the result code.
   return srv & 0xFF;
 }
 
@@ -1106,17 +1132,17 @@ Connection::executeSql(sqlite3 *aNativeC
 
   // Report very slow SQL statements to Telemetry
   TimeDuration duration = TimeStamp::Now() - startTime;
   const uint32_t threshold =
     NS_IsMainThread() ? Telemetry::kSlowSQLThresholdForMainThread
                       : Telemetry::kSlowSQLThresholdForHelperThreads;
   if (duration.ToMilliseconds() >= threshold) {
     nsDependentCString statementString(aSqlString);
-    Telemetry::RecordSlowSQLStatement(statementString, getFilename(),
+    Telemetry::RecordSlowSQLStatement(statementString, mTelemetryFilename,
                                       duration.ToMilliseconds());
   }
 
   return srv;
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 //// nsIInterfaceRequestor
--- a/storage/mozStorageConnection.h
+++ b/storage/mozStorageConnection.h
@@ -225,17 +225,17 @@ public:
    * try to use them.
    */
   bool isClosed();
 
   nsresult initializeClone(Connection *aClone, bool aReadOnly);
 
 private:
   ~Connection();
-  nsresult initializeInternal(nsIFile *aDatabaseFile);
+  nsresult initializeInternal();
 
   /**
    * Sets the database into a closed state so no further actions can be
    * performed.
    *
    * @note mDBConn is set to nullptr in this method.
    */
   nsresult setClosedState();
@@ -284,16 +284,22 @@ private:
   // if there is one. Do nothing in other cases.
   int progressHandler();
 
   sqlite3 *mDBConn;
   nsCOMPtr<nsIFileURL> mFileURL;
   nsCOMPtr<nsIFile> mDatabaseFile;
 
   /**
+   * The filename that will be reported to telemetry for this connection. By
+   * default this will be the leaf of the path to the database file.
+  */
+  nsCString mTelemetryFilename;
+
+  /**
    * Lazily created thread for asynchronous statement execution.  Consumers
    * should use getAsyncExecutionTarget rather than directly accessing this
    * field.
    */
   nsCOMPtr<nsIThread> mAsyncExecutionThread;
 
   /**
    * Set to true by Close() or AsyncClose() prior to shutdown.
--- a/toolkit/components/telemetry/Telemetry.cpp
+++ b/toolkit/components/telemetry/Telemetry.cpp
@@ -731,19 +731,16 @@ private:
   typedef nsBaseHashtableET<nsDepCharHashKey, Telemetry::ID> CharPtrEntryType;
   typedef AutoHashtable<CharPtrEntryType> HistogramMapType;
   HistogramMapType mHistogramMap;
   bool mCanRecordBase;
   bool mCanRecordExtended;
   static TelemetryImpl *sTelemetry;
   AutoHashtable<SlowSQLEntryType> mPrivateSQL;
   AutoHashtable<SlowSQLEntryType> mSanitizedSQL;
-  // This gets marked immutable in debug builds, so we can't use
-  // AutoHashtable here.
-  nsTHashtable<nsCStringHashKey> mTrackedDBs;
   Mutex mHashMutex;
   HangReports mHangReports;
   Mutex mHangReportsMutex;
   // mThreadHangStats stores recorded, inactive thread hang stats
   Vector<Telemetry::ThreadHangStats> mThreadHangStats;
   Mutex mThreadHangStatsMutex;
 
   CombinedStacks mLateWritesStacks; // This is collected out of the main thread.
@@ -1857,45 +1854,16 @@ mCanRecordExtended(XRE_GetProcessType() 
                    XRE_GetProcessType() == GeckoProcessType_Content),
 mHashMutex("Telemetry::mHashMutex"),
 mHangReportsMutex("Telemetry::mHangReportsMutex"),
 mThreadHangStatsMutex("Telemetry::mThreadHangStatsMutex"),
 mCachedTelemetryData(false),
 mLastShutdownTime(0),
 mFailedLockCount(0)
 {
-  // A whitelist to prevent Telemetry reporting on Addon & Thunderbird DBs
-  const char *trackedDBs[] = {
-    "818200132aebmoouht.sqlite", // IndexedDB for about:home, see aboutHome.js
-    "addons.sqlite",
-    "content-prefs.sqlite",
-    "cookies.sqlite",
-    "downloads.sqlite",
-    "extensions.sqlite",
-    "formhistory.sqlite",
-    "healthreport.sqlite",
-    "index.sqlite",
-    "netpredictions.sqlite",
-    "permissions.sqlite",
-    "places.sqlite",
-    "reading-list.sqlite",
-    "search.sqlite",
-    "signons.sqlite",
-    "urlclassifier3.sqlite",
-    "webappsstore.sqlite"
-  };
-
-  for (size_t i = 0; i < ArrayLength(trackedDBs); i++)
-    mTrackedDBs.PutEntry(nsDependentCString(trackedDBs[i]));
-
-#ifdef DEBUG
-  // Mark immutable to prevent asserts on simultaneous access from multiple threads
-  mTrackedDBs.MarkImmutable();
-#endif
-
   // Populate the static histogram name->id cache.
   // Note that the histogram names are statically allocated.
   for (uint32_t i = 0; i < Telemetry::HistogramCount; i++) {
     CharPtrEntryType *entry = mHistogramMap.PutEntry(gHistograms[i].id());
     entry->mData = (Telemetry::ID) i;
   }
 
 #ifdef DEBUG
@@ -3421,32 +3389,108 @@ TelemetryImpl::SanitizeSQL(const nsACStr
   }
 
   if ((fragmentStart >= 0) && fragmentStart < length)
     output += nsDependentCSubstring(sql, fragmentStart, length - fragmentStart);
 
   return output;
 }
 
+// A whitelist mechanism to prevent Telemetry reporting on Addon & Thunderbird
+// DBs.
+struct TrackedDBEntry
+{
+  const char* mName;
+  const uint32_t mNameLength;
+
+  // This struct isn't meant to be used beyond the static arrays below.
+  MOZ_CONSTEXPR
+  TrackedDBEntry(const char* aName, uint32_t aNameLength)
+    : mName(aName)
+    , mNameLength(aNameLength)
+  { }
+
+  TrackedDBEntry() = delete;
+  TrackedDBEntry(TrackedDBEntry&) = delete;
+};
+
+#define TRACKEDDB_ENTRY(_name) { _name, (sizeof(_name) - 1) }
+
+// A whitelist of database names. If the database name exactly matches one of
+// these then its SQL statements will always be recorded.
+static MOZ_CONSTEXPR_VAR TrackedDBEntry kTrackedDBs[] = {
+  // IndexedDB for about:home, see aboutHome.js
+  TRACKEDDB_ENTRY("818200132aebmoouht.sqlite"),
+  TRACKEDDB_ENTRY("addons.sqlite"),
+  TRACKEDDB_ENTRY("content-prefs.sqlite"),
+  TRACKEDDB_ENTRY("cookies.sqlite"),
+  TRACKEDDB_ENTRY("downloads.sqlite"),
+  TRACKEDDB_ENTRY("extensions.sqlite"),
+  TRACKEDDB_ENTRY("formhistory.sqlite"),
+  TRACKEDDB_ENTRY("healthreport.sqlite"),
+  TRACKEDDB_ENTRY("index.sqlite"),
+  TRACKEDDB_ENTRY("netpredictions.sqlite"),
+  TRACKEDDB_ENTRY("permissions.sqlite"),
+  TRACKEDDB_ENTRY("places.sqlite"),
+  TRACKEDDB_ENTRY("reading-list.sqlite"),
+  TRACKEDDB_ENTRY("search.sqlite"),
+  TRACKEDDB_ENTRY("signons.sqlite"),
+  TRACKEDDB_ENTRY("urlclassifier3.sqlite"),
+  TRACKEDDB_ENTRY("webappsstore.sqlite")
+};
+
+// A whitelist of database name prefixes. If the database name begins with
+// one of these prefixes then its SQL statements will always be recorded.
+static const TrackedDBEntry kTrackedDBPrefixes[] = {
+  TRACKEDDB_ENTRY("indexedDB-")
+};
+
+#undef TRACKEDDB_ENTRY
+
 // Slow SQL statements will be automatically
 // trimmed to kMaxSlowStatementLength characters.
 // This limit doesn't include the ellipsis and DB name,
 // that are appended at the end of the stored statement.
 const uint32_t kMaxSlowStatementLength = 1000;
 
 void
 TelemetryImpl::RecordSlowStatement(const nsACString &sql,
                                    const nsACString &dbName,
                                    uint32_t delay)
 {
+  MOZ_ASSERT(!sql.IsEmpty());
+  MOZ_ASSERT(!dbName.IsEmpty());
+
   if (!sTelemetry || !sTelemetry->mCanRecordExtended)
     return;
 
-  bool isFirefoxDB = sTelemetry->mTrackedDBs.Contains(dbName);
-  if (isFirefoxDB) {
+  bool recordStatement = false;
+
+  for (const TrackedDBEntry& nameEntry : kTrackedDBs) {
+    MOZ_ASSERT(nameEntry.mNameLength);
+    const nsDependentCString name(nameEntry.mName, nameEntry.mNameLength);
+    if (dbName == name) {
+      recordStatement = true;
+      break;
+    }
+  }
+
+  if (!recordStatement) {
+    for (const TrackedDBEntry& prefixEntry : kTrackedDBPrefixes) {
+      MOZ_ASSERT(prefixEntry.mNameLength);
+      const nsDependentCString prefix(prefixEntry.mName,
+                                      prefixEntry.mNameLength);
+      if (StringBeginsWith(dbName, prefix)) {
+        recordStatement = true;
+        break;
+      }
+    }
+  }
+
+  if (recordStatement) {
     nsAutoCString sanitizedSQL(SanitizeSQL(sql));
     if (sanitizedSQL.Length() > kMaxSlowStatementLength) {
       sanitizedSQL.SetLength(kMaxSlowStatementLength);
       sanitizedSQL += "...";
     }
     sanitizedSQL.AppendPrintf(" /* %s */", nsPromiseFlatCString(dbName).get());
     StoreSlowSQL(sanitizedSQL, delay, Sanitized);
   } else {
@@ -3566,17 +3610,16 @@ TelemetryImpl::SizeOfIncludingThis(mozil
   // Ignore the hashtables in mAddonMap; they are not significant.
   n += mAddonMap.SizeOfExcludingThis(nullptr, aMallocSizeOf);
   n += mHistogramMap.SizeOfExcludingThis(nullptr, aMallocSizeOf);
   { // Scope for mHashMutex lock
     MutexAutoLock lock(mHashMutex);
     n += mPrivateSQL.SizeOfExcludingThis(aMallocSizeOf);
     n += mSanitizedSQL.SizeOfExcludingThis(aMallocSizeOf);
   }
-  n += mTrackedDBs.SizeOfExcludingThis(aMallocSizeOf);
   { // Scope for mHangReportsMutex lock
     MutexAutoLock lock(mHangReportsMutex);
     n += mHangReports.SizeOfExcludingThis(aMallocSizeOf);
   }
   { // Scope for mThreadHangStatsMutex lock
     MutexAutoLock lock(mThreadHangStatsMutex);
     n += mThreadHangStats.sizeOfExcludingThis(aMallocSizeOf);
   }