Bug 945779 - Use transactions to batch up Seer I/O. r=honzab
authorNicholas Hurley <hurley@todesschaf.org>
Fri, 17 Jan 2014 17:45:46 -0800
changeset 164111 a61079647dc5fcd265a3b7d2f4c8b7cf6ba45af8
parent 164110 48289161bc17e4c87c061c63b5233eba07644704
child 164112 5bf92bd8d7ed4ee2ef372c421279dba54e166c77
push id26026
push userphilringnalda@gmail.com
push dateSat, 18 Jan 2014 23:17:27 +0000
treeherdermozilla-central@61fd0f987cf2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewershonzab
bugs945779
milestone29.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 945779 - Use transactions to batch up Seer I/O. r=honzab
netwerk/base/public/nsINetworkSeer.idl
netwerk/base/src/Seer.cpp
netwerk/base/src/Seer.h
netwerk/base/src/moz.build
netwerk/test/unit/test_seer.js
--- a/netwerk/base/public/nsINetworkSeer.idl
+++ b/netwerk/base/public/nsINetworkSeer.idl
@@ -12,17 +12,17 @@ interface nsINetworkSeerVerifier;
 typedef unsigned long SeerPredictReason;
 typedef unsigned long SeerLearnReason;
 
 /**
  * nsINetworkSeer - learn about pages users visit, and allow us to take
  *                  predictive actions upon future visits.
  *                  NOTE: nsINetworkSeer should only be used on the main thread
  */
-[scriptable, uuid(884a39a0-a3ed-4855-826a-fabb73ae878d)]
+[scriptable, uuid(25e323b6-99e0-4274-b5b3-1a9eb56e28ac)]
 interface nsINetworkSeer : nsISupports
 {
   /**
    * Prediction reasons
    *
    * PREDICT_LINK - we are being asked to take predictive action because
    * the user is hovering over a link.
    *
@@ -116,16 +116,22 @@ interface nsINetworkSeer : nsISupports
 
   /**
    * Clear out all our learned knowledge
    *
    * This removes everything from our database so that any predictions begun
    * after this completes will start from a blank slate.
    */
   void reset();
+
+  /**
+   * @deprecated THIS API IS FOR TESTING ONLY. IF YOU DON'T KNOW WHAT IT DOES,
+   * DON'T USE IT
+   */
+  void prepareForDnsTest(in long long timestamp, in string uri);
 };
 
 %{C++
 // Wrapper functions to make use of the seer easier and less invasive
 class nsIChannel;
 class nsIDocument;
 class nsILoadContext;
 class nsILoadGroup;
--- a/netwerk/base/src/Seer.cpp
+++ b/netwerk/base/src/Seer.cpp
@@ -16,16 +16,17 @@
 #include "nsIFile.h"
 #include "nsILoadContext.h"
 #include "nsILoadGroup.h"
 #include "nsINetworkSeerVerifier.h"
 #include "nsIObserverService.h"
 #include "nsIPrefBranch.h"
 #include "nsIPrefService.h"
 #include "nsISpeculativeConnect.h"
+#include "nsITimer.h"
 #include "nsIURI.h"
 #include "nsNetUtil.h"
 #include "nsServiceManagerUtils.h"
 #include "nsTArray.h"
 #include "nsThreadUtils.h"
 #ifdef MOZ_NUWA_PROCESS
 #include "ipc/Nuwa.h"
 #endif
@@ -285,24 +286,62 @@ Seer::RemoveObserver()
 
   nsCOMPtr<nsIObserverService> obs =
     mozilla::services::GetObserverService();
   if (obs) {
     obs->RemoveObserver(this, NS_XPCOM_SHUTDOWN_OBSERVER_ID);
   }
 }
 
+static const uint32_t COMMIT_TIMER_DELTA_MS = 5 * 1000;
+
+class SeerCommitTimerInitEvent : public nsRunnable
+{
+public:
+  NS_IMETHOD Run() MOZ_OVERRIDE
+  {
+    nsresult rv = NS_OK;
+
+    if (!gSeer->mCommitTimer) {
+      gSeer->mCommitTimer = do_CreateInstance(NS_TIMER_CONTRACTID, &rv);
+    } else {
+      gSeer->mCommitTimer->Cancel();
+    }
+    if (NS_SUCCEEDED(rv)) {
+      gSeer->mCommitTimer->Init(gSeer, COMMIT_TIMER_DELTA_MS,
+                                nsITimer::TYPE_ONE_SHOT);
+    }
+
+    return NS_OK;
+  }
+};
+
+class SeerNewTransactionEvent : public nsRunnable
+{
+  NS_IMETHODIMP Run() MOZ_OVERRIDE
+  {
+    gSeer->CommitTransaction();
+    gSeer->BeginTransaction();
+    nsRefPtr<SeerCommitTimerInitEvent> event = new SeerCommitTimerInitEvent();
+    NS_DispatchToMainThread(event);
+    return NS_OK;
+  }
+};
+
 NS_IMETHODIMP
 Seer::Observe(nsISupports *subject, const char *topic,
               const char16_t *data_unicode)
 {
   nsresult rv = NS_OK;
 
   if (!strcmp(NS_XPCOM_SHUTDOWN_OBSERVER_ID, topic)) {
     gSeer->Shutdown();
+  } else if (!strcmp(NS_TIMER_CALLBACK_TOPIC, topic)) {
+    nsRefPtr<SeerNewTransactionEvent> event = new SeerNewTransactionEvent();
+    gSeer->mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
   }
 
   return rv;
 }
 
 // Seer::nsISpeculativeConnectionOverrider
 
 NS_IMETHODIMP
@@ -462,16 +501,18 @@ Seer::EnsureInitStorage()
 
     rv = mStorageService->OpenDatabase(mDBFile, getter_AddRefs(mDB));
     NS_ENSURE_SUCCESS(rv, rv);
   }
 
   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA synchronous = OFF;"));
   mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("PRAGMA foreign_keys = ON;"));
 
+  BeginTransaction();
+
   // A table to make sure we're working with the database layout we expect
   rv = mDB->ExecuteSimpleSQL(
       NS_LITERAL_CSTRING("CREATE TABLE IF NOT EXISTS moz_seer_version (\n"
                          "  version INTEGER NOT NULL\n"
                          ");\n"));
   NS_ENSURE_SUCCESS(rv, rv);
 
   nsCOMPtr<mozIStorageStatement> stmt;
@@ -698,16 +739,22 @@ Seer::EnsureInitStorage()
                          "ON moz_redirects (pid, uri);"));
   NS_ENSURE_SUCCESS(rv, rv);
 
   rv = mDB->ExecuteSimpleSQL(
       NS_LITERAL_CSTRING("CREATE INDEX IF NOT EXISTS redirect_id_index "
                          "ON moz_redirects (id);"));
   NS_ENSURE_SUCCESS(rv, rv);
 
+  CommitTransaction();
+  BeginTransaction();
+
+  nsRefPtr<SeerCommitTimerInitEvent> event = new SeerCommitTimerInitEvent();
+  NS_DispatchToMainThread(event, NS_DISPATCH_NORMAL);
+
   return NS_OK;
 }
 
 class SeerThreadShutdownRunner : public nsRunnable
 {
 public:
   SeerThreadShutdownRunner(nsIThread *ioThread)
     :mIOThread(ioThread)
@@ -732,16 +779,19 @@ public:
   {
     mSeer = new nsMainThreadPtrHolder<nsINetworkSeer>(seer);
   }
 
   NS_IMETHODIMP Run() MOZ_OVERRIDE
   {
     MOZ_ASSERT(!NS_IsMainThread(), "Shutting down DB on main thread");
 
+    // Ensure everything is written to disk before we shut down the db
+    gSeer->CommitTransaction();
+
     gSeer->mStatements.FinalizeStatements();
     gSeer->mDB->Close();
     gSeer->mDB = nullptr;
 
     nsRefPtr<SeerThreadShutdownRunner> runner =
       new SeerThreadShutdownRunner(mIOThread);
     NS_DispatchToMainThread(runner);
 
@@ -760,16 +810,20 @@ Seer::Shutdown()
 {
   if (!NS_IsMainThread()) {
     MOZ_ASSERT(false, "Seer::Shutdown called off the main thread!");
     return;
   }
 
   mInitialized = false;
 
+  if (mCommitTimer) {
+    mCommitTimer->Cancel();
+  }
+
   if (mIOThread) {
     nsCOMPtr<nsIThread> ioThread;
     mIOThread.swap(ioThread);
 
     if (mDB) {
       nsRefPtr<SeerDBShutdownRunner> runner =
         new SeerDBShutdownRunner(ioThread, this);
       ioThread->Dispatch(runner, NS_DISPATCH_NORMAL);
@@ -2147,23 +2201,27 @@ public:
 void
 Seer::ResetInternal()
 {
   MOZ_ASSERT(!NS_IsMainThread(), "Resetting db on main thread");
 
   nsresult rv = EnsureInitStorage();
   RETURN_IF_FAILED(rv);
 
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_redirects"));
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startup_pages"));
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startups"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_redirects;"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startup_pages;"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_startups;"));
 
   // These cascade to moz_subresources and moz_subhosts, respectively.
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_pages"));
-  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_hosts"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_pages;"));
+  mDB->ExecuteSimpleSQL(NS_LITERAL_CSTRING("DELETE FROM moz_hosts;"));
+
+  // Go ahead and ensure this is flushed to disk
+  CommitTransaction();
+  BeginTransaction();
 }
 
 // Called on the main thread to clear out all our knowledge. Tabula Rasa FTW!
 NS_IMETHODIMP
 Seer::Reset()
 {
   MOZ_ASSERT(NS_IsMainThread(),
              "Seer interface methods must be called on the main thread");
@@ -2171,16 +2229,78 @@ Seer::Reset()
   if (!mInitialized) {
     return NS_ERROR_NOT_AVAILABLE;
   }
 
   nsRefPtr<SeerResetEvent> event = new SeerResetEvent();
   return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
 }
 
+#ifdef SEER_TESTS
+class SeerPrepareForDnsTestEvent : public nsRunnable
+{
+public:
+  SeerPrepareForDnsTestEvent(int64_t timestamp, const char *uri)
+    :mTimestamp(timestamp)
+    ,mUri(uri)
+  { }
+
+  NS_IMETHOD Run() MOZ_OVERRIDE
+  {
+    MOZ_ASSERT(!NS_IsMainThread(), "Preparing for DNS Test on main thread!");
+    gSeer->PrepareForDnsTestInternal(mTimestamp, mUri);
+    return NS_OK;
+  }
+
+private:
+  int64_t mTimestamp;
+  nsAutoCString mUri;
+};
+
+void
+Seer::PrepareForDnsTestInternal(int64_t timestamp, const nsACString &uri)
+{
+  nsCOMPtr<mozIStorageStatement> update = mStatements.GetCachedStatement(
+      NS_LITERAL_CSTRING("UPDATE moz_subresources SET last_hit = :timestamp, "
+                          "hits = 2 WHERE uri = :uri;"));
+  if (!update) {
+    return;
+  }
+  mozStorageStatementScoper scopedUpdate(update);
+
+  nsresult rv = update->BindInt64ByName(NS_LITERAL_CSTRING("timestamp"),
+                                        timestamp);
+  RETURN_IF_FAILED(rv);
+
+  rv = update->BindUTF8StringByName(NS_LITERAL_CSTRING("uri"), uri);
+  RETURN_IF_FAILED(rv);
+
+  update->Execute();
+}
+#endif
+
+NS_IMETHODIMP
+Seer::PrepareForDnsTest(int64_t timestamp, const char *uri)
+{
+#ifdef SEER_TESTS
+  MOZ_ASSERT(NS_IsMainThread(),
+             "Seer interface methods must be called on the main thread");
+
+  if (!mInitialized) {
+    return NS_ERROR_NOT_AVAILABLE;
+  }
+
+  nsRefPtr<SeerPrepareForDnsTestEvent> event =
+    new SeerPrepareForDnsTestEvent(timestamp, uri);
+  return mIOThread->Dispatch(event, NS_DISPATCH_NORMAL);
+#else
+  return NS_ERROR_NOT_IMPLEMENTED;
+#endif
+}
+
 // Helper functions to make using the seer easier from native code
 
 static nsresult
 EnsureGlobalSeer(nsINetworkSeer **aSeer)
 {
   nsresult rv;
   nsCOMPtr<nsINetworkSeer> seer = do_GetService("@mozilla.org/network/seer;1",
                                                 &rv);
--- a/netwerk/base/src/Seer.h
+++ b/netwerk/base/src/Seer.h
@@ -17,16 +17,17 @@
 
 #include "mozilla/Mutex.h"
 #include "mozilla/storage/StatementCache.h"
 #include "mozilla/TimeStamp.h"
 
 class nsIDNSService;
 class nsINetworkSeerVerifier;
 class nsIThread;
+class nsITimer;
 
 class mozIStorageConnection;
 class mozIStorageService;
 class mozIStorageStatement;
 
 namespace mozilla {
 namespace net {
 
@@ -56,16 +57,18 @@ public:
   static nsresult Create(nsISupports *outer, const nsIID& iid, void **result);
 
 private:
   friend class SeerPredictionEvent;
   friend class SeerLearnEvent;
   friend class SeerResetEvent;
   friend class SeerPredictionRunner;
   friend class SeerDBShutdownRunner;
+  friend class SeerCommitTimerInitEvent;
+  friend class SeerNewTransactionEvent;
 
   void CheckForAndDeleteOldDBFile();
   nsresult EnsureInitStorage();
 
   // This is a proxy for the information we need from an nsIURI
   struct UriInfo {
     nsAutoCString spec;
     nsAutoCString origin;
@@ -147,16 +150,26 @@ private:
 
   void LearnForToplevel(const UriInfo &uri);
   void LearnForSubresource(const UriInfo &targetURI, const UriInfo &sourceURI);
   void LearnForRedirect(const UriInfo &targetURI, const UriInfo &sourceURI);
   void LearnForStartup(const UriInfo &uri);
 
   void ResetInternal();
 
+  void BeginTransaction()
+  {
+    mDB->BeginTransaction();
+  }
+
+  void CommitTransaction()
+  {
+    mDB->CommitTransaction();
+  }
+
   // Observer-related stuff
   nsresult InstallObserver();
   void RemoveObserver();
 
   bool mInitialized;
 
   bool mEnabled;
   bool mEnableHoverOnSSL;
@@ -195,14 +208,21 @@ private:
   nsCOMPtr<nsIDNSService> mDnsService;
 
   int32_t mQueueSize;
   mozilla::Mutex mQueueSizeLock;
 
   nsAutoPtr<SeerTelemetryAccumulators> mAccumulators;
 
   nsRefPtr<SeerDNSListener> mDNSListener;
+
+  nsCOMPtr<nsITimer> mCommitTimer;
+
+#ifdef SEER_TESTS
+  friend class SeerPrepareForDnsTestEvent;
+  void PrepareForDnsTestInternal(int64_t timestamp, const nsACString &uri);
+#endif
 };
 
 } // ::mozilla::net
 } // ::mozilla
 
 #endif // mozilla_net_Seer_h
--- a/netwerk/base/src/moz.build
+++ b/netwerk/base/src/moz.build
@@ -124,8 +124,11 @@ if 'rtsp' in CONFIG['NECKO_PROTOCOLS']:
         '/netwerk/protocol/rtsp/controller',
         '/netwerk/protocol/rtsp/rtsp',
     ]
 
 if CONFIG['MOZ_ENABLE_QTNETWORK']:
     LOCAL_INCLUDES += [
         '/netwerk/system/qt',
     ]
+
+if CONFIG['ENABLE_TESTS']:
+    DEFINES['SEER_TESTS'] = True
--- a/netwerk/test/unit/test_seer.js
+++ b/netwerk/test/unit/test_seer.js
@@ -211,24 +211,17 @@ DnsContinueVerifier.prototype = {
     // This means that the seer has learned and done our "checkpoint" prediction
     // Now we can get on with the prediction we actually want to test
 
     // tstamp is 10 days older than now - just over 1 week, which will ensure we
     // hit our cutoff for dns vs. preconnect. This is all in usec, hence the
     // x1000 on the Date object value.
     var tstamp = (new Date().valueOf() * 1000) - (10 * 86400 * 1000000);
 
-    var dbfile = FileUtils.getFile("ProfD", ["netpredictions.sqlite"]);
-    var dbconn = Services.storage.openDatabase(dbfile);
-    // We also need to update hits, since the toplevel has been "loaded" a
-    // second time (from the prediction that kicked off this callback) to ensure
-    // that the seer will try to do anything for this subresource.
-    var stmt = "UPDATE moz_subresources SET last_hit = " + tstamp + ", hits = 2 WHERE uri = '" + this.subresource + "';";
-    dbconn.executeSimpleSQL(stmt);
-    dbconn.close();
+    seer.prepareForDnsTest(tstamp, this.subresource);
 
     var verifier = new Verifier("dns", [], this.preresolves);
     seer.predict(this.tluri, null, seer.PREDICT_LOAD, load_context, verifier);
   },
 
   onPredictDNS: function _dnsContinueVerifier_onPredictDNS() {
     do_check_true(false, "Shouldn't have gotten a preresolve prediction here!");
   }