Bug 1175562 - Persist last update time for SafeBrowsing. r=francois
authorGian-Carlo Pascutto <gcp@mozilla.com>
Tue, 06 Oct 2015 13:53:07 +0200
changeset 266373 988fdc8043e03c28301a45910c2c4076f58fd6f6
parent 266257 2722b65059df232c43dfaec075dfe484e0db952c
child 266374 f4c2f277aeae7bf8b05c6b01d1e140cd51b693b4
push id29493
push userkwierso@gmail.com
push dateWed, 07 Oct 2015 17:31:17 +0000
treeherdermozilla-central@49d87bbe0122 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfrancois
bugs1175562
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1175562 - Persist last update time for SafeBrowsing. r=francois
toolkit/components/build/nsToolkitCompsCID.h
toolkit/components/url-classifier/Classifier.cpp
toolkit/components/url-classifier/Classifier.h
toolkit/components/url-classifier/SafeBrowsing.jsm
toolkit/components/url-classifier/content/listmanager.js
toolkit/components/url-classifier/nsIUrlClassifierDBService.idl
toolkit/components/url-classifier/nsIUrlListManager.idl
toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
toolkit/components/url-classifier/nsUrlClassifierProxies.cpp
toolkit/components/url-classifier/nsUrlClassifierProxies.h
--- a/toolkit/components/build/nsToolkitCompsCID.h
+++ b/toolkit/components/build/nsToolkitCompsCID.h
@@ -135,19 +135,19 @@
 // {e7f70966-9a37-48d7-8aeb-35998f31090e}
 #define NS_TYPEAHEADFIND_CID \
 { 0xe7f70966, 0x9a37, 0x48d7, { 0x8a, 0xeb, 0x35, 0x99, 0x8f, 0x31, 0x09, 0x0e} }
 
 // {3d8579f0-75fa-4e00-ba41-38661d5b5d17}
  #define NS_URLCLASSIFIERPREFIXSET_CID \
 { 0x3d8579f0, 0x75fa, 0x4e00, { 0xba, 0x41, 0x38, 0x66, 0x1d, 0x5b, 0x5d, 0x17} }
 
-// {8a389f21-f821-4e29-9c6b-3de6f33cd7cf}
+// {7a258022-6765-11e5-b379-b37b1f2354be}
 #define NS_URLCLASSIFIERDBSERVICE_CID \
-{ 0x8a389f21, 0xf821, 0x4e29, { 0x9c, 0x6b, 0x3d, 0xe6, 0xf3, 0x3c, 0xd7, 0xcf} }
+{ 0x7a258022, 0x6765, 0x11e5, { 0xb3, 0x79, 0xb3, 0x7b, 0x1f, 0x23, 0x54, 0xbe} }
 
 // e1797597-f4d6-4dd3-a1e1-745ad352cd80
 #define NS_URLCLASSIFIERSTREAMUPDATER_CID \
 { 0xe1797597, 0xf4d6, 0x4dd3, { 0xa1, 0xe1, 0x74, 0x5a, 0xd3, 0x52, 0xcd, 0x80 }}
 
 // {b7b2ccec-7912-4ea6-a548-b038447004bd}
 #define NS_URLCLASSIFIERUTILS_CID \
 { 0xb7b2ccec, 0x7912, 0x4ea6, { 0xa5, 0x48, 0xb0, 0x38, 0x44, 0x70, 0x04, 0xbd} }
--- a/toolkit/components/url-classifier/Classifier.cpp
+++ b/toolkit/components/url-classifier/Classifier.cpp
@@ -364,16 +364,25 @@ Classifier::MarkSpoiled(nsTArray<nsCStri
     if (cache) {
       cache->ClearCompleteCache();
     }
   }
   return NS_OK;
 }
 
 void
+Classifier::SetLastUpdateTime(const nsACString &aTable,
+                              uint64_t updateTime)
+{
+  LOG(("Marking table %s as last updated on %u",
+       PromiseFlatCString(aTable).get(), updateTime));
+  mTableFreshness.Put(aTable, updateTime / PR_MSEC_PER_SEC);
+}
+
+void
 Classifier::DropStores()
 {
   for (uint32_t i = 0; i < mHashStores.Length(); i++) {
     delete mHashStores[i];
   }
   mHashStores.Clear();
   for (uint32_t i = 0; i < mLookupCaches.Length(); i++) {
     delete mLookupCaches[i];
--- a/toolkit/components/url-classifier/Classifier.h
+++ b/toolkit/components/url-classifier/Classifier.h
@@ -55,16 +55,17 @@ public:
    * the updates in the array and clears it.  Wacky!
    */
   nsresult ApplyUpdates(nsTArray<TableUpdate*>* aUpdates);
   /**
    * Failed update. Spoil the entries so we don't block hosts
    * unnecessarily
    */
   nsresult MarkSpoiled(nsTArray<nsCString>& aTables);
+  void SetLastUpdateTime(const nsACString& aTableName, uint64_t updateTime);
   nsresult CacheCompletions(const CacheResultArray& aResults);
   uint32_t GetHashKey(void) { return mHashKey; }
   /*
    * Get a bunch of extra prefixes to query for completion
    * and mask the real entry being requested
    */
   nsresult ReadNoiseEntries(const Prefix& aPrefix,
                             const nsACString& aTableName,
--- a/toolkit/components/url-classifier/SafeBrowsing.jsm
+++ b/toolkit/components/url-classifier/SafeBrowsing.jsm
@@ -75,17 +75,17 @@ this.SafeBrowsing = {
 
   registerTableWithURLs: function(listname) {
     let listManager = Cc["@mozilla.org/url-classifier/listmanager;1"].
       getService(Ci.nsIUrlListManager);
 
     let providerName = this.listToProvider[listname];
     let provider = this.providers[providerName];
 
-    listManager.registerTable(listname, provider.updateURL, provider.gethashURL);
+    listManager.registerTable(listname, providerName, provider.updateURL, provider.gethashURL);
   },
 
   registerTables: function() {
     for (let i = 0; i < phishingLists.length; ++i) {
       this.registerTableWithURLs(phishingLists[i]);
     }
     for (let i = 0; i < malwareLists.length; ++i) {
       this.registerTableWithURLs(malwareLists[i]);
--- a/toolkit/components/url-classifier/content/listmanager.js
+++ b/toolkit/components/url-classifier/content/listmanager.js
@@ -88,26 +88,28 @@ PROT_ListManager.prototype.shutdown_ = f
 /**
  * Register a new table table
  * @param tableName - the name of the table
  * @param updateUrl - the url for updating the table
  * @param gethashUrl - the url for fetching hash completions
  * @returns true if the table could be created; false otherwise
  */
 PROT_ListManager.prototype.registerTable = function(tableName,
+                                                    providerName,
                                                     updateUrl,
                                                     gethashUrl) {
   log("registering " + tableName + " with " + updateUrl);
   if (!updateUrl) {
     log("Can't register table " + tableName + " without updateUrl");
     return false;
   }
   this.tablesData[tableName] = {};
   this.tablesData[tableName].updateUrl = updateUrl;
   this.tablesData[tableName].gethashUrl = gethashUrl;
+  this.tablesData[tableName].provider = providerName;
 
   // Keep track of all of our update URLs.
   if (!this.needsUpdate_[updateUrl]) {
     this.needsUpdate_[updateUrl] = {};
     /* Backoff interval should be between 30 and 60 minutes. */
     var backoffInterval = 30 * 60 * 1000;
     backoffInterval += Math.floor(Math.random() * (30 * 60 * 1000));
 
@@ -191,30 +193,68 @@ PROT_ListManager.prototype.requireTableU
 
 /**
  * Acts as a nsIUrlClassifierCallback for getTables.
  */
 PROT_ListManager.prototype.kickoffUpdate_ = function (onDiskTableData)
 {
   this.startingUpdate_ = false;
   var initialUpdateDelay = 3000;
+  // Add a fuzz of 0-5 minutes.
+  initialUpdateDelay += Math.floor(Math.random() * (5 * 60 * 1000));
 
   // If the user has never downloaded tables, do the check now.
   log("needsUpdate: " + JSON.stringify(this.needsUpdate_, undefined, 2));
   for (var updateUrl in this.needsUpdate_) {
     // If we haven't already kicked off updates for this updateUrl, set a
     // non-repeating timer for it. The timer delay will be reset either on
     // updateSuccess to this.updateinterval, or backed off on downloadError.
     // Don't set the updateChecker unless at least one table has updates
     // enabled.
     if (this.updatesNeeded_(updateUrl) && !this.updateCheckers_[updateUrl]) {
-      log("Initializing update checker for " + updateUrl);
+      let provider = null;
+      Object.keys(this.tablesData).forEach(function(table) {
+        if (this.tablesData[table].updateUrl === updateUrl) {
+          let newProvider = this.tablesData[table].provider;
+          if (provider) {
+            if (newProvider !== provider) {
+              log("Multiple tables for the same updateURL have a different provider?!");
+            }
+          } else {
+            provider = newProvider;
+          }
+        }
+      }, this);
+      log("Initializing update checker for " + updateUrl
+          + " provided by " + provider);
+
+      // Use the initialUpdateDelay + fuzz unless we had previous updates
+      // and the server told us when to try again.
+      let updateDelay = initialUpdateDelay;
+      let targetPref = "browser.safebrowsing.provider." + provider + ".nextupdatetime";
+      let nextUpdate = this.prefs_.getPref(targetPref);
+      if (nextUpdate) {
+        updateDelay = Math.max(0, nextUpdate - Date.now());
+        log("Next update at " + nextUpdate
+            + " which is " + updateDelay + "ms from now");
+      }
+
+      // Set the last update time to verify if data is still valid.
+      let freshnessPref = "browser.safebrowsing.provider." + provider + ".lastupdatetime";
+      let freshness = this.prefs_.getPref(freshnessPref);
+      if (freshness) {
+        Object.keys(this.tablesData).forEach(function(table) {
+        if (this.tablesData[table].provider === provider) {
+          this.dbService_.setLastUpdateTime(table, freshness);
+        }}, this);
+      }
+
       this.updateCheckers_[updateUrl] =
         new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl),
-                    initialUpdateDelay, false /* repeating */);
+                    updateDelay, false /* repeating */);
     } else {
       log("No updates needed or already initialized for " + updateUrl);
     }
   }
 }
 
 PROT_ListManager.prototype.stopUpdateCheckers = function() {
   log("Stopping updates");
@@ -402,16 +442,44 @@ PROT_ListManager.prototype.updateSuccess
     delay = this.updateInterval;
   }
   this.updateCheckers_[updateUrl] =
     new G_Alarm(BindToObject(this.checkForUpdates, this, updateUrl),
                 delay, false);
 
   // Let the backoff object know that we completed successfully.
   this.requestBackoffs_[updateUrl].noteServerResponse(200);
+
+  // Set last update time for provider
+  // Get the provider for these tables, check for consistency
+  let tables = tableList.split(",");
+  let provider = null;
+  for (let table of tables) {
+    let newProvider = this.tablesData[table].provider;
+    if (provider) {
+      if (newProvider !== provider) {
+        log("Multiple tables for the same updateURL have a different provider?!");
+      }
+    } else {
+      provider = newProvider;
+    }
+  }
+
+  // Store the last update time (needed to know if the table is "fresh")
+  // and the next update time (to know when to update next).
+  let lastUpdatePref = "browser.safebrowsing.provider." + provider + ".lastupdatetime";
+  let now = Date.now();
+  log("Setting last update of " + provider + " to " + now);
+  this.prefs_.setPref(lastUpdatePref, now.toString());
+
+  let nextUpdatePref = "browser.safebrowsing.provider." + provider + ".nextupdatetime";
+  let targetTime = now + delay;
+  log("Setting next update of " + provider + " to " + targetTime
+      + " (" + delay + "ms from now)");
+  this.prefs_.setPref(nextUpdatePref, targetTime.toString());
 }
 
 /**
  * Callback function if the update request succeeded.
  * @param result String The error code of the failure
  */
 PROT_ListManager.prototype.updateError_ = function(table, updateUrl, result) {
   log("update error for " + table + " from " + updateUrl + ": " + result + "\n");
--- a/toolkit/components/url-classifier/nsIUrlClassifierDBService.idl
+++ b/toolkit/components/url-classifier/nsIUrlClassifierDBService.idl
@@ -61,17 +61,17 @@ interface nsIUrlClassifierUpdateObserver
   void updateSuccess(in unsigned long requestedTimeout);
 };
 
 /**
  * This is a proxy class that is instantiated and called from the JS thread.
  * It provides async methods for querying and updating the database.  As the
  * methods complete, they call the callback function.
  */
-[scriptable, uuid(3f9e61e5-01bd-45d0-8dd2-f1abcd20dbb7)]
+[scriptable, uuid(7a258022-6765-11e5-b379-b37b1f2354be)]
 interface nsIUrlClassifierDBService : nsISupports
 {
   /**
    * Looks up a URI in the specified tables.
    *
    * @param principal: The principal containing the URI to search.
    * @param c: The callback will be called with a comma-separated list
    *        of tables to which the key belongs.
@@ -95,16 +95,23 @@ interface nsIUrlClassifierDBService : ns
   /**
    * Set the nsIUrlClassifierCompleter object for a given table.  This
    * object will be used to request complete versions of partial
    * hashes.
    */
   void setHashCompleter(in ACString tableName,
                         in nsIUrlClassifierHashCompleter completer);
 
+  /**
+   * Set the last update time for the given table. We use this to
+   * remember freshness past restarts. Time is in milliseconds since epoch.
+   */
+  void setLastUpdateTime(in ACString tableName,
+                         in unsigned long long lastUpdateTime);
+
   ////////////////////////////////////////////////////////////////////////////
   // Incremental update methods.
   //
   // An update to the database has the following steps:
   //
   // 1) The update process is started with beginUpdate().  The client
   //    passes an nsIUrlClassifierUpdateObserver object which will be
   //    notified as the update is processed by the dbservice.
--- a/toolkit/components/url-classifier/nsIUrlListManager.idl
+++ b/toolkit/components/url-classifier/nsIUrlListManager.idl
@@ -13,34 +13,36 @@ interface nsIPrincipal;
 
 // Interface for JS function callbacks
 [scriptable, function, uuid(fa4caf12-d057-4e7e-81e9-ce066ceee90b)]
 interface nsIUrlListManagerCallback : nsISupports {
   void handleEvent(in ACString value);
 };
 
 
-[scriptable, uuid(5d5ed98f-72cd-46b6-a9fe-76418adfdfeb)]
+[scriptable, uuid(d60a08ee-5c83-4eb6-bdfb-79fd0716501e)]
 interface nsIUrlListManager : nsISupports
 {
     /**
      * Get the gethash url for this table
      */
     ACString getGethashUrl(in ACString tableName);
 
     /**
      * Add a table to the list of tables we are managing. The name is a
      * string of the format provider_name-semantic_type-table_type.  For
      * @param tableName A string of the format
      *        provider_name-semantic_type-table_type.  For example,
      *        goog-white-enchash or goog-black-url.
+     * @param providerName The name of the entity providing the list.
      * @param updateUrl The URL from which to fetch updates.
      * @param gethashUrl The URL from which to fetch hash completions.
      */
     boolean registerTable(in ACString tableName,
+                          in ACString providerName,
                           in ACString updateUrl,
                           in ACString gethashUrl);
 
     /**
      * Turn on update checking for a table. I.e., during the next server
      * check, download updates for this table.
      */
     void enableUpdate(in ACString tableName);
--- a/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierDBService.cpp
@@ -730,16 +730,28 @@ nsUrlClassifierDBServiceWorker::OpenDb()
   rv = classifier->Open(*mCacheDir);
   NS_ENSURE_SUCCESS(rv, rv);
 
   mClassifier = classifier;
 
   return NS_OK;
 }
 
+nsresult
+nsUrlClassifierDBServiceWorker::SetLastUpdateTime(const nsACString &table,
+                                                  uint64_t updateTime)
+{
+  MOZ_ASSERT(!NS_IsMainThread(), "Must be on the background thread");
+  MOZ_ASSERT(mClassifier, "Classifier connection must be opened");
+
+  mClassifier->SetLastUpdateTime(table, updateTime);
+
+  return NS_OK;
+}
+
 // -------------------------------------------------------------------------
 // nsUrlClassifierLookupCallback
 //
 // This class takes the results of a lookup found on the worker thread
 // and handles any necessary partial hash expansions before calling
 // the client callback.
 
 class nsUrlClassifierLookupCallback final : public nsIUrlClassifierLookupCallback
@@ -1404,16 +1416,25 @@ nsUrlClassifierDBService::SetHashComplet
   } else {
     mCompleters.Remove(tableName);
   }
 
   return NS_OK;
 }
 
 NS_IMETHODIMP
+nsUrlClassifierDBService::SetLastUpdateTime(const nsACString &tableName,
+                                            uint64_t lastUpdateTime)
+{
+  NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
+
+  return mWorkerProxy->SetLastUpdateTime(tableName, lastUpdateTime);
+}
+
+NS_IMETHODIMP
 nsUrlClassifierDBService::BeginUpdate(nsIUrlClassifierUpdateObserver *observer,
                                       const nsACString &updateTables)
 {
   NS_ENSURE_TRUE(gDbBackgroundThread, NS_ERROR_NOT_INITIALIZED);
 
   if (mInUpdate) {
     LOG(("Already updating, not available"));
     return NS_ERROR_NOT_AVAILABLE;
--- a/toolkit/components/url-classifier/nsUrlClassifierProxies.cpp
+++ b/toolkit/components/url-classifier/nsUrlClassifierProxies.cpp
@@ -211,16 +211,31 @@ UrlClassifierDBServiceWorkerProxy::Cache
 
 NS_IMETHODIMP
 UrlClassifierDBServiceWorkerProxy::CacheMissesRunnable::Run()
 {
   mTarget->CacheMisses(mEntries);
   return NS_OK;
 }
 
+NS_IMETHODIMP
+UrlClassifierDBServiceWorkerProxy::SetLastUpdateTime(const nsACString& table,
+                                                     uint64_t lastUpdateTime)
+{
+  nsCOMPtr<nsIRunnable> r =
+    new SetLastUpdateTimeRunnable(mTarget, table, lastUpdateTime);
+  return DispatchToWorkerThread(r);
+}
+
+NS_IMETHODIMP
+UrlClassifierDBServiceWorkerProxy::SetLastUpdateTimeRunnable::Run()
+{
+  mTarget->SetLastUpdateTime(mTable, mUpdateTime);
+  return NS_OK;
+}
 
 NS_IMPL_ISUPPORTS(UrlClassifierLookupCallbackProxy,
                   nsIUrlClassifierLookupCallback)
 
 NS_IMETHODIMP
 UrlClassifierLookupCallbackProxy::LookupComplete
   (LookupResultArray * aResults)
 {
--- a/toolkit/components/url-classifier/nsUrlClassifierProxies.h
+++ b/toolkit/components/url-classifier/nsUrlClassifierProxies.h
@@ -168,16 +168,34 @@ public:
   private:
     nsRefPtr<nsUrlClassifierDBServiceWorker> mTarget;
 
     nsCString mSpec;
     nsCString mTables;
     mozilla::safebrowsing::LookupResultArray* mResults;
   };
 
+  class SetLastUpdateTimeRunnable : public nsRunnable
+  {
+  public:
+    SetLastUpdateTimeRunnable(nsUrlClassifierDBServiceWorker* aTarget,
+                              const nsACString& table,
+                              uint64_t updateTime)
+      : mTarget(aTarget),
+        mTable(table),
+        mUpdateTime(updateTime)
+    { }
+
+    NS_DECL_NSIRUNNABLE
+  private:
+    nsRefPtr<nsUrlClassifierDBServiceWorker> mTarget;
+    nsCString mTable;
+    uint64_t mUpdateTime;
+  };
+
 public:
   nsresult DoLocalLookup(const nsACString& spec,
                          const nsACString& tables,
                          mozilla::safebrowsing::LookupResultArray* results);
 
 private:
   ~UrlClassifierDBServiceWorkerProxy() {}