Bug 1287059 - Part 2: Save/get the list states to/from prefs temporarily. r=francois.
authorHenry Chang <hchang@mozilla.com>
Fri, 12 Aug 2016 17:29:42 +0800
changeset 311074 f992cf77d4548ecc1495642741de1534e013e60d
parent 311073 49f638afba6e956d9267a5829ec71d4bfe701697
child 311075 d58a180022cf3405d9683c8a61d9ca7371d8b02a
push id30602
push userkwierso@gmail.com
push dateThu, 25 Aug 2016 23:53:05 +0000
treeherdermozilla-central@cd4ed9909dc9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfrancois
bugs1287059
milestone51.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1287059 - Part 2: Save/get the list states to/from prefs temporarily. r=francois. MozReview-Commit-ID: GaDExi2d0YX
toolkit/components/url-classifier/ProtocolParser.cpp
toolkit/components/url-classifier/content/listmanager.js
toolkit/components/url-classifier/tests/unit/test_listmanager.js
--- a/toolkit/components/url-classifier/ProtocolParser.cpp
+++ b/toolkit/components/url-classifier/ProtocolParser.cpp
@@ -7,16 +7,17 @@
 #include "LookupCache.h"
 #include "nsNetCID.h"
 #include "mozilla/Logging.h"
 #include "prnetdb.h"
 #include "prprf.h"
 
 #include "nsUrlClassifierUtils.h"
 #include "nsPrintfCString.h"
+#include "mozilla/Base64.h"
 
 // MOZ_LOG=UrlClassifierProtocolParser:5
 mozilla::LazyLogModule gUrlClassifierProtocolParserLog("UrlClassifierProtocolParser");
 #define PARSER_LOG(args) MOZ_LOG(gUrlClassifierProtocolParserLog, mozilla::LogLevel::Debug, args)
 
 namespace mozilla {
 namespace safebrowsing {
 
@@ -774,16 +775,37 @@ ProtocolParserProtobuf::End()
     if (NS_SUCCEEDED(rv)) {
       mUpdateStatus = rv;
     } else {
       NS_WARNING("Failed to process one response.");
     }
   }
 }
 
+// Save state of |aListName| to the following pref:
+//
+//   "browser.safebrowsing.provider.google4.state.[aListName]"
+//
+static nsresult
+SaveStateToPref(const nsACString& aListName, const nsACString& aState)
+{
+  nsresult rv;
+  nsCOMPtr<nsIPrefBranch> prefs(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCString prefName("browser.safebrowsing.provider.google4.state.");
+  prefName.Append(aListName);
+
+  nsCString stateBase64;
+  rv = Base64Encode(aState, stateBase64);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return prefs->SetCharPref(prefName.get(), stateBase64.get());
+}
+
 nsresult
 ProtocolParserProtobuf::ProcessOneResponse(const ListUpdateResponse& aResponse)
 {
   // A response must have a threat type.
   if (!aResponse.has_threat_type()) {
     NS_WARNING("Threat type not initialized. This seems to be an invalid response.");
     return NS_ERROR_FAILURE;
   }
@@ -833,16 +855,25 @@ ProtocolParserProtobuf::ProcessOneRespon
     NS_WARNING("New state not initialized.");
     return NS_ERROR_FAILURE;
   }
 
   auto tu = GetTableUpdate(nsCString(listName.get()));
   auto tuV4 = TableUpdate::Cast<TableUpdateV4>(tu);
   NS_ENSURE_TRUE(tuV4, NS_ERROR_FAILURE);
 
+  // See Bug 1287059. We save the state to prefs until we support
+  // "saving states to HashStore".
+  nsCString state(aResponse.new_client_state().c_str(),
+                  aResponse.new_client_state().size());
+  NS_DispatchToMainThread(NS_NewRunnableFunction([listName, state] () {
+    nsresult rv = SaveStateToPref(listName, state);
+    NS_WARN_IF(NS_FAILED(rv));
+  }));
+
   PARSER_LOG(("==== Update for threat type '%d' ====", aResponse.threat_type()));
   PARSER_LOG(("* listName: %s\n", listName.get()));
   PARSER_LOG(("* newState: %s\n", aResponse.new_client_state().c_str()));
   PARSER_LOG(("* isFullUpdate: %s\n", (isFullUpdate ? "yes" : "no")));
   ProcessAdditionOrRemoval(*tuV4, aResponse.additions(), true /*aIsAddition*/);
   ProcessAdditionOrRemoval(*tuV4, aResponse.removals(), false);
   PARSER_LOG(("\n\n"));
 
--- a/toolkit/components/url-classifier/content/listmanager.js
+++ b/toolkit/components/url-classifier/content/listmanager.js
@@ -391,23 +391,29 @@ PROT_ListManager.prototype.makeUpdateReq
     let tableArray = streamerMap.tableList.split(',');
 
     // The state is a byte stream which server told us from the
     // last table update. The state would be used to do the partial
     // update and the empty string means the table has
     // never been downloaded. See Bug 1287058 for supporting
     // partial update.
     let stateArray = [];
-    tableArray.forEach(() => stateArray.push(''));
+    tableArray.forEach(listName => {
+      // See Bug 1287059. We save the state to prefs until we support
+      // "saving states to HashStore".
+      let statePrefName = "browser.safebrowsing.provider.google4.state." + listName;
+      let stateBase64 = this.prefs_.getPref(statePrefName, "");
+      stateArray.push(stateBase64 ? atob(stateBase64) : "");
+    });
 
     let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
                      .getService(Ci.nsIUrlClassifierUtils);
     let requestPayload =  urlUtils.makeUpdateRequestV4(tableArray,
-                                                stateArray,
-                                                tableArray.length);
+                                                       stateArray,
+                                                       tableArray.length);
     // Use a base64-encoded request.
     streamerMap.requestPayload = btoa(requestPayload);
     streamerMap.isPostRequest = false;
   } else {
     // Build the request. For each table already in the database, include the
     // chunk data from the database
     var lines = tableData.split("\n");
     for (var i = 0; i < lines.length; i++) {
--- a/toolkit/components/url-classifier/tests/unit/test_listmanager.js
+++ b/toolkit/components/url-classifier/tests/unit/test_listmanager.js
@@ -57,16 +57,18 @@ let gExpectedQueryV4 = "";
 // Handles request for TEST_TABLE_DATA_V4.
 let gHttpServV4 = null;
 
 // These two variables are used to synchronize the last two racing updates
 // (in terms of "update URL") in test_update_all_tables().
 let gUpdatedCntForTableData = 0; // For TEST_TABLE_DATA_LIST.
 let gIsV4Updated = false;   // For TEST_TABLE_DATA_V4.
 
+const NEW_CLIENT_STATE = 'sta\0te';
+
 prefBranch.setBoolPref("browser.safebrowsing.debug", true);
 
 // Register tables.
 TEST_TABLE_DATA_LIST.forEach(function(t) {
   gListManager.registerTable(t.tableName,
                              t.providerName,
                              t.updateUrl,
                              t.gethashUrl);
@@ -138,16 +140,32 @@ const SERVER_INVOLVED_TEST_CASE_LIST = [
 
     forceTableUpdate();
   },
 
 ];
 
 SERVER_INVOLVED_TEST_CASE_LIST.forEach(t => add_test(t));
 
+add_test(function test_partialUpdateV4() {
+  disableAllUpdates();
+
+  gListManager.enableUpdate(TEST_TABLE_DATA_V4.tableName);
+
+  // Since the new client state has been responded and saved in
+  // test_update_all_tables, this update request should send
+  // a partial update to the server.
+  let requestV4 = gUrlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName],
+                                                [NEW_CLIENT_STATE],
+                                                1);
+  gExpectedQueryV4 = "&$req=" + btoa(requestV4);
+
+  forceTableUpdate();
+});
+
 // Tests nsIUrlListManager.getGethashUrl.
 add_test(function test_getGethashUrl() {
   TEST_TABLE_DATA_LIST.forEach(function (t) {
     equal(gListManager.getGethashUrl(t.tableName), t.gethashUrl);
   });
   equal(gListManager.getGethashUrl(TEST_TABLE_DATA_V4.tableName),
         TEST_TABLE_DATA_V4.gethashUrl);
   run_next_test();
@@ -208,28 +226,54 @@ function run_test() {
     equal(request.queryString, gExpectedQueryV4);
 
     // Respond a V2 compatible content for now. In the future we can
     // send a meaningful response to test Bug 1284178 to see if the
     // update is successfully stored to database.
     response.setHeader("Content-Type",
                        "application/vnd.google.safebrowsing-update", false);
     response.setStatusLine(request.httpVersion, 200, "OK");
-    let content = "n:1000\n";
+
+    // The protobuf binary represention of response:
+    //
+    // [
+    //   {
+    //     'threat_type': 2, // SOCIAL_ENGINEERING_PUBLIC
+    //     'response_type': 2, // FULL_UPDATE
+    //     'new_client_state': 'sta\x00te' // NEW_CLIENT_STATE
+    //   }
+    // ]
+    //
+    let content = "\x0A\x0C\x08\x02\x20\x02\x3A\x06\x73\x74\x61\x00\x74\x65";
+
     response.bodyOutputStream.write(content, content.length);
 
-    gIsV4Updated = true;
-
-    if (gUpdatedCntForTableData === SERVER_INVOLVED_TEST_CASE_LIST.length) {
-      // All tests are done!
+    if (gIsV4Updated) {
+      // This falls to the case where test_partialUpdateV4 is running.
+      // We are supposed to have verified the update request contains
+      // the state we set in the previous request.
       run_next_test();
       return;
     }
 
-    do_print("Wait for all sever-involved tests to be done ...");
+    // See Bug 1284204. We save the state to pref at the moment to
+    // support partial update until "storing to HashStore" is supported.
+    // Here we poll the pref until the state has been saved.
+    waitUntilStateSavedToPref(NEW_CLIENT_STATE, () => {
+      gIsV4Updated = true;
+
+      if (gUpdatedCntForTableData === SERVER_INVOLVED_TEST_CASE_LIST.length) {
+        // All tests are done!
+        run_next_test();
+        return;
+      }
+
+      do_print("Wait for all sever-involved tests to be done ...");
+    });
+
   });
 
   gHttpServV4.start(5555);
 
   run_next_test();
 }
 
 // A trick to force updating tables. However, before calling this, we have to
@@ -269,8 +313,31 @@ function readFileToString(aFilename) {
 
 function buildUpdateRequestV4InBase64() {
 
   let request =  urlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName],
                                               [""],
                                               1);
   return btoa(request);
 }
+
+function waitUntilStateSavedToPref(expectedState, callback) {
+  const STATE_PREF_NAME_PREFIX = 'browser.safebrowsing.provider.google4.state.';
+
+  let stateBase64 = '';
+
+  try {
+    // The reason we get pref from 'googpub-phish-proto' instead of
+    // 'test-phish-proto' is 'googpub-phish-proto' would be returned
+    // while we look up the list name from SOCIAL_ENGINEERING_PUBLIC.
+    // See nsUrlClassifierUtils::THREAT_TYPE_CONV_TABLE.
+    stateBase64 =
+      prefBranch.getCharPref(STATE_PREF_NAME_PREFIX + 'test-phish-proto');
+  } catch (e) {}
+
+  if (stateBase64 === btoa(expectedState)) {
+    do_print('State has been saved to pref!');
+    callback();
+    return;
+  }
+
+  do_timeout(1000, waitUntilStateSavedToPref.bind(null, expectedState, callback));
+}