Bug 1287059 - Part 2: Save/get the list states to/from prefs temporarily. r=francois. draft
authorHenry Chang <hchang@mozilla.com>
Fri, 12 Aug 2016 17:29:42 +0800
changeset 404310 e841f30417471689a9ad197d2f5cd770cdfe76f3
parent 404308 8c4a8054eabf169ed8203a3ff1709bb9c53cb5d4
child 529144 684cc2f11eb7a1996795b773191d27a8fabc5b6a
push id27168
push userhchang@mozilla.com
push dateTue, 23 Aug 2016 07:28:33 +0000
reviewersfrancois
bugs1287059
milestone51.0a1
Bug 1287059 - Part 2: Save/get the list states to/from prefs temporarily. r=francois. MozReview-Commit-ID: GaDExi2d0YX
toolkit/components/url-classifier/ProtocolParser.cpp
toolkit/components/url-classifier/content/listmanager.js
toolkit/components/url-classifier/tests/unit/test_listmanager.js
--- a/toolkit/components/url-classifier/ProtocolParser.cpp
+++ b/toolkit/components/url-classifier/ProtocolParser.cpp
@@ -7,16 +7,17 @@
 #include "LookupCache.h"
 #include "nsNetCID.h"
 #include "mozilla/Logging.h"
 #include "prnetdb.h"
 #include "prprf.h"
 
 #include "nsUrlClassifierUtils.h"
 #include "nsPrintfCString.h"
+#include "mozilla/Base64.h"
 
 // MOZ_LOG=UrlClassifierProtocolParser:5
 mozilla::LazyLogModule gUrlClassifierProtocolParserLog("UrlClassifierProtocolParser");
 #define PARSER_LOG(args) MOZ_LOG(gUrlClassifierProtocolParserLog, mozilla::LogLevel::Debug, args)
 
 namespace mozilla {
 namespace safebrowsing {
 
@@ -774,16 +775,37 @@ ProtocolParserProtobuf::End()
     if (NS_SUCCEEDED(rv)) {
       mUpdateStatus = rv;
     } else {
       NS_WARNING("Failed to process one response.");
     }
   }
 }
 
+// Save state of |aListName| to the following pref:
+//
+//   "browser.safebrowsing.provider.google4.state.[aListName]"
+//
+static nsresult
+SaveStateToPref(const nsACString& aListName, const nsACString& aState)
+{
+  nsresult rv;
+  nsCOMPtr<nsIPrefBranch> prefs(do_GetService(NS_PREFSERVICE_CONTRACTID, &rv));
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  nsCString prefName("browser.safebrowsing.provider.google4.state.");
+  prefName.Append(aListName);
+
+  nsCString stateBase64;
+  rv = Base64Encode(aState, stateBase64);
+  NS_ENSURE_SUCCESS(rv, rv);
+
+  return prefs->SetCharPref(prefName.get(), stateBase64.get());
+}
+
 nsresult
 ProtocolParserProtobuf::ProcessOneResponse(const ListUpdateResponse& aResponse)
 {
   // A response must have a threat type.
   if (!aResponse.has_threat_type()) {
     NS_WARNING("Threat type not initialized. This seems to be an invalid response.");
     return NS_ERROR_FAILURE;
   }
@@ -833,16 +855,25 @@ ProtocolParserProtobuf::ProcessOneRespon
     NS_WARNING("New state not initialized.");
     return NS_ERROR_FAILURE;
   }
 
   auto tu = GetTableUpdate(nsCString(listName.get()));
   auto tuV4 = TableUpdate::Cast<TableUpdateV4>(tu);
   NS_ENSURE_TRUE(tuV4, NS_ERROR_FAILURE);
 
+  // See Bug 1287059. We save the state to prefs until we support
+  // "saving states to HashStore".
+  nsCString state(aResponse.new_client_state().c_str(),
+                  aResponse.new_client_state().size());
+  NS_DispatchToMainThread(NS_NewRunnableFunction([listName, state] () {
+    nsresult rv = SaveStateToPref(listName, state);
+    NS_WARN_IF(NS_FAILED(rv));
+  }));
+
   PARSER_LOG(("==== Update for threat type '%d' ====", aResponse.threat_type()));
   PARSER_LOG(("* listName: %s\n", listName.get()));
   PARSER_LOG(("* newState: %s\n", aResponse.new_client_state().c_str()));
   PARSER_LOG(("* isFullUpdate: %s\n", (isFullUpdate ? "yes" : "no")));
   ProcessAdditionOrRemoval(*tuV4, aResponse.additions(), true /*aIsAddition*/);
   ProcessAdditionOrRemoval(*tuV4, aResponse.removals(), false);
   PARSER_LOG(("\n\n"));
 
--- a/toolkit/components/url-classifier/content/listmanager.js
+++ b/toolkit/components/url-classifier/content/listmanager.js
@@ -391,23 +391,29 @@ PROT_ListManager.prototype.makeUpdateReq
     let tableArray = streamerMap.tableList.split(',');
 
     // The state is a byte stream which server told us from the
     // last table update. The state would be used to do the partial
     // update and the empty string means the table has
     // never been downloaded. See Bug 1287058 for supporting
     // partial update.
     let stateArray = [];
-    tableArray.forEach(() => stateArray.push(''));
+    tableArray.forEach(listName => {
+      // See Bug 1287059. We save the state to prefs until we support
+      // "saving states to HashStore".
+      let statePrefName = "browser.safebrowsing.provider.google4.state." + listName;
+      let stateBase64 = this.prefs_.getPref(statePrefName, "");
+      stateArray.push(stateBase64 ? atob(stateBase64) : "");
+    });
 
     let urlUtils = Cc["@mozilla.org/url-classifier/utils;1"]
                      .getService(Ci.nsIUrlClassifierUtils);
     let requestPayload =  urlUtils.makeUpdateRequestV4(tableArray,
-                                                stateArray,
-                                                tableArray.length);
+                                                       stateArray,
+                                                       tableArray.length);
     // Use a base64-encoded request.
     streamerMap.requestPayload = btoa(requestPayload);
     streamerMap.isPostRequest = false;
   } else {
     // Build the request. For each table already in the database, include the
     // chunk data from the database
     var lines = tableData.split("\n");
     for (var i = 0; i < lines.length; i++) {
--- a/toolkit/components/url-classifier/tests/unit/test_listmanager.js
+++ b/toolkit/components/url-classifier/tests/unit/test_listmanager.js
@@ -57,16 +57,18 @@ let gExpectedQueryV4 = "";
 // Handles request for TEST_TABLE_DATA_V4.
 let gHttpServV4 = null;
 
 // These two variables are used to synchronize the last two racing updates
 // (in terms of "update URL") in test_update_all_tables().
 let gUpdatedCntForTableData = 0; // For TEST_TABLE_DATA_LIST.
 let gIsV4Updated = false;   // For TEST_TABLE_DATA_V4.
 
+const NEW_CLIENT_STATE = 'sta\0te';
+
 prefBranch.setBoolPref("browser.safebrowsing.debug", true);
 
 // Register tables.
 TEST_TABLE_DATA_LIST.forEach(function(t) {
   gListManager.registerTable(t.tableName,
                              t.providerName,
                              t.updateUrl,
                              t.gethashUrl);
@@ -138,16 +140,32 @@ const SERVER_INVOLVED_TEST_CASE_LIST = [
 
     forceTableUpdate();
   },
 
 ];
 
 SERVER_INVOLVED_TEST_CASE_LIST.forEach(t => add_test(t));
 
+add_test(function test_partialUpdateV4() {
+  disableAllUpdates();
+
+  gListManager.enableUpdate(TEST_TABLE_DATA_V4.tableName);
+
+  // Since the new client state has been responded and saved in
+  // test_update_all_tables, this update request should send
+  // a partial update to the server.
+  let requestV4 = gUrlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName],
+                                                [NEW_CLIENT_STATE],
+                                                1);
+  gExpectedQueryV4 = "&$req=" + btoa(requestV4);
+
+  forceTableUpdate();
+});
+
 // Tests nsIUrlListManager.getGethashUrl.
 add_test(function test_getGethashUrl() {
   TEST_TABLE_DATA_LIST.forEach(function (t) {
     equal(gListManager.getGethashUrl(t.tableName), t.gethashUrl);
   });
   equal(gListManager.getGethashUrl(TEST_TABLE_DATA_V4.tableName),
         TEST_TABLE_DATA_V4.gethashUrl);
   run_next_test();
@@ -208,28 +226,54 @@ function run_test() {
     equal(request.queryString, gExpectedQueryV4);
 
     // Respond a V2 compatible content for now. In the future we can
     // send a meaningful response to test Bug 1284178 to see if the
     // update is successfully stored to database.
     response.setHeader("Content-Type",
                        "application/vnd.google.safebrowsing-update", false);
     response.setStatusLine(request.httpVersion, 200, "OK");
-    let content = "n:1000\n";
+
+    // The protobuf binary represention of response:
+    //
+    // [
+    //   {
+    //     'threat_type': 2, // SOCIAL_ENGINEERING_PUBLIC
+    //     'response_type': 2, // FULL_UPDATE
+    //     'new_client_state': 'sta\x00te' // NEW_CLIENT_STATE
+    //   }
+    // ]
+    //
+    let content = "\x0A\x0C\x08\x02\x20\x02\x3A\x06\x73\x74\x61\x00\x74\x65";
+
     response.bodyOutputStream.write(content, content.length);
 
-    gIsV4Updated = true;
-
-    if (gUpdatedCntForTableData === SERVER_INVOLVED_TEST_CASE_LIST.length) {
-      // All tests are done!
+    if (gIsV4Updated) {
+      // This falls to the case where test_partialUpdateV4 is running.
+      // We are supposed to have verified the update request contains
+      // the state we set in the previous request.
       run_next_test();
       return;
     }
 
-    do_print("Wait for all sever-involved tests to be done ...");
+    // See Bug 1284204. We save the state to pref at the moment to
+    // support partial update until "storing to HashStore" is supported.
+    // Here we poll the pref until the state has been saved.
+    waitUntilStateSavedToPref(NEW_CLIENT_STATE, () => {
+      gIsV4Updated = true;
+
+      if (gUpdatedCntForTableData === SERVER_INVOLVED_TEST_CASE_LIST.length) {
+        // All tests are done!
+        run_next_test();
+        return;
+      }
+
+      do_print("Wait for all sever-involved tests to be done ...");
+    });
+
   });
 
   gHttpServV4.start(5555);
 
   run_next_test();
 }
 
 // A trick to force updating tables. However, before calling this, we have to
@@ -269,8 +313,31 @@ function readFileToString(aFilename) {
 
 function buildUpdateRequestV4InBase64() {
 
   let request =  urlUtils.makeUpdateRequestV4([TEST_TABLE_DATA_V4.tableName],
                                               [""],
                                               1);
   return btoa(request);
 }
+
+function waitUntilStateSavedToPref(expectedState, callback) {
+  const STATE_PREF_NAME_PREFIX = 'browser.safebrowsing.provider.google4.state.';
+
+  let stateBase64 = '';
+
+  try {
+    // The reason we get pref from 'googpub-phish-proto' instead of
+    // 'test-phish-proto' is 'googpub-phish-proto' would be returned
+    // while we look up the list name from SOCIAL_ENGINEERING_PUBLIC.
+    // See nsUrlClassifierUtils::THREAT_TYPE_CONV_TABLE.
+    stateBase64 =
+      prefBranch.getCharPref(STATE_PREF_NAME_PREFIX + 'test-phish-proto');
+  } catch (e) {}
+
+  if (stateBase64 === btoa(expectedState)) {
+    do_print('State has been saved to pref!');
+    callback();
+    return;
+  }
+
+  do_timeout(1000, waitUntilStateSavedToPref.bind(null, expectedState, callback));
+}