Bug 1329558 - Implement Minimum wait duration for V4 gethash draft
authorThomas Nguyen <tnguyen@mozilla.com>
Wed, 08 Feb 2017 13:12:21 +0800
changeset 480412 e14fa1919144a87094d08586a003b60518eb2c5f
parent 479651 af8a2573d0f1e9cc6f2ba0ab67d7a702a197f177
child 544935 4225a1d1dc4f8571be5e4ce4d6831535264eb8f0
push id44525
push usertnguyen@mozilla.com
push dateWed, 08 Feb 2017 05:12:49 +0000
bugs1329558
milestone54.0a1
Bug 1329558 - Implement Minimum wait duration for V4 gethash MozReview-Commit-ID: 7i9Wz7pq0yJ
toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
toolkit/components/url-classifier/tests/unit/test_hashcompleter_v4.js
--- a/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
+++ b/toolkit/components/url-classifier/nsUrlClassifierHashCompleter.js
@@ -160,16 +160,19 @@ function HashCompleter() {
   this._pendingRequests = {};
 
   // A map of gethash URLs to RequestBackoff objects.
   this._backoffs = {};
 
   // Whether we have been informed of a shutdown by the shutdown event.
   this._shuttingDown = false;
 
+  // A map of getHash URLs to next gethash time
+  this._nextGethashTimes = {};
+
   Services.obs.addObserver(this, "quit-application", false);
 
 }
 
 HashCompleter.prototype = {
   classID: Components.ID("{9111de73-9322-4bfc-8b65-2b727f3e6ec8}"),
   QueryInterface: XPCOMUtils.generateQI([Ci.nsIUrlClassifierHashCompleter,
                                          Ci.nsIRunnable,
@@ -205,29 +208,36 @@ HashCompleter.prototype = {
       var jslib = Cc["@mozilla.org/url-classifier/jslib;1"]
                   .getService().wrappedJSObject;
 
       // Using the V4 backoff algorithm for both V2 and V4. See bug 1273398.
       this._backoffs[aGethashUrl] = new jslib.RequestBackoffV4(
         10 /* keep track of max requests */,
         0  /* don't throttle on successful requests per time period */);
     }
+
+    if (!this._nextGethashTimes[aGethashUrl]) {
+      this._nextGethashTimes[aGethashUrl] = 0;
+    }
+
     // Start off this request. Without dispatching to a thread, every call to
     // complete makes an individual HTTP request.
     Services.tm.currentThread.dispatch(this, Ci.nsIThread.DISPATCH_NORMAL);
   },
 
   // This is called after several calls to |complete|, or after the
   // currentRequest has finished.  It starts off the HTTP request by making a
   // |begin| call to the HashCompleterRequest.
   run: function() {
     // Clear everything on shutdown
     if (this._shuttingDown) {
       this._currentRequest = null;
       this._pendingRequests = null;
+      this._nextGethashTimes = null;
+
       for (var url in this._backoffs) {
         this._backoffs[url] = null;
       }
       throw Cr.NS_ERROR_NOT_INITIALIZED;
     }
 
     // If we don't have an in-flight request, make one
     let pendingUrls = Object.keys(this._pendingRequests);
@@ -251,17 +261,18 @@ HashCompleter.prototype = {
   // gethashUrl and fetch the next pending request, if there is one.
   finishRequest: function(url, aStatus) {
     this._backoffs[url].noteServerResponse(aStatus);
     Services.tm.currentThread.dispatch(this, Ci.nsIThread.DISPATCH_NORMAL);
   },
 
   // Returns true if we can make a request from the given url, false otherwise.
   canMakeRequest: function(aGethashUrl) {
-    return this._backoffs[aGethashUrl].canMakeRequest();
+    return this._backoffs[aGethashUrl].canMakeRequest() &&
+           (Date.now() > this._nextGethashTimes[aGethashUrl]);
   },
 
   // Notifies the RequestBackoff of a new request so we can throttle based on
   // max requests/time period. This must be called before a channel is opened,
   // and finishRequest must be called once the response is received.
   noteRequest: function(aGethashUrl) {
     return this._backoffs[aGethashUrl].noteRequest();
   },
@@ -551,16 +562,19 @@ HashCompleterRequest.prototype = {
       },
 
       onResponseParsed : (aMinWaitDuration,
                           aNegCacheDuration) => {
         log("V4 fullhash response parsed callback: " +
             "MinWaitDuration(" + aMinWaitDuration + "), " +
             "NegativeCacheDuration(" + aNegCacheDuration + ")");
 
+        this._completer._nextGethashTimes[this.gethashUrl] =
+          Date.now() + aMinWaitDuration;
+
         // TODO: Bug 1311935 - Implement v4 cache.
       },
     };
 
     gUrlUtil.parseFindFullHashResponseV4(this._response, callback);
   },
 
   // This parses a table entry in the response body and calls |handleItem|
@@ -646,16 +660,17 @@ HashCompleterRequest.prototype = {
               createInstance(Ci.nsIScriptableInputStream);
     sis.init(aInputStream);
     this._response += sis.readBytes(aCount);
   },
 
   onStartRequest: function HCR_onStartRequest(aRequest, aContext) {
     // At this point no data is available for us and we have no reason to
     // terminate the connection, so we do nothing until |onStopRequest|.
+    this._completer._nextGethashTimes[this.gethashUrl] = 0;
   },
 
   onStopRequest: function HCR_onStopRequest(aRequest, aContext, aStatusCode) {
     Services.obs.removeObserver(this, "quit-application");
 
     if (this.timer_) {
       this.timer_.cancel();
       this.timer_ = null;
--- a/toolkit/components/url-classifier/tests/unit/test_hashcompleter_v4.js
+++ b/toolkit/components/url-classifier/tests/unit/test_hashcompleter_v4.js
@@ -5,33 +5,60 @@ Cu.import("resource://gre/modules/Servic
 const TEST_TABLE_DATA_V4 = {
   tableName: "test-phish-proto",
   providerName: "google4",
   updateUrl: "http://localhost:5555/safebrowsing/update?",
   gethashUrl: "http://localhost:5555/safebrowsing/gethash-v4?",
 };
 
 const PREF_NEXTUPDATETIME_V4 = "browser.safebrowsing.provider.google4.nextupdatetime";
+const GETHASH_PATH = "/safebrowsing/gethash-v4";
+
+// The protobuf binary represention of gethash response:
+// minimumWaitDuration : 12 secs 10 nanosecs
+// negativeCacheDuration : 120 secs 9 nanosecs
+//
+// { nsCString("01234567890123456789012345678901"), SOCIAL_ENGINEERING_PUBLIC, { 8, 500 } },
+// { nsCString("12345678901234567890123456789012"), SOCIAL_ENGINEERING_PUBLIC, { 7, 100} },
+// { nsCString("23456789012345678901234567890123"), SOCIAL_ENGINEERING_PUBLIC, { 1, 20 } },
+
+const GETHASH_RESPONSE_CONTENT = "\x0A\x2D\x08\x02\x1A\x22\x0A\x20\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x2A\x05\x08\x08\x10\xF4\x03\x0A\x2C\x08\x02\x1A\x22\x0A\x20\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x2A\x04\x08\x07\x10\x64\x0A\x2C\x08\x02\x1A\x22\x0A\x20\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x2A\x04\x08\x01\x10\x14\x12\x04\x08\x0C\x10\x0A\x1A\x04\x08\x78\x10\x09";
+
+// The protobuf binary represention of update response:
+//
+// [
+//   {
+//     'threat_type': 2, // SOCIAL_ENGINEERING_PUBLIC
+//     'response_type': 2, // FULL_UPDATE
+//     'new_client_state': 'sta\x00te', // NEW_CLIENT_STATE
+//     'checksum': { "sha256": CHECKSUM }, // CHECKSUM
+//     'additions': { 'compression_type': RAW,
+//                    'prefix_size': 4,
+//                    'raw_hashes': "00000001000000020000000300000004"}
+//   }
+// ]
+//
+const UPDATE_RESPONSE_CONTENT = "\x0A\x4A\x08\x02\x20\x02\x2A\x18\x08\x01\x12\x14\x08\x04\x12\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x3A\x06\x73\x74\x61\x00\x74\x65\x42\x22\x0A\x20\x30\x67\xC7\x2C\x5E\x50\x1C\x31\xE3\xFE\xCA\x73\xF0\x47\xDC\x34\x1A\x95\x63\x99\xEC\x70\x5E\x0A\xEE\x9E\xFB\x17\xA1\x55\x35\x78\x12\x08\x08\x08\x10\x80\x94\xEB\xDC\x03";
+const UPDATE_PATH = "/safebrowsing/update";
 
 let gListManager = Cc["@mozilla.org/url-classifier/listmanager;1"]
                      .getService(Ci.nsIUrlListManager);
 
 let gCompleter = Cc["@mozilla.org/url-classifier/hashcompleter;1"]
                     .getService(Ci.nsIUrlClassifierHashCompleter);
 
-XPCOMUtils.defineLazyServiceGetter(this, 'gUrlUtil',
-                                   '@mozilla.org/url-classifier/utils;1',
-                                   'nsIUrlClassifierUtils');
+XPCOMUtils.defineLazyServiceGetter(this, "gUrlUtil",
+                                   "@mozilla.org/url-classifier/utils;1",
+                                   "nsIUrlClassifierUtils");
 
 // Handles request for TEST_TABLE_DATA_V4.
 let gHttpServV4 = null;
-let gExpectedGetHashQueryV4 = "";
 
-const NEW_CLIENT_STATE = 'sta\0te';
-const CHECKSUM = '\x30\x67\xc7\x2c\x5e\x50\x1c\x31\xe3\xfe\xca\x73\xf0\x47\xdc\x34\x1a\x95\x63\x99\xec\x70\x5e\x0a\xee\x9e\xfb\x17\xa1\x55\x35\x78';
+const NEW_CLIENT_STATE = "sta\0te";
+const CHECKSUM = "\x30\x67\xc7\x2c\x5e\x50\x1c\x31\xe3\xfe\xca\x73\xf0\x47\xdc\x34\x1a\x95\x63\x99\xec\x70\x5e\x0a\xee\x9e\xfb\x17\xa1\x55\x35\x78";
 
 prefBranch.setBoolPref("browser.safebrowsing.debug", true);
 
 // The "\xFF\xFF" is to generate a base64 string with "/".
 prefBranch.setCharPref("browser.safebrowsing.id", "Firefox\xFF\xFF");
 
 // Register tables.
 gListManager.registerTable(TEST_TABLE_DATA_V4.tableName,
@@ -53,113 +80,137 @@ add_test(function test_update_v4() {
 });
 
 add_test(function test_getHashRequestV4() {
   let request = gUrlUtil.makeFindFullHashRequestV4([TEST_TABLE_DATA_V4.tableName],
                                                    [btoa(NEW_CLIENT_STATE)],
                                                    [btoa("0123"), btoa("1234567"), btoa("1111")],
                                                    1,
                                                    3);
-  gExpectedGetHashQueryV4 = '&$req=' + request;
-
+  registerHandlerGethashV4("&$req=" + request);
   let completeFinishedCnt = 0;
 
   gCompleter.complete("0123", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
-    completion: function (hash, table, chunkId) {
+    completion(hash, table, chunkId) {
       equal(hash, "01234567890123456789012345678901");
       equal(table, TEST_TABLE_DATA_V4.tableName);
       equal(chunkId, 0);
       do_print("completion: " + hash + ", " + table + ", " + chunkId);
     },
 
-    completionFinished: function (status) {
+    completionFinished(status) {
       equal(status, Cr.NS_OK);
       completeFinishedCnt++;
       if (3 === completeFinishedCnt) {
         run_next_test();
       }
     },
   });
 
   gCompleter.complete("1234567", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
-    completion: function (hash, table, chunkId) {
+    completion(hash, table, chunkId) {
       equal(hash, "12345678901234567890123456789012");
       equal(table, TEST_TABLE_DATA_V4.tableName);
       equal(chunkId, 0);
       do_print("completion: " + hash + ", " + table + ", " + chunkId);
     },
 
-    completionFinished: function (status) {
+    completionFinished(status) {
       equal(status, Cr.NS_OK);
       completeFinishedCnt++;
       if (3 === completeFinishedCnt) {
         run_next_test();
       }
     },
   });
 
   gCompleter.complete("1111", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
-    completion: function (hash, table, chunkId) {
+    completion(hash, table, chunkId) {
       ok(false, "1111 is not the prefix of " + hash);
     },
 
-    completionFinished: function (status) {
+    completionFinished(status) {
       equal(status, Cr.NS_OK);
       completeFinishedCnt++;
       if (3 === completeFinishedCnt) {
         run_next_test();
       }
     },
   });
 });
 
-function run_test() {
-  gHttpServV4 = new HttpServer();
-  gHttpServV4.registerDirectory("/", do_get_cwd());
+add_test(function test_minWaitDuration() {
+  let failedComplete = function() {
+    gCompleter.complete("0123", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
+      completionFinished(status) {
+        equal(status, Cr.NS_ERROR_ABORT);
+      },
+    });
+  };
+
+  let successComplete = function() {
+    gCompleter.complete("1234567", TEST_TABLE_DATA_V4.gethashUrl, TEST_TABLE_DATA_V4.tableName, {
+      completion(hash, table, chunkId) {
+        equal(hash, "12345678901234567890123456789012");
+        equal(table, TEST_TABLE_DATA_V4.tableName);
+        equal(chunkId, 0);
+        do_print("completion: " + hash + ", " + table + ", " + chunkId);
+      },
+
+      completionFinished(status) {
+        equal(status, Cr.NS_OK);
+        run_next_test();
+      },
+    });
+  };
 
+  let request = gUrlUtil.makeFindFullHashRequestV4([TEST_TABLE_DATA_V4.tableName],
+                                                   [btoa(NEW_CLIENT_STATE)],
+                                                   [btoa("1234567")],
+                                                   1,
+                                                   1);
+  registerHandlerGethashV4("&$req=" + request);
+
+  // The last gethash response contained a min wait duration 12 secs 10 nano
+  // So subsequent requests can happen only after the min wait duration
+  do_timeout(1000, failedComplete);
+  do_timeout(2000, failedComplete);
+  do_timeout(4000, failedComplete);
+  do_timeout(13000, successComplete);
+});
+
+function registerHandlerGethashV4(aExpectedQuery) {
+  gHttpServV4.registerPathHandler(GETHASH_PATH, null);
+  // V4 gethash handler.
+  gHttpServV4.registerPathHandler(GETHASH_PATH, function(request, response) {
+    equal(request.queryString, aExpectedQuery);
+
+    response.setStatusLine(request.httpVersion, 200, "OK");
+    response.bodyOutputStream.write(GETHASH_RESPONSE_CONTENT,
+                                    GETHASH_RESPONSE_CONTENT.length);
+  });
+}
+
+function registerHandlerUpdateV4() {
   // Update handler. Will respond a valid state to be verified in the
   // gethash handler.
-  gHttpServV4.registerPathHandler("/safebrowsing/update", function(request, response) {
+  gHttpServV4.registerPathHandler(UPDATE_PATH, function(request, response) {
     response.setHeader("Content-Type",
                        "application/vnd.google.safebrowsing-update", false);
     response.setStatusLine(request.httpVersion, 200, "OK");
-
-    // The protobuf binary represention of response:
-    //
-    // [
-    //   {
-    //     'threat_type': 2, // SOCIAL_ENGINEERING_PUBLIC
-    //     'response_type': 2, // FULL_UPDATE
-    //     'new_client_state': 'sta\x00te', // NEW_CLIENT_STATE
-    //     'checksum': { "sha256": CHECKSUM }, // CHECKSUM
-    //     'additions': { 'compression_type': RAW,
-    //                    'prefix_size': 4,
-    //                    'raw_hashes': "00000001000000020000000300000004"}
-    //   }
-    // ]
-    //
-    let content = "\x0A\x4A\x08\x02\x20\x02\x2A\x18\x08\x01\x12\x14\x08\x04\x12\x10\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x02\x00\x00\x00\x03\x3A\x06\x73\x74\x61\x00\x74\x65\x42\x22\x0A\x20\x30\x67\xC7\x2C\x5E\x50\x1C\x31\xE3\xFE\xCA\x73\xF0\x47\xDC\x34\x1A\x95\x63\x99\xEC\x70\x5E\x0A\xEE\x9E\xFB\x17\xA1\x55\x35\x78\x12\x08\x08\x08\x10\x80\x94\xEB\xDC\x03";
-
-    response.bodyOutputStream.write(content, content.length);
+    response.bodyOutputStream.write(UPDATE_RESPONSE_CONTENT,
+                                    UPDATE_RESPONSE_CONTENT.length);
 
     waitUntilMetaDataSaved(NEW_CLIENT_STATE, CHECKSUM, () => {
       run_next_test();
     });
 
   });
-
-  // V4 gethash handler.
-  gHttpServV4.registerPathHandler("/safebrowsing/gethash-v4", function(request, response) {
-    equal(request.queryString, gExpectedGetHashQueryV4);
+}
 
-    // { nsCString("01234567890123456789012345678901"), SOCIAL_ENGINEERING_PUBLIC, { 8, 500 } },
-    // { nsCString("12345678901234567890123456789012"), SOCIAL_ENGINEERING_PUBLIC, { 7, 100} },
-    // { nsCString("23456789012345678901234567890123"), SOCIAL_ENGINEERING_PUBLIC, { 1, 20 } },
-    let content = "\x0A\x2D\x08\x02\x1A\x22\x0A\x20\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x2A\x05\x08\x08\x10\xF4\x03\x0A\x2C\x08\x02\x1A\x22\x0A\x20\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x2A\x04\x08\x07\x10\x64\x0A\x2C\x08\x02\x1A\x22\x0A\x20\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x34\x35\x36\x37\x38\x39\x30\x31\x32\x33\x2A\x04\x08\x01\x10\x14\x12\x04\x08\x0C\x10\x0A\x1A\x04\x08\x78\x10\x09";
+function run_test() {
+  gHttpServV4 = new HttpServer();
+  gHttpServV4.registerDirectory("/", do_get_cwd());
 
-    response.setStatusLine(request.httpVersion, 200, "OK");
-    response.bodyOutputStream.write(content, content.length);
-  });
-
+  registerHandlerUpdateV4();
   gHttpServV4.start(5555);
-
   run_next_test();
-}
\ No newline at end of file
+}