Bug 1272239 - Part 3: Testcase, test gethash. r=francois
authordimi <dlee@mozilla.com>
Thu, 21 Jul 2016 15:40:03 +0800
changeset 346424 59ea967f04c8fd9f29756eb1c7f6a6923ae1ec83
parent 346423 cfe1ca72b7d7ac25016da955cbc9fc259b971226
child 346425 75c5c25b63f33e670204bf07dd1f1066d0b92d6c
push id6389
push userraliiev@mozilla.com
push dateMon, 19 Sep 2016 13:38:22 +0000
treeherdermozilla-beta@01d67bfe6c81 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersfrancois
bugs1272239
milestone50.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1272239 - Part 3: Testcase, test gethash. r=francois MozReview-Commit-ID: 3IkrdJgZNP1
toolkit/components/url-classifier/tests/mochitest/bad.css
toolkit/components/url-classifier/tests/mochitest/classifierFrame.html
toolkit/components/url-classifier/tests/mochitest/classifierHelper.js
toolkit/components/url-classifier/tests/mochitest/gethash.sjs
toolkit/components/url-classifier/tests/mochitest/gethashFrame.html
toolkit/components/url-classifier/tests/mochitest/import.css
toolkit/components/url-classifier/tests/mochitest/mochitest.ini
toolkit/components/url-classifier/tests/mochitest/test_gethash.html
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/mochitest/bad.css
@@ -0,0 +1,1 @@
+#styleBad { visibility: hidden; }
--- a/toolkit/components/url-classifier/tests/mochitest/classifierFrame.html
+++ b/toolkit/components/url-classifier/tests/mochitest/classifierFrame.html
@@ -10,16 +10,24 @@ function checkLoads() {
   // Make sure the javascript did not load.
   window.parent.is(scriptItem, "untouched", "Should not load bad javascript");
 
   // Make sure the css did not load.
   var elt = document.getElementById("styleCheck");
   var style = document.defaultView.getComputedStyle(elt, "");
   window.parent.isnot(style.visibility, "hidden", "Should not load bad css");
 
+  elt = document.getElementById("styleBad");
+  style = document.defaultView.getComputedStyle(elt, "");
+  window.parent.isnot(style.visibility, "hidden", "Should not load bad css");
+
+  elt = document.getElementById("styleImport");
+  style = document.defaultView.getComputedStyle(elt, "");
+  window.parent.isnot(style.visibility, "visible", "Should import clean css");
+
   // Call parent.loadTestFrame again to test classification metadata in HTTP
   // cache entries.
   if (window.parent.firstLoad) {
     window.parent.info("Reloading from cache...");
     window.parent.firstLoad = false;
     window.parent.loadTestFrame();
     return;
   }
@@ -31,18 +39,19 @@ function checkLoads() {
 </script>
 
 <!-- Try loading from a malware javascript URI -->
 <script type="text/javascript" src="http://malware.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"></script>
 
 <!-- Try loading from an uwanted software css URI -->
 <link rel="stylesheet" type="text/css" href="http://unwanted.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.css"></link>
 
-<!-- XXX How is this part of the test supposed to work (= be checked)? -->
 <!-- Try loading a marked-as-malware css through an @import from a clean URI -->
 <link rel="stylesheet" type="text/css" href="import.css"></link>
 </head>
 
 <body onload="checkLoads()">
 The following should not be hidden:
 <div id="styleCheck">STYLE TEST</div>
+<div id="styleBad">STYLE BAD</div>
+<div id="styleImport">STYLE IMPORT</div>
 </body>
 </html>
--- a/toolkit/components/url-classifier/tests/mochitest/classifierHelper.js
+++ b/toolkit/components/url-classifier/tests/mochitest/classifierHelper.js
@@ -4,33 +4,61 @@ if (typeof(classifierHelper) == "undefin
 
 const CLASSIFIER_COMMON_URL = SimpleTest.getTestFileURL("classifierCommon.js");
 var gScript = SpecialPowers.loadChromeScript(CLASSIFIER_COMMON_URL);
 
 const ADD_CHUNKNUM = 524;
 const SUB_CHUNKNUM = 523;
 const HASHLEN = 32;
 
+const PREFS = {
+  PROVIDER_LISTS : "browser.safebrowsing.provider.mozilla.lists",
+  DISALLOW_COMPLETIONS : "urlclassifier.disallow_completions",
+  PROVIDER_GETHASHURL : "browser.safebrowsing.provider.mozilla.gethashURL"
+};
+
 // addUrlToDB & removeUrlFromDB are asynchronous, queue the task to ensure
 // the callback follow correct order.
 classifierHelper._updates = [];
 
 // Keep urls added to database, those urls should be automatically
 // removed after test complete.
 classifierHelper._updatesToCleanup = [];
 
+// This function is used to allow completion for specific "list",
+// some lists like "test-malware-simple" is default disabled to ask for complete.
+// "list" is the db we would like to allow it
+// "url" is the completion server
+classifierHelper.allowCompletion = function(lists, url) {
+  for (var list of lists) {
+    // Add test db to provider
+    var pref = SpecialPowers.getCharPref(PREFS.PROVIDER_LISTS);
+    pref += "," + list;
+    SpecialPowers.setCharPref(PREFS.PROVIDER_LISTS, pref);
+
+    // Rename test db so we will not disallow it from completions
+    pref = SpecialPowers.getCharPref(PREFS.DISALLOW_COMPLETIONS);
+    pref = pref.replace(list, list + "-backup");
+    SpecialPowers.setCharPref(PREFS.DISALLOW_COMPLETIONS, pref);
+  }
+
+  // Set get hash url
+  SpecialPowers.setCharPref(PREFS.PROVIDER_GETHASHURL, url);
+}
+
 // Pass { url: ..., db: ... } to add url to database,
 // onsuccess/onerror will be called when update complete.
 classifierHelper.addUrlToDB = function(updateData) {
   return new Promise(function(resolve, reject) {
     var testUpdate = "";
     for (var update of updateData) {
       var LISTNAME = update.db;
       var CHUNKDATA = update.url;
       var CHUNKLEN = CHUNKDATA.length;
+      var HASHLEN = update.len ? update.len : 32;
 
       classifierHelper._updatesToCleanup.push(update);
       testUpdate +=
         "n:1000\n" +
         "i:" + LISTNAME + "\n" +
         "ad:1\n" +
         "a:" + ADD_CHUNKNUM + ":" + HASHLEN + ":" + CHUNKLEN + "\n" +
         CHUNKDATA;
@@ -44,16 +72,17 @@ classifierHelper.addUrlToDB = function(u
 // onsuccess/onerror will be called when update complete.
 classifierHelper.removeUrlFromDB = function(updateData) {
   return new Promise(function(resolve, reject) {
     var testUpdate = "";
     for (var update of updateData) {
       var LISTNAME = update.db;
       var CHUNKDATA = ADD_CHUNKNUM + ":" + update.url;
       var CHUNKLEN = CHUNKDATA.length;
+      var HASHLEN = update.len ? update.len : 32;
 
       testUpdate +=
         "n:1000\n" +
         "i:" + LISTNAME + "\n" +
         "s:" + SUB_CHUNKNUM + ":" + HASHLEN + ":" + CHUNKLEN + "\n" +
         CHUNKDATA;
     }
 
@@ -122,16 +151,21 @@ classifierHelper._setup = function() {
   gScript.addMessageListener("updateSuccess", classifierHelper._updateSuccess);
   gScript.addMessageListener("updateError", classifierHelper._updateError);
 
   // cleanup will be called at end of each testcase to remove all the urls added to database.
   SimpleTest.registerCleanupFunction(classifierHelper._cleanup);
 };
 
 classifierHelper._cleanup = function() {
+  // clean all the preferences may touch by helper
+  for (var pref in PREFS) {
+    SpecialPowers.clearUserPref(pref);
+  }
+
   if (!classifierHelper._updatesToCleanup) {
     return Promise.resolve();
   }
 
   return classifierHelper.resetDB();
 };
 
 classifierHelper._setup();
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/mochitest/gethash.sjs
@@ -0,0 +1,119 @@
+const CC = Components.Constructor;
+const BinaryInputStream = CC("@mozilla.org/binaryinputstream;1",
+                             "nsIBinaryInputStream",
+                             "setInputStream");
+
+function handleRequest(request, response)
+{
+  var query = {};
+  request.queryString.split('&').forEach(function (val) {
+    var idx = val.indexOf('=');
+    query[val.slice(0, idx)] = unescape(val.slice(idx + 1));
+  });
+
+  // Store fullhash in the server side.
+  if ("list" in query && "fullhash" in query) {
+    // In the server side we will store:
+    // 1. All the full hashes for a given list
+    // 2. All the lists we have right now
+    // data is separate by '\n'
+    let list = query["list"];
+    let hashes = getState(list);
+
+    let hash = base64ToString(query["fullhash"]);
+    hashes += hash + "\n";
+    setState(list, hashes);
+
+    let lists = getState("lists");
+    if (lists.indexOf(list) == -1) {
+      lists += list + "\n";
+      setState("lists", lists);
+    }
+
+    return;
+  }
+
+  var body = new BinaryInputStream(request.bodyInputStream);
+  var avail;
+  var bytes = [];
+
+  while ((avail = body.available()) > 0) {
+    Array.prototype.push.apply(bytes, body.readByteArray(avail));
+  }
+
+  var responseBody = parseV2Request(bytes);
+
+  response.setHeader("Content-Type", "text/plain", false);
+  response.write(responseBody);
+
+}
+
+function parseV2Request(bytes) {
+  var request = String.fromCharCode.apply(this, bytes);
+  var [HEADER, PREFIXES] = request.split("\n");
+  var [PREFIXSIZE, LENGTH] = HEADER.split(":").map(val => {
+      return parseInt(val);
+    });
+
+  var ret = "";
+  for(var start = 0; start < LENGTH; start += PREFIXSIZE) {
+    getState("lists").split("\n").forEach(function(list) {
+      var completions = getState(list).split("\n");
+
+      for (var completion of completions) {
+        if (completion.indexOf(PREFIXES.substr(start, PREFIXSIZE)) == 0) {
+          ret += list + ":" + "1" + ":" + "32" + "\n";
+          ret += completion;
+        }
+      }
+    });
+  }
+
+  return ret;
+}
+
+/* Convert Base64 data to a string */
+const toBinaryTable = [
+    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,-1,
+    -1,-1,-1,-1, -1,-1,-1,-1, -1,-1,-1,62, -1,-1,-1,63,
+    52,53,54,55, 56,57,58,59, 60,61,-1,-1, -1, 0,-1,-1,
+    -1, 0, 1, 2,  3, 4, 5, 6,  7, 8, 9,10, 11,12,13,14,
+    15,16,17,18, 19,20,21,22, 23,24,25,-1, -1,-1,-1,-1,
+    -1,26,27,28, 29,30,31,32, 33,34,35,36, 37,38,39,40,
+    41,42,43,44, 45,46,47,48, 49,50,51,-1, -1,-1,-1,-1
+];
+const base64Pad = '=';
+
+function base64ToString(data) {
+    var result = '';
+    var leftbits = 0; // number of bits decoded, but yet to be appended
+    var leftdata = 0; // bits decoded, but yet to be appended
+
+    // Convert one by one.
+    for (var i = 0; i < data.length; i++) {
+        var c = toBinaryTable[data.charCodeAt(i) & 0x7f];
+        var padding = (data[i] == base64Pad);
+        // Skip illegal characters and whitespace
+        if (c == -1) continue;
+
+        // Collect data into leftdata, update bitcount
+        leftdata = (leftdata << 6) | c;
+        leftbits += 6;
+
+        // If we have 8 or more bits, append 8 bits to the result
+        if (leftbits >= 8) {
+            leftbits -= 8;
+            // Append if not padding.
+            if (!padding)
+                result += String.fromCharCode((leftdata >> leftbits) & 0xff);
+            leftdata &= (1 << leftbits) - 1;
+        }
+    }
+
+    // If there are any bits left, the base64 string was corrupted
+    if (leftbits)
+        throw Components.Exception('Corrupted base64 string');
+
+    return result;
+}
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/mochitest/gethashFrame.html
@@ -0,0 +1,62 @@
+<html>
+<head>
+<title></title>
+
+<script type="text/javascript">
+
+var scriptItem = "untouched";
+
+function checkLoads() {
+
+  var title = document.getElementById("title");
+  title.innerHTML = window.parent.shouldLoad ?
+                    "The following should be hidden:" :
+                    "The following should not be hidden:"
+
+  if (window.parent.shouldLoad) {
+    window.parent.is(scriptItem, "loaded malware javascript!", "Should load bad javascript");
+  } else {
+    window.parent.is(scriptItem, "untouched", "Should not load bad javascript");
+  }
+
+  var elt = document.getElementById("styleImport");
+  var style = document.defaultView.getComputedStyle(elt, "");
+  window.parent.isnot(style.visibility, "visible", "Should load clean css");
+
+  // Make sure the css did not load.
+  elt = document.getElementById("styleCheck");
+  style = document.defaultView.getComputedStyle(elt, "");
+  if (window.parent.shouldLoad) {
+    window.parent.isnot(style.visibility, "visible", "Should load bad css");
+  } else {
+    window.parent.isnot(style.visibility, "hidden", "Should not load bad css");
+  }
+
+  elt = document.getElementById("styleBad");
+  style = document.defaultView.getComputedStyle(elt, "");
+  if (window.parent.shouldLoad) {
+    window.parent.isnot(style.visibility, "visible", "Should import bad css");
+  } else {
+    window.parent.isnot(style.visibility, "hidden", "Should not import bad css");
+  }
+}
+
+</script>
+
+<!-- Try loading from a malware javascript URI -->
+<script type="text/javascript" src="http://malware.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.js"></script>
+
+<!-- Try loading from an uwanted software css URI -->
+<link rel="stylesheet" type="text/css" href="http://unwanted.example.com/tests/toolkit/components/url-classifier/tests/mochitest/evil.css"></link>
+
+<!-- Try loading a marked-as-malware css through an @import from a clean URI -->
+<link rel="stylesheet" type="text/css" href="import.css"></link>
+</head>
+
+<body onload="checkLoads()">
+<div id="title"></div>
+<div id="styleCheck">STYLE EVIL</div>
+<div id="styleBad">STYLE BAD</div>
+<div id="styleImport">STYLE IMPORT</div>
+</body>
+</html>
--- a/toolkit/components/url-classifier/tests/mochitest/import.css
+++ b/toolkit/components/url-classifier/tests/mochitest/import.css
@@ -1,3 +1,3 @@
-/* malware.example.com is in the malware database.
-   classifierBad.css does not actually exist. */
-@import url("http://malware.example.com/tests/docshell/test/classifierBad.css");
+/* malware.example.com is in the malware database. */
+@import url("http://malware.example.com/tests/toolkit/components/url-classifier/tests/mochitest/bad.css");
+#styleImport { visibility: hidden; }
--- a/toolkit/components/url-classifier/tests/mochitest/mochitest.ini
+++ b/toolkit/components/url-classifier/tests/mochitest/mochitest.ini
@@ -18,14 +18,18 @@ support-files =
   vp9.webm
   whitelistFrame.html
   workerFrame.html
   ping.sjs
   basic.vtt
   dnt.html
   dnt.sjs
   update.sjs
+  bad.css
+  gethash.sjs
+  gethashFrame.html
 
 [test_classifier.html]
 skip-if = (os == 'linux' && debug) #Bug 1199778
 [test_classifier_worker.html]
 [test_classify_ping.html]
 [test_classify_track.html]
+[test_gethash.html]
new file mode 100644
--- /dev/null
+++ b/toolkit/components/url-classifier/tests/mochitest/test_gethash.html
@@ -0,0 +1,150 @@
+<!DOCTYPE HTML>
+<html>
+<head>
+  <title>Bug 1272239 - Test gethash.</title>
+  <script type="text/javascript" src="/tests/SimpleTest/SimpleTest.js"></script>
+  <script type="text/javascript" src="classifierHelper.js"></script>
+  <link rel="stylesheet" type="text/css" href="/tests/SimpleTest/test.css" />
+</head>
+
+<body>
+<p id="display"></p>
+<div id="content" style="display: none">
+</div>
+<pre id="test">
+<iframe id="testFrame1" onload=""></iframe>
+<iframe id="testFrame2" onload=""></iframe>
+
+<script class="testbody" type="text/javascript">
+
+const MALWARE_LIST = "test-malware-simple";
+const MALWARE_HOST = "malware.example.com/";
+
+const UNWANTED_LIST = "test-unwanted-simple";
+const UNWANTED_HOST = "unwanted.example.com/";
+
+const GETHASH_URL = "http://mochi.test:8888/tests/toolkit/components/url-classifier/tests/mochitest/gethash.sjs";
+const NOTEXIST_URL = "http://mochi.test:8888/tests/toolkit/components/url-classifier/tests/mochitest/nonexistserver.sjs";
+
+var shouldLoad = false;
+
+// In this testcase we store prefixes to localdb and send the fullhash to gethash server.
+// When access the test page gecko should trigger gethash request to server and
+// get the completion response.
+function loadTestFrame(id) {
+  return new Promise(function(resolve, reject) {
+
+    var iframe = document.getElementById(id);
+    iframe.setAttribute("src", "gethashFrame.html");
+
+    iframe.onload = function() {
+      resolve();
+    };
+  });
+}
+
+// add 4-bytes prefixes to local database, so when we access the url,
+// it will trigger gethash request.
+function addPrefixToDB(list, url) {
+  var testData = [{ db: list, url: url, len: 4 }];
+
+  return classifierHelper.addUrlToDB(testData)
+    .catch(function(err) {
+      ok(false, "Couldn't update classifier. Error code: " + err);
+      // Abort test.
+      SimpleTest.finish();
+    });
+}
+
+// calculate the fullhash and send it to gethash server
+function addCompletionToServer(list, url) {
+  return new Promise(function(resolve, reject) {
+    var listParam = "list=" + list;
+    var fullhashParam = "fullhash=" + hash(url);
+
+    var xhr = new XMLHttpRequest;
+    xhr.open("PUT", GETHASH_URL + "?" +
+             listParam + "&" +
+             fullhashParam, true);
+    xhr.setRequestHeader("Content-Type", "text/plain");
+    xhr.onreadystatechange = function() {
+      if (this.readyState == this.DONE) {
+        resolve();
+      }
+    };
+    xhr.send();
+  });
+}
+
+function hash(str) {
+  function bytesFromString(str) {
+    var converter =
+      SpecialPowers.Cc["@mozilla.org/intl/scriptableunicodeconverter"]
+                       .createInstance(SpecialPowers.Ci.nsIScriptableUnicodeConverter);
+    converter.charset = "UTF-8";
+    return converter.convertToByteArray(str);
+  }
+
+  var hasher = SpecialPowers.Cc["@mozilla.org/security/hash;1"]
+                               .createInstance(SpecialPowers.Ci.nsICryptoHash);
+
+  var data = bytesFromString(str);
+  hasher.init(hasher.SHA256);
+  hasher.update(data, data.length);
+
+  return hasher.finish(true);
+}
+
+function setup404() {
+  shouldLoad = true;
+
+  classifierHelper.allowCompletion([MALWARE_LIST, UNWANTED_LIST], NOTEXIST_URL);
+
+  return Promise.all([
+    addPrefixToDB(MALWARE_LIST, MALWARE_HOST),
+    addPrefixToDB(UNWANTED_LIST, UNWANTED_HOST)
+  ]);
+}
+
+function setupCompletion() {
+  classifierHelper.allowCompletion([MALWARE_LIST, UNWANTED_LIST], GETHASH_URL);
+
+  return Promise.all([
+    addPrefixToDB(MALWARE_LIST, MALWARE_HOST),
+    addPrefixToDB(UNWANTED_LIST, UNWANTED_HOST),
+    addCompletionToServer(MALWARE_LIST, MALWARE_HOST),
+    addCompletionToServer(UNWANTED_LIST, UNWANTED_HOST),
+  ]);
+}
+
+// manually reset DB to make sure next test won't be affected by cache.
+function reset() {
+  return classifierHelper.resetDB;
+}
+
+function runTest() {
+  Promise.resolve()
+    // This test resources get blocked when gethash returns successfully
+    .then(setupCompletion)
+    .then(() => loadTestFrame("testFrame1"))
+    .then(reset)
+    // This test resources are not blocked when gethash returns an error
+    .then(setup404)
+    .then(() => loadTestFrame("testFrame2"))
+    .then(function() {
+      SimpleTest.finish();
+    }).catch(function(e) {
+      ok(false, "Some test failed with error " + e);
+      SimpleTest.finish();
+    });
+}
+
+SimpleTest.waitForExplicitFinish();
+SpecialPowers.pushPrefEnv({"set": [
+  ["browser.safebrowsing.malware.enabled", true]
+]}, runTest);
+
+</script>
+</pre>
+</body>
+</html>