Bug 1527280 - Add deterministic js and injection script to raptor mitmproxy r=davehunt
authorFlorin Strugariu <fstrugariu@mozilla.com>
Fri, 22 Feb 2019 19:33:13 +0000
changeset 460641 7bf935a896522a135e55205236bd2ea7f99aa31f
parent 460640 77eb18940eb116616c0a8b5e252823879a4b3655
child 460642 244cd329c3bbb03ca23cc24ea1468225a306bc9c
push id35596
push userrmaries@mozilla.com
push dateSat, 23 Feb 2019 04:13:22 +0000
treeherdermozilla-central@fdd04819e350 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersdavehunt
bugs1527280
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1527280 - Add deterministic js and injection script to raptor mitmproxy r=davehunt Differential Revision: https://phabricator.services.mozilla.com/D19485
.eslintignore
testing/raptor/raptor/playback/scripts/catapult/LICENSE
testing/raptor/raptor/playback/scripts/catapult/deterministic.js
testing/raptor/raptor/playback/scripts/inject-deterministic.py
--- a/.eslintignore
+++ b/.eslintignore
@@ -324,16 +324,18 @@ testing/talos/talos/scripts/jszip.min.js
 testing/talos/talos/startup_test/sessionrestore/profile/sessionstore.js
 testing/talos/talos/startup_test/sessionrestore/profile-manywindows/sessionstore.js
 testing/talos/talos/tests/devtools/addon/content/pages/**
 testing/talos/talos/tests/dromaeo/**
 testing/talos/talos/tests/v8_7/**
 testing/talos/talos/tests/kraken/**
 # Runing Talos may extract data here, see bug 1435677.
 testing/talos/talos/tests/tp5n/**
+# Raptor third party
+testing/raptor/raptor/playback/scripts/catapult/**
 
 testing/web-platform/**
 testing/xpcshell/moz-http2/**
 testing/xpcshell/node-http2/**
 
 # Third party services
 services/common/kinto-http-client.js
 services/common/kinto-offline-client.js
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/scripts/catapult/LICENSE
@@ -0,0 +1,27 @@
+Copyright 2015 The Chromium Authors. All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of catapult nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/scripts/catapult/deterministic.js
@@ -0,0 +1,71 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+
+'use strict';
+
+(function () {
+  var random_count = 0;
+  var random_count_threshold = 25;
+  var random_seed = 0.462;
+  Math.random = function() {
+    random_count++;
+    if (random_count > random_count_threshold){
+     random_seed += 0.1;
+     random_count = 1;
+    }
+    return (random_seed % 1);
+  };
+  if (typeof(crypto) == 'object' &&
+      typeof(crypto.getRandomValues) == 'function') {
+    crypto.getRandomValues = function(arr) {
+      var scale = Math.pow(256, arr.BYTES_PER_ELEMENT);
+      for (var i = 0; i < arr.length; i++) {
+        arr[i] = Math.floor(Math.random() * scale);
+      }
+      return arr;
+    };
+  }
+})();
+(function () {
+  var date_count = 0;
+  var date_count_threshold = 25;
+  var orig_date = Date;
+  // Time since epoch in milliseconds. This is replaced by script injector with
+  // the date when the recording is done.
+  var time_seed = REPLACE_LOAD_TIMESTAMP;
+  Date = function() {
+    if (this instanceof Date) {
+      date_count++;
+      if (date_count > date_count_threshold){
+        time_seed += 50;
+        date_count = 1;
+      }
+      switch (arguments.length) {
+      case 0: return new orig_date(time_seed);
+      case 1: return new orig_date(arguments[0]);
+      default: return new orig_date(arguments[0], arguments[1],
+         arguments.length >= 3 ? arguments[2] : 1,
+         arguments.length >= 4 ? arguments[3] : 0,
+         arguments.length >= 5 ? arguments[4] : 0,
+         arguments.length >= 6 ? arguments[5] : 0,
+         arguments.length >= 7 ? arguments[6] : 0);
+      }
+    }
+    return new Date().toString();
+  };
+  Date.__proto__ = orig_date;
+  Date.prototype = orig_date.prototype;
+  Date.prototype.constructor = Date;
+  orig_date.now = function() {
+    return new Date().getTime();
+  };
+  orig_date.prototype.getTimezoneOffset = function() {
+    var dst2010Start = 1268560800000;
+    var dst2010End = 1289120400000;
+    if (this.getTime() >= dst2010Start && this.getTime() < dst2010End)
+      return 420;
+    return 480;
+  };
+})();
new file mode 100644
--- /dev/null
+++ b/testing/raptor/raptor/playback/scripts/inject-deterministic.py
@@ -0,0 +1,168 @@
+import base64
+import hashlib
+import re
+import time
+
+from mitmproxy import ctx
+
+
+class AddDeterministic():
+
+    def get_csp_directives(self, headers):
+        csp = headers.get("Content-Security-Policy", "")
+        return [d.strip() for d in csp.split(";")]
+
+    def get_csp_script_sources(self, headers):
+        sources = []
+        for directive in self.get_csp_directives(headers):
+            if directive.startswith("script-src "):
+                sources = directive.split()[1:]
+        return sources
+
+    def get_nonce_from_headers(self, headers):
+        """
+        get_nonce_from_headers returns the nonce token from a
+        Content-Security-Policy (CSP) header's script source directive.
+
+        Note:
+        For more background information on CSP and nonce, please refer to
+        https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/
+        Content-Security-Policy/script-src
+        https://developers.google.com/web/fundamentals/security/csp/
+        """
+
+        for source in self.get_csp_script_sources(headers) or []:
+            if source.startswith("'nonce-"):
+                return source.partition("'nonce-")[-1][:-1]
+
+    def get_script_with_nonce(self, script, nonce=None):
+        """
+        Given a nonce, get_script_with_nonce returns the injected script text with the nonce.
+
+        If nonce None, get_script_with_nonce returns the script block
+        without attaching a nonce attribute.
+
+        Note:
+        Some responses may specify a nonce inside their Content-Security-Policy,
+        script-src directive.
+        The script injector needs to set the injected script's nonce attribute to
+        open execute permission for the injected script.
+        """
+
+        if nonce:
+            return '<script nonce="{}">{}</script>'.format(nonce, script)
+        return '<script>{}</script>'.format(script)
+
+    def update_csp_script_src(self, headers, sha256):
+        """
+        Update the CSP script directives with appropriate information
+
+        Without this permissions a page with a
+        restrictive CSP will not execute injected scripts.
+
+        Note:
+        For more background information on CSP, please refer to
+        https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/
+        Content-Security-Policy/script-src
+        https://developers.google.com/web/fundamentals/security/csp/
+        """
+
+        sources = self.get_csp_script_sources(headers)
+        add_unsafe = True
+
+        for token in sources:
+            if token == "'unsafe-inline'":
+                add_unsafe = False
+                ctx.log.info("Contains unsafe-inline")
+            elif token.startswith("'sha"):
+                sources.append("'sha256-{}'".format(sha256))
+                add_unsafe = False
+                ctx.log.info("Add sha hash directive")
+                break
+
+        if add_unsafe:
+            ctx.log.info("Add unsafe")
+            sources.append("'unsafe-inline'")
+
+        return "script-src {}".format(" ".join(sources))
+
+    def get_new_csp_header(self, headers, updated_csp_script):
+        """
+        get_new_csp_header generates a new header object containing
+        the updated elements from new_csp_script_directives
+        """
+
+        if updated_csp_script:
+            directives = self.get_csp_directives(headers)
+            for index, directive in enumerate(directives):
+                if directive.startswith("script-src "):
+                    directives[index] = updated_csp_script
+
+            ctx.log.info("Original Header %s \n" % headers["Content-Security-Policy"])
+            headers["Content-Security-Policy"] = "; ".join(directives)
+            ctx.log.info("Updated  Header %s \n" % headers["Content-Security-Policy"])
+
+        return headers
+
+    def response(self, flow):
+
+        millis = int(round(time.time() * 1000))
+
+        if "content-type" in flow.response.headers:
+            if 'text/html' in flow.response.headers["content-type"]:
+                ctx.log.info("Working on {}".format(flow.response.headers["content-type"]))
+
+                flow.response.decode()
+                html = flow.response.text
+
+                with open("scripts/catapult/deterministic.js", "r") as jsfile:
+                    js = jsfile.read().replace("REPLACE_LOAD_TIMESTAMP", str(millis))
+
+                    if js not in html:
+                        script_index = re.search('(?i).*?<head.*?>', html)
+                        if script_index is None:
+                            script_index = re.search('(?i).*?<html.*?>', html)
+                        if script_index is None:
+                            script_index = re.search('(?i).*?<!doctype html>', html)
+                        if script_index is None:
+                            ctx.log.info("No start tags found in request {}. Skip injecting".
+                                         format(flow.request.url))
+                            return
+                        script_index = script_index.end()
+
+                        nonce = None
+
+                        if flow.response.headers.get("Content-Security-Policy", False):
+                            nonce = self.get_nonce_from_headers(flow.response.headers)
+                            ctx.log.info("nonce : %s" % nonce)
+
+                            if self.get_csp_script_sources(flow.response.headers) and not nonce:
+                                # generate sha256 for the script
+                                hash_object = hashlib.sha256(js.encode('utf-8'))
+                                script_sha256 = base64.b64encode(hash_object.digest()). \
+                                    decode("utf-8")
+
+                                # generate the new response headers
+                                updated_script_sources = self.update_csp_script_src(
+                                    flow.response.headers,
+                                    script_sha256)
+                                flow.response.headers = self.get_new_csp_header(
+                                    flow.response.headers,
+                                    updated_script_sources)
+
+                        # generate new html file
+                        new_html = html[:script_index] + \
+                            self.get_script_with_nonce(js, nonce) + \
+                            html[script_index:]
+                        flow.response.text = new_html
+
+                        ctx.log.info("In request {} injected deterministic JS".
+                                     format(flow.request.url))
+                    else:
+                        ctx.log.info("Script already injected in request {}".
+                                     format(flow.request.url))
+
+
+def start():
+    ctx.log.info("Load Deterministic JS")
+    return AddDeterministic()