Backed out changeset 121e4d470c11 (bug 1391703) for breaking periodic HSTS/HPKP updates.
authorRyan VanderMeulen <ryanvm@gmail.com>
Fri, 25 Aug 2017 10:16:27 -0400
changeset 376788 56188620cce00b19700fbb8efaafea65e6ca8c61
parent 376787 b6b8e616de32af50c9a174006b3a7ed914130aa5
child 376853 03d7b6dd65b93afaa6981269f69e9f7cd34224bc
push id32391
push userryanvm@gmail.com
push dateFri, 25 Aug 2017 14:16:33 +0000
treeherdermozilla-central@56188620cce0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1391703
milestone57.0a1
backs out121e4d470c111f07d1dd45959bd4777de529f473
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Backed out changeset 121e4d470c11 (bug 1391703) for breaking periodic HSTS/HPKP updates.
security/manager/moz.build
security/manager/tools/.eslintrc.js
security/manager/tools/PSMToolUtils.jsm
security/manager/tools/dumpGoogleRoots.js
security/manager/tools/genHPKPStaticPins.js
security/manager/tools/genRootCAHashes.js
security/manager/tools/getHSTSPreloadList.js
security/manager/tools/moz.build
security/manager/tools/tests/.eslintrc.js
security/manager/tools/tests/test_psmtoolutils.js
security/manager/tools/tests/xpcshell.ini
--- a/security/manager/moz.build
+++ b/security/manager/moz.build
@@ -2,16 +2,12 @@
 # vim: set filetype=python:
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 with Files("**"):
     BUG_COMPONENT = ("Core", "Security: PSM")
 
-DIRS += [
-    'locales',
-    'ssl',
-    'tools',
-]
+DIRS += ['ssl', 'locales']
 
 if CONFIG['MOZ_XUL'] and CONFIG['MOZ_BUILD_APP'] != 'mobile/android':
     DIRS += ['pki']
--- a/security/manager/tools/.eslintrc.js
+++ b/security/manager/tools/.eslintrc.js
@@ -1,10 +1,9 @@
 "use strict";
 
 module.exports = {
   "globals": {
     // JS files in this folder are commonly xpcshell scripts where |arguments|
-    // and |__LOCATION__| are defined in the global scope.
-    "__LOCATION__": false,
+    // is defined in the global scope.
     "arguments": false
   }
 };
deleted file mode 100644
--- a/security/manager/tools/PSMToolUtils.jsm
+++ /dev/null
@@ -1,59 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-
-// Common code for PSM scripts in security/manager/tools/.
-
-"use strict";
-
-const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
-
-/**
- * Synchronously downloads a file.
- *
- * @param {String} url
- *        The URL to download the file from.
- * @param {Boolean} decodeContentsAsBase64
- *        Whether the downloaded contents should be Base64 decoded before being
- *        returned.
- * @returns {String}
- *          The downloaded contents.
- */
-function downloadFile(url, decodeContentsAsBase64) {
-  let req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
-              .createInstance(Ci.nsIXMLHttpRequest);
-  req.open("GET", url, false); // doing the request synchronously
-  try {
-    req.send();
-  } catch (e) {
-    throw new Error(`ERROR: problem downloading '${url}': ${e}`);
-  }
-
-  if (req.status != 200) {
-    throw new Error(`ERROR: problem downloading '${url}': status ${req.status}`);
-  }
-
-  if (!decodeContentsAsBase64) {
-    return req.responseText;
-  }
-
-  try {
-    return atob(req.responseText);
-  } catch (e) {
-    throw new Error(`ERROR: could not decode data as base64 from '${url}': ` +
-                    e);
-  }
-}
-
-/**
- * Removes //-style block (but not trailing) comments from a string.
- * @param {String} input
- *        Potentially multi-line input.
- * @returns {String}
- */
-function stripComments(input) {
-  return input.replace(/^(\s*)?\/\/[^\n]*\n/mg, "");
-}
-
-this.PSMToolUtils = { downloadFile, stripComments };
-this.EXPORTED_SYMBOLS = ["PSMToolUtils"];
--- a/security/manager/tools/dumpGoogleRoots.js
+++ b/security/manager/tools/dumpGoogleRoots.js
@@ -8,23 +8,35 @@
 //
 // How to run this file:
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell dumpGoogleRoots.js'
 // 4. [paste the output into the appropriate section in
 //     security/manager/tools/PreloadedHPKPins.json]
 
-const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
-
-const { PSMToolUtils } =
-  Cu.import(`file:///${__LOCATION__.parent.path}/PSMToolUtils.jsm`, {});
+var Cc = Components.classes;
+var Ci = Components.interfaces;
 
 function downloadRoots() {
-  let pem = PSMToolUtils.downloadFile("https://pki.google.com/roots.pem", false);
+  let req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+              .createInstance(Ci.nsIXMLHttpRequest);
+  req.open("GET", "https://pki.google.com/roots.pem", false);
+  try {
+    req.send();
+  } catch (e) {
+    throw new Error("ERROR: problem downloading Google Root PEMs: " + e);
+  }
+
+  if (req.status != 200) {
+    throw new Error("ERROR: problem downloading Google Root PEMs. Status: " +
+                    req.status);
+  }
+
+  let pem = req.responseText;
   let roots = [];
   let currentPEM = "";
   let readingRoot = false;
   let certDB = Cc["@mozilla.org/security/x509certdb;1"]
                  .getService(Ci.nsIX509CertDB);
   for (let line of pem.split(/[\r\n]/)) {
     if (line == "-----END CERTIFICATE-----") {
       if (currentPEM) {
--- a/security/manager/tools/genHPKPStaticPins.js
+++ b/security/manager/tools/genHPKPStaticPins.js
@@ -16,21 +16,19 @@ if (arguments.length != 3) {
   throw new Error("Usage: genHPKPStaticPins.js " +
                   "<absolute path to PreloadedHPKPins.json> " +
                   "<an unused argument - see bug 1205406> " +
                   "<absolute path to StaticHPKPins.h>");
 }
 
 const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
 
-const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
-const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
-const { PSMToolUtils } =
-  Cu.import(`file:///${__LOCATION__.parent.path}/PSMToolUtils.jsm`, {});
-const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
+var { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
+var { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
+var { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
 
 var gCertDB = Cc["@mozilla.org/security/x509certdb;1"]
                 .getService(Ci.nsIX509CertDB);
 
 const SHA256_PREFIX = "sha256/";
 const GOOGLE_PIN_PREFIX = "GOOGLE_PIN_";
 
 // Pins expire in 14 weeks (6 weeks on Beta + 8 weeks on stable)
@@ -84,19 +82,57 @@ function readFileToString(filename) {
   file.initWithPath(filename);
   let stream = Cc["@mozilla.org/network/file-input-stream;1"]
                  .createInstance(Ci.nsIFileInputStream);
   stream.init(file, -1, 0, 0);
   let buf = NetUtil.readInputStreamToString(stream, stream.available());
   return buf;
 }
 
+function stripComments(buf) {
+  let lines = buf.split("\n");
+  let entryRegex = /^\s*\/\//;
+  let data = "";
+  for (let i = 0; i < lines.length; ++i) {
+    let match = entryRegex.exec(lines[i]);
+    if (!match) {
+      data = data + lines[i];
+    }
+  }
+  return data;
+}
+
+function download(filename) {
+  let req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+              .createInstance(Ci.nsIXMLHttpRequest);
+  req.open("GET", filename, false); // doing the request synchronously
+  try {
+    req.send();
+  } catch (e) {
+    throw new Error(`ERROR: problem downloading '${filename}': ${e}`);
+  }
+
+  if (req.status != 200) {
+    throw new Error("ERROR: problem downloading '" + filename + "': status " +
+                    req.status);
+  }
+
+  let resultDecoded;
+  try {
+    resultDecoded = atob(req.responseText);
+  } catch (e) {
+    throw new Error("ERROR: could not decode data as base64 from '" + filename +
+                    "': " + e);
+  }
+  return resultDecoded;
+}
+
 function downloadAsJson(filename) {
-  let jsonWithComments = PSMToolUtils.downloadFile(filename, true);
-  let result = PSMToolUtils.stripComments(jsonWithComments);
+  // we have to filter out '//' comments, while not mangling the json
+  let result = download(filename).replace(/^(\s*)?\/\/[^\n]*\n/mg, "");
   let data = null;
   try {
     data = JSON.parse(result);
   } catch (e) {
     throw new Error("ERROR: could not parse data from '" + filename + "': " + e);
   }
   return data;
 }
@@ -168,17 +204,17 @@ function downloadAndParseChromeCerts(fil
 
   // Parsing states.
   const PRE_NAME = 0;
   const POST_NAME = 1;
   const IN_CERT = 2;
   const IN_PUB_KEY = 3;
   let state = PRE_NAME;
 
-  let lines = PSMToolUtils.downloadFile(filename, true).split("\n");
+  let lines = download(filename).split("\n");
   let pemCert = "";
   let pemPubKey = "";
   let hash = "";
   let chromeNameToHash = {};
   let chromeNameToMozName = {};
   let chromeName;
   for (let line of lines) {
     // Skip comments and newlines.
@@ -367,17 +403,17 @@ function loadNSSCertinfo(extraCertificat
     let SKD = "VCIlmPM9NkgFQtrs4Oa5TeFcDu6MWRTKSNdePEhOgD8=";
     certNameToSKD[name] = SKD;
     certSKDToName[SKD] = name;
   }
   return [certNameToSKD, certSKDToName];
 }
 
 function parseJson(filename) {
-  let json = PSMToolUtils.stripComments(readFileToString(filename));
+  let json = stripComments(readFileToString(filename));
   return JSON.parse(json);
 }
 
 function nameToAlias(certName) {
   // change the name to a string valid as a c identifier
   // remove  non-ascii characters
   certName = certName.replace(/[^[:ascii:]]/g, "_");
   // replace non word characters
--- a/security/manager/tools/genRootCAHashes.js
+++ b/security/manager/tools/genRootCAHashes.js
@@ -4,27 +4,28 @@
 "use strict";
 
 // How to run this file:
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell genRootCAHashes.js \
 //                                  [absolute path to]/RootHashes.inc'
 
-const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+var Cr = Components.results;
 
 const nsX509CertDB = "@mozilla.org/security/x509certdb;1";
 const CertDb = Components.classes[nsX509CertDB].getService(Ci.nsIX509CertDB);
 
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+Cu.import("resource://gre/modules/NetUtil.jsm");
 const { CommonUtils } = Cu.import("resource://services-common/utils.js", {});
-const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
-const { NetUtil } = Cu.import("resource://gre/modules/NetUtil.jsm", {});
-const { PSMToolUtils } =
-  Cu.import(`file:///${__LOCATION__.parent.path}/PSMToolUtils.jsm`, {});
-const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
 
 const FILENAME_OUTPUT = "RootHashes.inc";
 const FILENAME_TRUST_ANCHORS = "KnownRootHashes.json";
 const ROOT_NOT_ASSIGNED = -1;
 
 const JSON_HEADER = "// This Source Code Form is subject to the terms of the Mozilla Public\n" +
 "// License, v. 2.0. If a copy of the MPL was not distributed with this\n" +
 "// file, You can obtain one at http://mozilla.org/MPL/2.0/. */\n" +
@@ -73,24 +74,38 @@ function hexSlice(bytes, start, end) {
     ret += "0x" + hex;
     if (i < end - 1) {
       ret += ", ";
     }
   }
   return ret;
 }
 
+function stripComments(buf) {
+  let lines = buf.split("\n");
+  let entryRegex = /^\s*\/\//;
+  let data = "";
+  for (let i = 0; i < lines.length; i++) {
+    let match = entryRegex.exec(lines[i]);
+    if (!match) {
+      data = data + lines[i];
+    }
+  }
+  return data;
+}
+
+
 // Load the trust anchors JSON object from disk
 function loadTrustAnchors(file) {
   if (file.exists()) {
     let stream = Cc["@mozilla.org/network/file-input-stream;1"]
                    .createInstance(Ci.nsIFileInputStream);
     stream.init(file, -1, 0, 0);
     let buf = NetUtil.readInputStreamToString(stream, stream.available());
-    return JSON.parse(PSMToolUtils.stripComments(buf));
+    return JSON.parse(stripComments(buf));
   }
   // If there's no input file, bootstrap.
   return { roots: [], maxBin: 0 };
 }
 
 // Saves our persistence file so that we don't lose track of the mapping
 // between bin numbers and the CA-hashes, even as CAs come and go.
 function writeTrustAnchors(file) {
--- a/security/manager/tools/getHSTSPreloadList.js
+++ b/security/manager/tools/getHSTSPreloadList.js
@@ -7,23 +7,24 @@
 // 1. [obtain firefox source code]
 // 2. [build/obtain firefox binaries]
 // 3. run `[path to]/run-mozilla.sh [path to]/xpcshell \
 //                                  [path to]/getHSTSPreloadlist.js \
 //                                  [absolute path to]/nsSTSPreloadlist.inc'
 // Note: Running this file outputs a new nsSTSPreloadlist.inc in the current
 //       working directory.
 
-const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
+var Cc = Components.classes;
+var Ci = Components.interfaces;
+var Cu = Components.utils;
+var Cr = Components.results;
 
-const { FileUtils } = Cu.import("resource://gre/modules/FileUtils.jsm", {});
-const { PSMToolUtils } =
-  Cu.import(`file:///${__LOCATION__.parent.path}/PSMToolUtils.jsm`, {});
-const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
-const { XPCOMUtils } = Cu.import("resource://gre/modules/XPCOMUtils.jsm", {});
+Cu.import("resource://gre/modules/Services.jsm");
+Cu.import("resource://gre/modules/FileUtils.jsm");
+Cu.import("resource://gre/modules/XPCOMUtils.jsm");
 
 const SOURCE = "https://chromium.googlesource.com/chromium/src/net/+/master/http/transport_security_state_static.json?format=TEXT";
 const OUTPUT = "nsSTSPreloadList.inc";
 const ERROR_OUTPUT = "nsSTSPreloadList.errors";
 const MINIMUM_REQUIRED_MAX_AGE = 60 * 60 * 24 * 7 * 18;
 const MAX_CONCURRENT_REQUESTS = 5;
 const MAX_RETRIES = 3;
 const REQUEST_TIMEOUT = 30 * 1000;
@@ -39,18 +40,40 @@ const HEADER = "/* This Source Code Form
 "/* This is an automatically generated file. If you're not                    */\n" +
 "/* nsSiteSecurityService.cpp, you shouldn't be #including it.     */\n" +
 "/*****************************************************************************/\n" +
 "\n" +
 "#include <stdint.h>\n";
 const GPERF_DELIM = "%%\n";
 
 function download() {
-  let resultDecoded = PSMToolUtils.downloadFile(SOURCE, true);
-  let result = PSMToolUtils.stripComments(resultDecoded);
+  var req = Cc["@mozilla.org/xmlextras/xmlhttprequest;1"]
+            .createInstance(Ci.nsIXMLHttpRequest);
+  req.open("GET", SOURCE, false); // doing the request synchronously
+  try {
+    req.send();
+  } catch (e) {
+    throw new Error(`ERROR: problem downloading '${SOURCE}': ${e}`);
+  }
+
+  if (req.status != 200) {
+    throw new Error("ERROR: problem downloading '" + SOURCE + "': status " +
+                    req.status);
+  }
+
+  var resultDecoded;
+  try {
+    resultDecoded = atob(req.responseText);
+  } catch (e) {
+    throw new Error("ERROR: could not decode data as base64 from '" + SOURCE +
+                    "': " + e);
+  }
+
+  // we have to filter out '//' comments, while not mangling the json
+  var result = resultDecoded.replace(/^(\s*)?\/\/[^\n]*\n/mg, "");
   var data = null;
   try {
     data = JSON.parse(result);
   } catch (e) {
     throw new Error(`ERROR: could not parse data from '${SOURCE}': ${e}`);
   }
   return data;
 }
deleted file mode 100644
--- a/security/manager/tools/moz.build
+++ /dev/null
@@ -1,15 +0,0 @@
-# -*- Mode: python; indent-tabs-mode: nil; tab-width: 40 -*-
-# vim: set filetype=python:
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-
-XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell.ini']
-
-# This JSM is only used by the xpcshell scripts in this directory and not by
-# Firefox itself, so we only register the JSM as a test module to:
-#  - Avoid bloat.
-#  - Make tests slightly cleaner.
-# xpcshell scripts can instead import the JSM using a file:// URL constructed
-# using the built in |__LOCATION__| nsIFile constant.
-TESTING_JS_MODULES.psm += ['PSMToolUtils.jsm']
deleted file mode 100644
--- a/security/manager/tools/tests/.eslintrc.js
+++ /dev/null
@@ -1,5 +0,0 @@
-"use strict";
-
-module.exports = {
-  "extends": "plugin:mozilla/xpcshell-test"
-};
deleted file mode 100644
--- a/security/manager/tools/tests/test_psmtoolutils.js
+++ /dev/null
@@ -1,108 +0,0 @@
-// -*- indent-tabs-mode: nil; js-indent-level: 2 -*-
-// Any copyright is dedicated to the Public Domain.
-// http://creativecommons.org/publicdomain/zero/1.0/
-"use strict";
-
-// Tests PSMToolUtils.jsm.
-
-const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components;
-
-const { HttpServer } = Cu.import("resource://testing-common/httpd.js", {});
-const { PSMToolUtils } =
-  Cu.import("resource://testing-common/psm/PSMToolUtils.jsm", {});
-const { Services } = Cu.import("resource://gre/modules/Services.jsm", {});
-
-const ResponseType = {
-  BadStatusCode: { contents: "", statusCode: 404 },
-  Base64: { contents: "eyJmb28iOiJiYXIifQ==" }, // btoa('{"foo":"bar"}')
-  NotBase64: { contents: "not*base 64." },
-};
-
-var gHttpServer = new HttpServer();
-var gHttpServerPrePath;
-
-add_task(function testDownloadFile_Setup() {
-  gHttpServer = new HttpServer();
-  gHttpServer.registerPrefixHandler("/", function(request, response) {
-    let pathWithoutLeadingSlash = request.path.substring(1);
-    let responseType = ResponseType[pathWithoutLeadingSlash];
-    ok(responseType,
-       `'${pathWithoutLeadingSlash}' should be a valid ResponseType`);
-
-    response.setStatusLine(request.httpVersion, responseType.statusCode || 200,
-                           null);
-    response.setHeader("Content-Type", "text/plain");
-    response.write(responseType.contents);
-  });
-  gHttpServer.start(-1);
-  do_register_cleanup(() => {
-    gHttpServer.stop(() => {});
-  });
-  gHttpServerPrePath = `http://localhost:${gHttpServer.identity.primaryPort}`;
-});
-
-add_task(function testDownloadFile_BadStatusCode() {
-  let url = `${gHttpServerPrePath}/BadStatusCode`;
-  throws(() => PSMToolUtils.downloadFile(url, true),
-         /problem downloading.*status 404/,
-         "Exception should be thrown for a non-200 status code");
-});
-
-add_task(function testDownloadFile_DecodeAsBase64_Base64() {
-  let url = `${gHttpServerPrePath}/Base64`;
-  let contents = PSMToolUtils.downloadFile(url, true);
-  equal(contents, '{"foo":"bar"}', "Should get back decoded Base64 contents");
-});
-
-add_task(function testDownloadFile_DecodeAsBase64_NotBase64() {
-  let url = `${gHttpServerPrePath}/NotBase64`;
-  throws(() => PSMToolUtils.downloadFile(url, true),
-         /could not decode data as base64/,
-         "Exception should be thrown if we get non-Base64 content when we " +
-         "were expecting otherwise");
-});
-
-add_task(function testDownloadFile_DontDecodeAsBase64_Base64() {
-  let url = `${gHttpServerPrePath}/Base64`;
-  let contents = PSMToolUtils.downloadFile(url, false);
-  equal(contents, ResponseType.Base64.contents,
-        "Should get back undecoded Base64 contents");
-});
-
-add_task(function testDownloadFile_DontDecodeAsBase64_NotBase64() {
-  let url = `${gHttpServerPrePath}/NotBase64`;
-  let contents = PSMToolUtils.downloadFile(url, false);
-  equal(contents, ResponseType.NotBase64.contents,
-        "Should get back expected contents");
-});
-
-add_task(function testDownloadFile_InvalidDomain() {
-  let domain = "test-psmtoolutils.js.nosuchdomain.test";
-  Services.prefs.setCharPref("network.dns.localDomains", domain);
-  throws(() => PSMToolUtils.downloadFile(`http://${domain}/foo`, true),
-         /problem downloading/,
-         "Exception should be thrown if trying to download from an invalid " +
-         "domain");
-});
-
-add_task(function testStripComments() {
-  const tests = [
-    {input: "", expected: ""},
-    {input: "{}", expected: "{}"},
-    {input: "     //\n", expected: ""},
-    {input: "\t//\n//\n   //\n", expected: ""},
-    {input: '{"foo":"bar"}', expected: '{"foo":"bar"}'},
-    // From Bug 1197607.
-    {input: '"report_uri": "http://clients3.google.com/cert_upload_json"',
-     expected: '"report_uri": "http://clients3.google.com/cert_upload_json"'},
-    {input: '//abc\n{"foo":"bar"}', expected: '{"foo":"bar"}'},
-    {input: '{"foo":"bar",\n//baz\n"aaa": "bbb"}',
-     expected: '{"foo":"bar",\n"aaa": "bbb"}'},
-    {input: "// foo http://example.com/baz\n", expected: ""},
-  ];
-  for (let test of tests) {
-    let result = PSMToolUtils.stripComments(test.input);
-    equal(test.expected, result,
-          `Expected and actual result should match for input '${test.input}'`);
-  }
-});
deleted file mode 100644
--- a/security/manager/tools/tests/xpcshell.ini
+++ /dev/null
@@ -1,4 +0,0 @@
-[DEFAULT]
-tags = psm
-
-[test_psmtoolutils.js]