Bug 1553932 - Update mozproxy certutil usage. r=tarek, a=test-only
authorFlorin Strugariu <fstrugariu@mozilla.com>
Wed, 07 Aug 2019 08:22:55 +0000
changeset 545083 b529d6d87af4ded8d4bfed19a5ad895d5952054a
parent 545082 c1cad7515d96c90f1def64b21ba2de19e2f52fcf
child 545084 0e87368478c67095b1d0a24c9e48e17980607dbd
push id2131
push userffxbld-merge
push dateMon, 26 Aug 2019 18:30:20 +0000
treeherdermozilla-release@b19ffb3ca153 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewerstarek, test-only
bugs1553932
milestone69.0
Bug 1553932 - Update mozproxy certutil usage. r=tarek, a=test-only Differential Revision: https://phabricator.services.mozilla.com/D39200
testing/mozbase/mozproxy/mozproxy/backends/mitm.py
--- a/testing/mozbase/mozproxy/mozproxy/backends/mitm.py
+++ b/testing/mozbase/mozproxy/mozproxy/backends/mitm.py
@@ -17,17 +17,16 @@ from mozprocess import ProcessHandler
 from mozproxy.backends.base import Playback
 from mozproxy.utils import (
     transform_platform,
     tooltool_download,
     download_file_from_url,
     LOG,
 )
 
-
 here = os.path.dirname(__file__)
 # path for mitmproxy certificate, generated auto after mitmdump is started
 # on local machine it is 'HOME', however it is different on production machines
 try:
     DEFAULT_CERT_PATH = os.path.join(
         os.getenv("HOME"), ".mitmproxy", "mitmproxy-ca-cert.cer"
     )
 except Exception:
@@ -300,18 +299,18 @@ class MitmproxyDesktop(Mitmproxy):
         if not os.path.exists(self.policies_dir):
             LOG.info("creating folder: %s" % self.policies_dir)
             os.makedirs(self.policies_dir)
         else:
             LOG.info("folder already exists: %s" % self.policies_dir)
 
         self.write_policies_json(
             self.policies_dir,
-            policies_content=POLICIES_CONTENT_ON
-            % {"cert": self.cert_path, "host": self.config["host"]},
+            policies_content=POLICIES_CONTENT_ON % {"cert": self.cert_path,
+                                                    "host": self.config["host"]},
         )
 
         # cannot continue if failed to add CA cert to Firefox, need to check
         if not self.is_mitmproxy_cert_installed():
             LOG.error(
                 "Aborting: failed to install mitmproxy CA cert into Firefox desktop"
             )
             self.stop_mitmproxy_playback()
@@ -334,18 +333,18 @@ class MitmproxyDesktop(Mitmproxy):
     def is_mitmproxy_cert_installed(self):
         """Verify mitmxproy CA cert was added to Firefox"""
         try:
             # read autoconfig file, confirm mitmproxy cert is in there
             contents = self.read_policies_json(self.policies_dir)
             LOG.info("Firefox policies file contents:")
             LOG.info(contents)
             if (
-                POLICIES_CONTENT_ON
-                % {"cert": self.cert_path, "host": self.config["host"]}
+                    POLICIES_CONTENT_ON
+                    % {"cert": self.cert_path, "host": self.config["host"]}
             ) in contents:
                 LOG.info("Verified mitmproxy CA certificate is installed in Firefox")
             else:
 
                 return False
         except Exception as e:
             LOG.info("failed to read Firefox policies file, exeption: %s" % e)
             return False
@@ -373,193 +372,193 @@ class MitmproxyAndroid(Mitmproxy):
         self.android_device = android_device
 
     @property
     def certutil_sleep_seconds(self):
         """Time to sleep, in seconds, after issuing a `certutil` command."""
         return 10 if not self.config["run_local"] else 1
 
     def setup(self):
-        """For geckoview we need to install the generated mitmproxy CA cert"""
-        if self.config["app"] in ["fennec", "geckoview", "refbrow", "fenix"]:
-            # install the generated CA certificate into android geckoview
-            self.install_mitmproxy_cert(self.browser_path)
+        self.download_and_install_host_utils()
+        self.install_mitmproxy_cert(self.browser_path)
 
-    def install_mitmproxy_cert(self, browser_path):
-        """Install the CA certificate generated by mitmproxy, into geckoview android
+    def download_and_install_host_utils(self):
+        """
         If running locally:
         1. Will use the `certutil` tool from the local Firefox desktop build
 
         If running in production:
         1. Get the tooltools manifest file for downloading hostutils (contains certutil)
         2. Get the `certutil` tool by downloading hostutils using the tooltool manifest
-
-        Then, both locally and in production:
-        1. Create an NSS certificate database in the geckoview browser profile dir, only
-           if it doesn't already exist. Use this certutil command:
-           `certutil -N -d sql:<path to profile> --empty-password`
-        2. Import the mitmproxy certificate into the database, i.e.:
-           `certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>`
         """
         if self.config["run_local"]:
             # when running locally, it is found in the Firefox desktop build (..obj../dist/bin)
             self.certutil = os.path.join(os.environ["MOZ_HOST_BIN"], "certutil")
             if not (
-                os.path.isfile(self.certutil) and os.access(self.certutil, os.X_OK)
+                    os.path.isfile(self.certutil) and os.access(self.certutil, os.X_OK)
             ):
-                LOG.critical(
-                    "Abort: unable to execute certutil: {}".format(self.certutil)
-                )
-                raise
+                raise Exception("Abort: unable to execute certutil: {}".format(self.certutil))
             self.certutil = os.environ["MOZ_HOST_BIN"]
             os.environ["LD_LIBRARY_PATH"] = self.certutil
         else:
             # must download certutil inside hostutils via tooltool; use this manifest:
             # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
             # after it will be found here inside the worker/bitbar container:
             # /builds/worker/workspace/build/hostutils/host-utils-66.0a1.en-US.linux-x86_64
             LOG.info("downloading certutil binary (hostutils)")
 
             # get path to the hostutils tooltool manifest; was set earlier in
             # mozharness/configs/raptor/android_hw_config.py, to the path i.e.
             # mozilla-central/testing/config/tooltool-manifests/linux64/hostutils.manifest
             # the bitbar container is always linux64
             if os.environ.get("GECKO_HEAD_REPOSITORY", None) is None:
-                LOG.critical("Abort: unable to get GECKO_HEAD_REPOSITORY")
-                raise
+                raise Exception("Abort: unable to get GECKO_HEAD_REPOSITORY")
 
             if os.environ.get("GECKO_HEAD_REV", None) is None:
-                LOG.critical("Abort: unable to get GECKO_HEAD_REV")
-                raise
+                raise Exception("Abort: unable to get GECKO_HEAD_REV")
 
             if os.environ.get("HOSTUTILS_MANIFEST_PATH", None) is not None:
                 manifest_url = os.path.join(
                     os.environ["GECKO_HEAD_REPOSITORY"],
                     "raw-file",
                     os.environ["GECKO_HEAD_REV"],
                     os.environ["HOSTUTILS_MANIFEST_PATH"],
                 )
             else:
-                LOG.critical("Abort: unable to get HOSTUTILS_MANIFEST_PATH!")
-                raise
+                raise Exception("Abort: unable to get HOSTUTILS_MANIFEST_PATH!")
 
             # first need to download the hostutils tooltool manifest file itself
             _dest = os.path.join(self.mozproxy_dir, "hostutils.manifest")
             have_manifest = download_file_from_url(manifest_url, _dest)
             if not have_manifest:
-                LOG.critical("failed to download the hostutils tooltool manifest")
-                raise
+                raise Exception("failed to download the hostutils tooltool manifest")
 
             # now use the manifest to download hostutils so we can get certutil
             tooltool_download(_dest, self.config["run_local"], self.mozproxy_dir)
 
             # the production bitbar container host is always linux
             self.certutil = glob.glob(
                 os.path.join(self.mozproxy_dir, "host-utils*[!z|checksum]")
             )[0]
 
             # must add hostutils/certutil to the path
             os.environ["LD_LIBRARY_PATH"] = self.certutil
 
         bin_suffix = mozinfo.info.get("bin_suffix", "")
         self.certutil = os.path.join(self.certutil, "certutil" + bin_suffix)
-
         if os.path.isfile(self.certutil):
             LOG.info("certutil is found at: %s" % self.certutil)
         else:
-            LOG.critical("unable to find certutil at %s" % self.certutil)
-            raise
+            raise Exception("unable to find certutil at %s" % self.certutil)
+
+    def install_mitmproxy_cert(self, browser_path):
+        """Install the CA certificate generated by mitmproxy, into geckoview android
+
+        1. Create an NSS certificate database in the geckoview browser profile dir, only
+           if it doesn't already exist. Use this certutil command:
+           `certutil -N -d sql:<path to profile> --empty-password`
+        2. Import the mitmproxy certificate into the database, i.e.:
+           `certutil -A -d sql:<path to profile> -n "some nickname" -t TC,, -a -i <path to CA.pem>`
+        """
+
+        cert_db_location = "sql:%s/" % self.config["local_profile_dir"]
+
+        if not self.cert_db_exists(cert_db_location):
+            self.create_cert_db(cert_db_location)
 
         # DEFAULT_CERT_PATH has local path and name of mitmproxy cert i.e.
         # /home/cltbld/.mitmproxy/mitmproxy-ca-cert.cer
-        self.local_cert_path = DEFAULT_CERT_PATH
-
-        # check if the nss ca cert db already exists in the device profile
-        LOG.info(
-            "checking if the nss cert db already exists in the android browser profile"
-        )
-        param1 = "sql:%s/" % self.config["local_profile_dir"]
-        command = [self.certutil, "-d", param1, "-L"]
-
-        try:
-            subprocess.check_output(command, env=os.environ.copy())
-            LOG.info("the nss cert db already exists")
-            cert_db_exists = True
-        except subprocess.CalledProcessError:
-            # this means the nss cert db doesn't exist yet
-            LOG.info("nss cert db doesn't exist yet")
-            cert_db_exists = False
+        self.import_certificate_in_cert_db(cert_db_location, DEFAULT_CERT_PATH)
 
-        # try a forced pause between certutil cmds; possibly reduce later
-        time.sleep(self.certutil_sleep_seconds)
-
-        if not cert_db_exists:
-            # create cert db if it doesn't already exist; it may exist already
-            # if a previous pageload test ran in the same test suite
-            param1 = "sql:%s/" % self.config["local_profile_dir"]
-            command = [self.certutil, "-N", "-v", "-d", param1, "--empty-password"]
+        # cannot continue if failed to add CA cert to Firefox, need to check
+        if not self.is_mitmproxy_cert_installed(cert_db_location):
+            LOG.error("Aborting: failed to install mitmproxy CA cert into Firefox")
+            self.stop_mitmproxy_playback()
+            sys.exit()
 
-            LOG.info("creating nss cert database using command: %s" % " ".join(command))
-            cmd_proc = subprocess.Popen(command, env=os.environ.copy())
-            time.sleep(self.certutil_sleep_seconds)
-            cmd_terminated = cmd_proc.poll()
-            if cmd_terminated is None:  # None value indicates process hasn't terminated
-                LOG.critical("nss cert db creation command failed to complete")
-                raise
-
+    def import_certificate_in_cert_db(self, cert_db_location, local_cert_path):
         # import mitmproxy cert into the db
         command = [
             self.certutil,
             "-A",
             "-d",
-            param1,
+            cert_db_location,
             "-n",
             "mitmproxy-cert",
             "-t",
             "TC,,",
             "-a",
             "-i",
-            self.local_cert_path,
+            local_cert_path,
         ]
 
         LOG.info(
             "importing mitmproxy cert into db using command: %s" % " ".join(command)
         )
+
         cmd_proc = subprocess.Popen(command, env=os.environ.copy())
         time.sleep(self.certutil_sleep_seconds)
+
         cmd_terminated = cmd_proc.poll()
         if cmd_terminated is None:  # None value indicates process hasn't terminated
             LOG.critical(
                 "command to import mitmproxy cert into cert db failed to complete"
             )
 
-        # cannot continue if failed to add CA cert to Firefox, need to check
-        if not self.is_mitmproxy_cert_installed():
-            LOG.error("Aborting: failed to install mitmproxy CA cert into Firefox")
-            self.stop_mitmproxy_playback()
-            sys.exit()
+    def create_cert_db(self, cert_db_location):
+        # create cert db if it doesn't already exist; it may exist already
+        # if a previous pageload test ran in the same test suite
+        command = [self.certutil, "-d", cert_db_location, "-N", "--empty-password"]
+
+        LOG.info("creating nss cert database using command: %s" % " ".join(command))
+        cmd_proc = subprocess.Popen(command, env=os.environ.copy())
+        time.sleep(self.certutil_sleep_seconds)
+        cmd_terminated = cmd_proc.poll()
+        if cmd_terminated is None:  # None value indicates process hasn't terminated
+            raise Exception("nss cert db creation command failed to complete")
+        if not self.cert_db_exists(cert_db_location):
+            raise Exception("nss cert db creation command failed. Cert db not created.")
 
-    def is_mitmproxy_cert_installed(self):
+    def cert_db_exists(self, cert_db_location):
+        # check if the nss ca cert db already exists in the device profile
+        LOG.info(
+            "checking if the nss cert db already exists in the android browser profile"
+        )
+        command = [self.certutil, "-d", cert_db_location, "-L"]
+
+        try:
+            subprocess.check_call(command, env=os.environ.copy())
+            LOG.info("the nss cert db exists")
+            cert_db_exists = True
+        except subprocess.CalledProcessError:
+            # this means the nss cert db doesn't exist yet
+            LOG.info("nss cert db doesn't exist yet. Note: certutil error is expected!!!")
+            cert_db_exists = False
+
+        # try a forced pause between certutil cmds; possibly reduce later
+        time.sleep(self.certutil_sleep_seconds)
+
+        return cert_db_exists
+
+    def is_mitmproxy_cert_installed(self, cert_db_location):
         """Verify mitmxproy CA cert was added to Firefox on android"""
         LOG.info("verifying that the mitmproxy ca cert is installed on android")
 
         # list the certifcates that are in the nss cert db (inside the browser profile dir)
         LOG.info(
             "getting the list of certs in the nss cert db in the android browser profile"
         )
-        param1 = "sql:%s/" % self.config["local_profile_dir"]
-        command = [self.certutil, "-d", param1, "-L"]
+        command = [self.certutil, "-d", cert_db_location, "-L"]
 
         try:
             cmd_output = subprocess.check_output(command, env=os.environ.copy())
-
         except subprocess.CalledProcessError:
             # cmd itself failed
-            LOG.critical("certutil command failed")
-            raise
+            LOG.error(cmd_output)
+            raise Exception("certutil command failed")
 
         # check output from the certutil command, see if 'mitmproxy-cert' is listed
         time.sleep(self.certutil_sleep_seconds)
         LOG.info(cmd_output)
         if "mitmproxy-cert" in cmd_output:
             LOG.info(
                 "verfied the mitmproxy-cert is installed in the nss cert db on android"
             )