Bug 1390908 - Make mitmproxy osx binary available on tooltool and update talos tp6 accordingly. r=jmaher, a=test-only
authorRob Wood <rwood@mozilla.com>
Thu, 21 Sep 2017 14:35:38 -0400
changeset 432784 433a682865c1eb4bd7e1e150f0d1dc7619ba3e19
parent 432783 747911087c22b438e0095b9e5c49ddc54cde4ade
child 432785 073a2a43a15cd103d2671b7e77d6700ddf3690f6
push id8059
push userryanvm@gmail.com
push dateThu, 26 Oct 2017 16:45:03 +0000
treeherdermozilla-beta@c5ed62b37c2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher, test-only
bugs1390908
milestone57.0
Bug 1390908 - Make mitmproxy osx binary available on tooltool and update talos tp6 accordingly. r=jmaher, a=test-only MozReview-Commit-ID: 4YfZMifpq2p
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/talos.json
testing/talos/talos/mitmproxy/mitmproxy-rel-bin-osx.manifest
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -193,16 +193,17 @@ class Talos(TestingMixin, MercurialScrip
         self.talos_json = self.config.get("talos_json")
         self.talos_json_config = self.config.get("talos_json_config")
         self.repo_path = self.config.get("repo_path")
         self.obj_path = self.config.get("obj_path")
         self.tests = None
         self.gecko_profile = self.config.get('gecko_profile')
         self.gecko_profile_interval = self.config.get('gecko_profile_interval')
         self.pagesets_name = None
+        self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary
         self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings
         self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set
         self.mitmdump = None # path to mitdump tool itself, in py3 venv
 
     # We accept some configuration options from the try commit message in the format mozharness: <options>
     # Example try commit message:
     #   mozharness: --geckoProfile try: <stuff>
     def query_gecko_profile_options(self):
@@ -452,24 +453,55 @@ class Talos(TestingMixin, MercurialScrip
     def install_mitmproxy(self):
         """Install the mitmproxy tool into the Python 3.x env"""
         if 'win' in self.platform_name():
             self.info("Installing mitmproxy")
             self.py3_install_modules(modules=['mitmproxy'])
             self.mitmdump = os.path.join(self.py3_path_to_executables(), 'mitmdump')
         else:
             # on macosx we use a prebuilt mitmproxy release binary
-            mitmproxy_bin_url = 'https://github.com/mitmproxy/mitmproxy/releases/download/v2.0.2/mitmproxy-2.0.2-osx.tar.gz'
             mitmproxy_path = os.path.join(self.talos_path, 'talos', 'mitmproxy')
             self.mitmdump = os.path.join(mitmproxy_path, 'mitmdump')
             if not os.path.exists(self.mitmdump):
-                self.download_unpack(mitmproxy_bin_url, mitmproxy_path)
+                # download the mitmproxy release binary; will be overridden by the --no-download
+                if '--no-download' not in self.config['talos_extra_options']:
+                    self.query_mitmproxy_rel_bin('osx')
+                    if self.mitmproxy_rel_bin is None:
+                        self.fatal("Aborting: mitmproxy_release_bin_osx not found in talos.json")
+                    self.download_mitmproxy_binary('osx')
+                else:
+                    self.info("Not downloading mitmproxy rel binary because no-download was specified")
             self.info('The mitmdump macosx binary is found at: %s' % self.mitmdump)
         self.run_command([self.mitmdump, '--version'], env=self.query_env())
 
+    def query_mitmproxy_rel_bin(self, platform):
+        """Mitmproxy requires external playback archives to be downloaded and extracted"""
+        if self.mitmproxy_rel_bin:
+            return self.mitmproxy_rel_bin
+        if self.query_talos_json_config() and self.suite is not None:
+            config_key = "mitmproxy_release_bin_" + platform
+            self.mitmproxy_rel_bin = self.talos_json_config['suites'][self.suite].get(config_key, False)
+            return self.mitmproxy_rel_bin
+
+    def download_mitmproxy_binary(self, platform):
+        """Download the mitmproxy release binary from tooltool"""
+        self.info("Downloading the mitmproxy release binary using tooltool")
+        dest = os.path.join(self.talos_path, 'talos', 'mitmproxy')
+        _manifest = "mitmproxy-rel-bin-%s.manifest" % platform
+        manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', _manifest)
+        self.tooltool_fetch(
+            manifest_file,
+            output_dir=dest,
+            cache=self.config.get('tooltool_cache')
+        )
+        archive = os.path.join(dest, self.mitmproxy_rel_bin)
+        tar = self.query_exe('tar')
+        unzip_cmd = [tar, '-xvzf', archive, '-C', dest]
+        self.run_command(unzip_cmd, halt_on_failure=True)
+
     def query_mitmproxy_recording_set(self):
         """Mitmproxy requires external playback archives to be downloaded and extracted"""
         if self.mitmproxy_recording_set:
             return self.mitmproxy_recording_set
         if self.query_talos_json_config() and self.suite is not None:
             self.mitmproxy_recording_set = self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False)
             return self.mitmproxy_recording_set
 
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -106,35 +106,38 @@
             "talos_options": [
                 "--disable-stylo",
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
             ]
         },
         "tp6-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--mitmproxy",
                 "mitmproxy-recording-google.mp mitmproxy-recording-youtube.mp mitmproxy-recording-amazon.mp mitmproxy-recording-facebook.mp",
                 "--firstNonBlankPaint"
             ]
         },
         "tp6-stylo-disabled-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--disable-stylo",
                 "--mitmproxy",
                 "mitmproxy-recording-google.mp mitmproxy-recording-youtube.mp mitmproxy-recording-amazon.mp mitmproxy-recording-facebook.mp",
                 "--firstNonBlankPaint"
             ]
         },
         "tp6-stylo-threads-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--stylo-threads=1",
                 "--mitmproxy",
                 "mitmproxy-recording-google.mp mitmproxy-recording-youtube.mp mitmproxy-recording-amazon.mp mitmproxy-recording-facebook.mp",
                 "--firstNonBlankPaint"
             ]
         }
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/mitmproxy/mitmproxy-rel-bin-osx.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "mitmproxy-2.0.2-osx.tar.gz",
+        "size": 32324573,
+        "digest": "06423c76e7e99fd9705eae3dc6e2423b1ffb8c42caa98fd010d59dc6ed1f0827376e238c48108106da558444b826e085a58aeb30cf9c79e9d0122a2cb17ae8e6",
+        "algorithm": "sha512",
+        "unpack": false
+    }
+]