Bug 1390908 - Make mitmproxy osx binary available on tooltool and update talos tp6 accordingly; r=jmaher
authorRob Wood <rwood@mozilla.com>
Thu, 21 Sep 2017 14:35:38 -0400
changeset 382469 8247c544a7e2246549af8dfcef70308f5c36f676
parent 382468 d7435cf7c4e7b2f09f07aaa70e7e6e5981e3a24c
child 382470 0bd07c5647226c0b7bc384e83310a22a743305fc
push id32558
push userkwierso@gmail.com
push dateFri, 22 Sep 2017 21:29:46 +0000
treeherdermozilla-central@61e58a7d800b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1390908
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1390908 - Make mitmproxy osx binary available on tooltool and update talos tp6 accordingly; r=jmaher MozReview-Commit-ID: 4YfZMifpq2p
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/talos.json
testing/talos/talos/mitmproxy/mitmproxy-rel-bin-osx.manifest
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -193,16 +193,17 @@ class Talos(TestingMixin, MercurialScrip
         self.talos_json = self.config.get("talos_json")
         self.talos_json_config = self.config.get("talos_json_config")
         self.repo_path = self.config.get("repo_path")
         self.obj_path = self.config.get("obj_path")
         self.tests = None
         self.gecko_profile = self.config.get('gecko_profile')
         self.gecko_profile_interval = self.config.get('gecko_profile_interval')
         self.pagesets_name = None
+        self.mitmproxy_rel_bin = None # some platforms download a mitmproxy release binary
         self.mitmproxy_recording_set = None # zip file found on tooltool that contains all of the mitmproxy recordings
         self.mitmproxy_recordings_file_list = self.config.get('mitmproxy', None) # files inside the recording set
         self.mitmdump = None # path to mitdump tool itself, in py3 venv
 
     # We accept some configuration options from the try commit message in the format mozharness: <options>
     # Example try commit message:
     #   mozharness: --geckoProfile try: <stuff>
     def query_gecko_profile_options(self):
@@ -452,24 +453,55 @@ class Talos(TestingMixin, MercurialScrip
     def install_mitmproxy(self):
         """Install the mitmproxy tool into the Python 3.x env"""
         if 'win' in self.platform_name():
             self.info("Installing mitmproxy")
             self.py3_install_modules(modules=['mitmproxy'])
             self.mitmdump = os.path.join(self.py3_path_to_executables(), 'mitmdump')
         else:
             # on macosx we use a prebuilt mitmproxy release binary
-            mitmproxy_bin_url = 'https://github.com/mitmproxy/mitmproxy/releases/download/v2.0.2/mitmproxy-2.0.2-osx.tar.gz'
             mitmproxy_path = os.path.join(self.talos_path, 'talos', 'mitmproxy')
             self.mitmdump = os.path.join(mitmproxy_path, 'mitmdump')
             if not os.path.exists(self.mitmdump):
-                self.download_unpack(mitmproxy_bin_url, mitmproxy_path)
+                # download the mitmproxy release binary; will be overridden by the --no-download
+                if '--no-download' not in self.config['talos_extra_options']:
+                    self.query_mitmproxy_rel_bin('osx')
+                    if self.mitmproxy_rel_bin is None:
+                        self.fatal("Aborting: mitmproxy_release_bin_osx not found in talos.json")
+                    self.download_mitmproxy_binary('osx')
+                else:
+                    self.info("Not downloading mitmproxy rel binary because no-download was specified")
             self.info('The mitmdump macosx binary is found at: %s' % self.mitmdump)
         self.run_command([self.mitmdump, '--version'], env=self.query_env())
 
+    def query_mitmproxy_rel_bin(self, platform):
+        """Mitmproxy requires external playback archives to be downloaded and extracted"""
+        if self.mitmproxy_rel_bin:
+            return self.mitmproxy_rel_bin
+        if self.query_talos_json_config() and self.suite is not None:
+            config_key = "mitmproxy_release_bin_" + platform
+            self.mitmproxy_rel_bin = self.talos_json_config['suites'][self.suite].get(config_key, False)
+            return self.mitmproxy_rel_bin
+
+    def download_mitmproxy_binary(self, platform):
+        """Download the mitmproxy release binary from tooltool"""
+        self.info("Downloading the mitmproxy release binary using tooltool")
+        dest = os.path.join(self.talos_path, 'talos', 'mitmproxy')
+        _manifest = "mitmproxy-rel-bin-%s.manifest" % platform
+        manifest_file = os.path.join(self.talos_path, 'talos', 'mitmproxy', _manifest)
+        self.tooltool_fetch(
+            manifest_file,
+            output_dir=dest,
+            cache=self.config.get('tooltool_cache')
+        )
+        archive = os.path.join(dest, self.mitmproxy_rel_bin)
+        tar = self.query_exe('tar')
+        unzip_cmd = [tar, '-xvzf', archive, '-C', dest]
+        self.run_command(unzip_cmd, halt_on_failure=True)
+
     def query_mitmproxy_recording_set(self):
         """Mitmproxy requires external playback archives to be downloaded and extracted"""
         if self.mitmproxy_recording_set:
             return self.mitmproxy_recording_set
         if self.query_talos_json_config() and self.suite is not None:
             self.mitmproxy_recording_set = self.talos_json_config['suites'][self.suite].get('mitmproxy_recording_set', False)
             return self.mitmproxy_recording_set
 
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -106,35 +106,38 @@
             "talos_options": [
                 "--disable-stylo",
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
             ]
         },
         "tp6-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--mitmproxy",
                 "mitmproxy-recording-google.mp mitmproxy-recording-youtube.mp mitmproxy-recording-amazon.mp mitmproxy-recording-facebook.mp",
                 "--firstNonBlankPaint"
             ]
         },
         "tp6-stylo-disabled-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--disable-stylo",
                 "--mitmproxy",
                 "mitmproxy-recording-google.mp mitmproxy-recording-youtube.mp mitmproxy-recording-amazon.mp mitmproxy-recording-facebook.mp",
                 "--firstNonBlankPaint"
             ]
         },
         "tp6-stylo-threads-e10s": {
             "tests": ["tp6_google", "tp6_youtube", "tp6_amazon", "tp6_facebook"],
+            "mitmproxy_release_bin_osx": "mitmproxy-2.0.2-osx.tar.gz",
             "mitmproxy_recording_set": "mitmproxy-recording-set-win10.zip",
             "talos_options": [
                 "--stylo-threads=1",
                 "--mitmproxy",
                 "mitmproxy-recording-google.mp mitmproxy-recording-youtube.mp mitmproxy-recording-amazon.mp mitmproxy-recording-facebook.mp",
                 "--firstNonBlankPaint"
             ]
         }
new file mode 100644
--- /dev/null
+++ b/testing/talos/talos/mitmproxy/mitmproxy-rel-bin-osx.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "mitmproxy-2.0.2-osx.tar.gz",
+        "size": 32324573,
+        "digest": "06423c76e7e99fd9705eae3dc6e2423b1ffb8c42caa98fd010d59dc6ed1f0827376e238c48108106da558444b826e085a58aeb30cf9c79e9d0122a2cb17ae8e6",
+        "algorithm": "sha512",
+        "unpack": false
+    }
+]