Bug 1234500 - Allow mach talos-test to download tp5 pageset; r=jmaher
authorRob Wood <rwood@mozilla.com>
Mon, 07 Nov 2016 19:35:12 -0500
changeset 367588 8e93080f949ac7d164e3b74402aed90dfa77c282
parent 367587 71f75f75b4e73cfa09a564f8c3c1a7e56f797641
child 367589 1bd4e2805bec3d9c665331098b61977aa9f31293
push id6996
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 20:48:21 +0000
treeherdermozilla-beta@d89512dab048 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1234500
milestone53.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1234500 - Allow mach talos-test to download tp5 pageset; r=jmaher MozReview-Commit-ID: Jj5H3G2hLg0
.hgignore
testing/mozharness/mozharness/mozilla/testing/talos.py
testing/talos/mach_commands.py
testing/talos/talos.json
testing/talos/tp5n-pageset.manifest
--- a/.hgignore
+++ b/.hgignore
@@ -110,25 +110,27 @@ GPATH
 # Ignore tox generated dir
 .tox/
 
 # Ignore node_modules
 ^tools/lint/eslint/node_modules/
 
 # Ignore talos virtualenv and tp5n files.
 # The tp5n set is supposed to be decompressed at
-# testing/talos/talos/page_load_test/tp5n in order to run tests like tps
+# testing/talos/talos/tests/tp5n in order to run tests like tps
 # locally. Similarly, running talos requires a Python package virtual
 # environment. Both the virtual environment and tp5n files end up littering
 # the status command, so we ignore them.
 ^testing/talos/.Python
 ^testing/talos/bin/
 ^testing/talos/include/
 ^testing/talos/lib/
 ^testing/talos/talos/tests/tp5n.zip
+^testing/talos/talos/tests/tp5n.tar.gz
 ^testing/talos/talos/tests/tp5n
 ^testing/talos/talos/tests/devtools/damp.manifest.develop
+^talos-venv
 
 # Ignore files created when running a reftest.
 ^lextab.py$
 
 # tup database
 ^\.tup
--- a/testing/mozharness/mozharness/mozilla/testing/talos.py
+++ b/testing/mozharness/mozharness/mozilla/testing/talos.py
@@ -20,16 +20,17 @@ from mozharness.base.errors import Pytho
 from mozharness.base.log import OutputParser, DEBUG, ERROR, CRITICAL
 from mozharness.base.log import INFO, WARNING
 from mozharness.mozilla.blob_upload import BlobUploadMixin, blobupload_config_options
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.mozilla.testing.errors import TinderBoxPrintRe
 from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WORST_LEVEL_TUPLE
 from mozharness.mozilla.buildbot import TBPL_RETRY, TBPL_FAILURE, TBPL_WARNING
+from mozharness.mozilla.tooltool import TooltoolMixin
 
 external_tools_path = os.path.join(
     os.path.abspath(os.path.dirname(os.path.dirname(mozharness.__file__))),
     'external_tools',
 )
 
 TalosErrorList = PythonErrorList + [
     {'regex': re.compile(r'''run-as: Package '.*' is unknown'''), 'level': DEBUG},
@@ -81,17 +82,17 @@ class TalosOutputParser(OutputParser):
         harness_retry_re = TinderBoxPrintRe['harness_error']['retry_regex']
         if harness_retry_re.search(line):
             self.critical(' %s' % line)
             self.update_worst_log_and_tbpl_levels(CRITICAL, TBPL_RETRY)
             return  # skip base parse_single_line
         super(TalosOutputParser, self).parse_single_line(line)
 
 
-class Talos(TestingMixin, MercurialScript, BlobUploadMixin):
+class Talos(TestingMixin, MercurialScript, BlobUploadMixin, TooltoolMixin):
     """
     install and run Talos tests:
     https://wiki.mozilla.org/Buildbot/Talos
     """
     config_options = [
         [["--use-talos-json"],
          {"action": "store_true",
           "dest": "use_talos_json",
@@ -159,19 +160,19 @@ class Talos(TestingMixin, MercurialScrip
         self.workdir = self.query_abs_dirs()['abs_work_dir']  # convenience
 
         self.run_local = self.config.get('run_local')
         self.installer_url = self.config.get("installer_url")
         self.talos_json_url = self.config.get("talos_json_url")
         self.talos_json = self.config.get("talos_json")
         self.talos_json_config = self.config.get("talos_json_config")
         self.tests = None
-        self.pagesets_url = None
         self.sps_profile = self.config.get('sps_profile')
         self.sps_profile_interval = self.config.get('sps_profile_interval')
+        self.pagesets_name = None
 
     # We accept some configuration options from the try commit message in the format mozharness: <options>
     # Example try commit message:
     #   mozharness: --spsProfile try: <stuff>
     def query_sps_profile_options(self):
         sps_results = []
         if self.buildbot_config:
             # this is inside automation
@@ -218,25 +219,52 @@ class Talos(TestingMixin, MercurialScrip
         if self.talos_json_config:
             return self.talos_json_config
         if not self.talos_json:
             self.talos_json = os.path.join(self.talos_path, 'talos.json')
         self.talos_json_config = parse_config_file(self.talos_json)
         self.info(pprint.pformat(self.talos_json_config))
         return self.talos_json_config
 
-    def query_pagesets_url(self):
+    def query_pagesets_name(self):
         """Certain suites require external pagesets to be downloaded and
         extracted.
         """
-        if self.pagesets_url:
-            return self.pagesets_url
-        if self.query_talos_json_config() and 'suite' in self.config:
-            self.pagesets_url = self.talos_json_config['suites'][self.config['suite']].get('pagesets_url')
-            return self.pagesets_url
+        if self.pagesets_name:
+            return self.pagesets_name
+        if self.query_talos_json_config() and self.suite is not None:
+            self.pagesets_name = self.talos_json_config['suites'][self.suite].get('pagesets_name')
+            return self.pagesets_name
+
+    def get_suite_from_test(self):
+        """ Retrieve the talos suite name from a given talos test name."""
+        # running locally, single test name provided instead of suite; go through tests and find suite name
+        suite_name = None
+        if self.query_talos_json_config():
+            if '-a' in self.config['talos_extra_options']:
+                test_name_index = self.config['talos_extra_options'].index('-a') + 1
+            if '--activeTests' in self.config['talos_extra_options']:
+                test_name_index = self.config['talos_extra_options'].index('--activeTests') + 1
+            if test_name_index < len(self.config['talos_extra_options']):
+                test_name = self.config['talos_extra_options'][test_name_index]
+                for talos_suite in self.talos_json_config['suites']:
+                    if test_name in self.talos_json_config['suites'][talos_suite].get('tests'):
+                        suite_name = talos_suite
+            if not suite_name:
+                # no suite found to contain the specified test, error out
+                self.fatal("Test name is missing or invalid")
+        else:
+            self.fatal("Talos json config not found, cannot verify suite")
+        return suite_name
+
+    def validate_suite(self):
+        """ Ensure suite name is a valid talos suite. """
+        if self.query_talos_json_config() and self.suite is not None:
+            if not self.suite in self.talos_json_config.get('suites'):
+                self.fatal("Suite '%s' is not valid (not found in talos json config)" % self.suite)
 
     def talos_options(self, args=None, **kw):
         """return options to talos"""
         # binary path
         binary_path = self.binary_path or self.config.get('binary_path')
         if not binary_path:
             self.fatal("Talos requires a path to the binary.  You can specify binary_path or add download-and-extract to your action list.")
 
@@ -269,31 +297,51 @@ class Talos(TestingMixin, MercurialScrip
         if args is not None:
             options += args
         if 'talos_extra_options' in self.config:
             options += self.config['talos_extra_options']
         return options
 
     def populate_webroot(self):
         """Populate the production test slaves' webroots"""
-        c = self.config
-
         self.talos_path = os.path.join(
             self.query_abs_dirs()['abs_work_dir'], 'tests', 'talos'
         )
-        if c.get('run_local'):
-            self.talos_path = os.path.dirname(self.talos_json)
-
-        src_talos_webdir = os.path.join(self.talos_path, 'talos')
 
-        if self.query_pagesets_url():
-            self.info('Downloading pageset...')
-            dirs = self.query_abs_dirs()
-            src_talos_pageset = os.path.join(src_talos_webdir, 'tests')
-            archive = self.download_file(self.pagesets_url, parent_dir=dirs['abs_work_dir'])
+        # need to determine if talos pageset is required to be downloaded
+        if self.config.get('run_local'):
+            # talos initiated locally, get and verify test/suite from cmd line
+            self.talos_path = os.path.dirname(self.talos_json)
+            if '-a' in self.config['talos_extra_options'] or '--activeTests' in self.config['talos_extra_options']:
+                # test name (-a or --activeTests) specified, find out what suite it is a part of
+                self.suite = self.get_suite_from_test()
+            elif '--suite' in self.config['talos_extra_options']:
+                # --suite specified, get suite from cmd line and ensure is valid
+                suite_name_index = self.config['talos_extra_options'].index('--suite') + 1
+                if suite_name_index < len(self.config['talos_extra_options']):
+                    self.suite = self.config['talos_extra_options'][suite_name_index]
+                    self.validate_suite()
+                else:
+                    self.fatal("Suite name not provided")
+        else:
+            # talos initiated in production via mozharness
+            self.suite = self.config['suite']
+
+        # now that have the suite name, check if pageset is required, if so download it
+        if self.query_pagesets_name():
+            self.info("Downloading pageset with tooltool...")
+            self.src_talos_webdir = os.path.join(self.talos_path, 'talos')
+            src_talos_pageset = os.path.join(self.src_talos_webdir, 'tests')
+            manifest_file = os.path.join(self.talos_path, 'tp5n-pageset.manifest')
+            self.tooltool_fetch(
+                manifest_file,
+                output_dir=src_talos_pageset,
+                cache=self.config.get('tooltool_cache')
+            )
+            archive = os.path.join(src_talos_pageset, self.pagesets_name)
             unzip = self.query_exe('unzip')
             unzip_cmd = [unzip, '-q', '-o', archive, '-d', src_talos_pageset]
             self.run_command(unzip_cmd, halt_on_failure=True)
 
     # Action methods. {{{1
     # clobber defined in BaseScript
     # read_buildbot_config defined in BuildbotMixin
 
--- a/testing/talos/mach_commands.py
+++ b/testing/talos/mach_commands.py
@@ -61,16 +61,17 @@ class TalosRunner(MozbuildObject):
                 'virtualenv': [self.python_interp, self.virtualenv_script]
             },
             'title': socket.gethostname(),
             'default_actions': [
                 'populate-webroot',
                 'create-virtualenv',
                 'run-tests',
             ],
+            'download_tooltool': True,
             'talos_extra_options': ['--develop'] + self.talos_args,
         }
 
     def make_args(self):
         self.args = {
             'config': {},
             'initial_config_file': self.config_file_path,
         }
--- a/testing/talos/talos.json
+++ b/testing/talos/talos.json
@@ -40,30 +40,30 @@
             "talos_options": ["--disable-e10s"]
         },
         "other-e10s_l64": {
             "tests": ["a11yr", "ts_paint", "tpaint", "sessionrestore", "sessionrestore_no_auto_restore", "tabpaint"]
         },
         "g1": {
             "tests": ["tp5o_scroll", "glterrain"],
             "talos_options": ["--disable-e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "g1-e10s": {
             "tests": ["tp5o_scroll", "glterrain"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "g2": {
             "tests": ["damp", "tps"],
             "talos_options": ["--disable-e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "g2-e10s": {
             "tests": ["damp", "tps"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "g3": {
             "tests": ["dromaeo_dom"],
             "talos_options": ["--disable-e10s"]
         },
         "g3-e10s": {
             "tests": ["dromaeo_dom"]
         },
@@ -78,35 +78,35 @@
             "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart"],
             "talos_options": ["--disable-e10s"]
         },
         "svgr-e10s": {
             "tests": ["tsvgx", "tsvgr_opacity", "tart", "tscrollx", "cart"]
         },
         "tp5o": {
             "tests": ["tp5o"],
-            "talos_options": ["--disable-e10s"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip",
+            "talos_options": ["--disable-e10s"]
         },
         "tp5o-e10s": {
             "tests": ["tp5o"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip"
+            "pagesets_name": "tp5n.zip"
         },
         "xperf": {
             "tests": ["tp5n"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
+            "pagesets_name": "tp5n.zip",
             "talos_options": [
                 "--disable-e10s",
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
             ]
         },
         "xperf-e10s": {
             "tests": ["tp5n"],
-            "pagesets_url": "http://talos-bundles.pvt.build.mozilla.org/zips/tp5n.zip",
+            "pagesets_name": "tp5n.zip",
             "talos_options": [
                 "--xperf_path",
                 "\"c:/Program Files/Microsoft Windows Performance Toolkit/xperf.exe\""
             ]
         }
     },
     "mobile-suites": {
         "remote-tsvgx": {
new file mode 100644
--- /dev/null
+++ b/testing/talos/tp5n-pageset.manifest
@@ -0,0 +1,9 @@
+[
+    {
+        "filename": "tp5n.zip",
+        "size": 81753769,
+        "digest": "7e74bc532d220fa2484f84bd7c2659da7d2ae3aa0bc225ba63e3db70dc0c0697503427209098afa85e235397c4ec58cd488cab7b3435e8079583d3994fff8326",
+        "algorithm": "sha512",
+        "unpack": false
+    }
+]
\ No newline at end of file