Bug 1149670 - Add a mach command to find tests in specified directories and prepare a commit to push them to try.;r=ahal
authorChris Manchester <cmanchester@mozilla.com>
Thu, 28 May 2015 15:57:21 -0700
changeset 247775 ec493d9f59965561d949bc0ab95712317829df37
parent 247774 f9473f4529e42d2aecdb7b20a1e2185e6d78c6db
child 247776 17c8c58041f4645584ead0c2170a21a2b4a3b698
push id60804
push usercmanchester@mozilla.com
push dateWed, 10 Jun 2015 00:37:08 +0000
treeherdermozilla-inbound@ec493d9f5996 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal
bugs1149670
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1149670 - Add a mach command to find tests in specified directories and prepare a commit to push them to try.;r=ahal
build/mach_bootstrap.py
testing/mach_commands.py
testing/tools/autotry/__init__.py
testing/tools/autotry/autotry.py
--- a/build/mach_bootstrap.py
+++ b/build/mach_bootstrap.py
@@ -42,16 +42,17 @@ SEARCH_PATHS = [
     'build/pymake',
     'config',
     'dom/bindings',
     'dom/bindings/parser',
     'layout/tools/reftest',
     'other-licenses/ply',
     'xpcom/idl-parser',
     'testing',
+    'testing/tools/autotry',
     'testing/taskcluster',
     'testing/xpcshell',
     'testing/web-platform',
     'testing/web-platform/harness',
     'testing/marionette/client',
     'testing/marionette/client/marionette/runner/mixins/browsermob-proxy-py',
     'testing/marionette/transport',
     'testing/marionette/driver',
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -1,23 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import print_function, unicode_literals
 
 import os
+import pprint
 import sys
 
 from mach.decorators import (
     CommandArgument,
     CommandProvider,
     Command,
 )
 
+from autotry import AutoTry
 from mozbuild.base import MachCommandBase
 
 
 UNKNOWN_TEST = '''
 I was unable to find tests in the argument(s) given.
 
 You need to specify a test directory, filename, test suite name, or
 abbreviation.
@@ -355,8 +357,117 @@ class JsapiTestsCommand(MachCommandBase)
         print('running jsapi-tests')
         jsapi_tests_cmd = [os.path.join(self.bindir, executable_name('jsapi-tests'))]
         if params['test_name']:
             jsapi_tests_cmd.append(params['test_name'])
 
         jsapi_tests_result = subprocess.call(jsapi_tests_cmd)
 
         return jsapi_tests_result
+
+
+AUTOTRY_HELP_MSG = """
+Autotry is in beta, please file bugs blocking 1149670.
+
+Push test from the specified paths to try. A set of test
+jobs will be selected based on the tests present in the tree, however
+specifying platforms is still required with the -p argument (a default
+is taken from the AUTOTRY_PLATFORM_HINT environment variable if set).
+
+The -u argument may be used to specify additional unittest suites to run.
+
+Selected tests will be run in a single chunk of the relevant suite, at this
+time in chunk 1.
+
+The following types of tests are eligible to be selected automatically
+by this command at this time: %s
+""" % list(AutoTry.test_flavors)
+
+@CommandProvider
+class PushToTry(MachCommandBase):
+
+    def validate_args(self, paths, tests, builds, platforms):
+        if not len(paths) and not tests:
+            print("Paths or tests must be specified as an argument to autotry.")
+            sys.exit(1)
+
+        if platforms is None:
+            platforms = os.environ['AUTOTRY_PLATFORM_HINT']
+
+        for p in paths:
+            p = os.path.normpath(os.path.abspath(p))
+            if not p.startswith(self.topsrcdir):
+                print('Specified path "%s" is outside of the srcdir, unable to'
+                      ' specify tests outside of the srcdir' % p)
+                sys.exit(1)
+            if len(p) <= len(self.topsrcdir):
+                print('Specified path "%s" is at the top of the srcdir and would'
+                      ' select all tests.' % p)
+                sys.exit(1)
+
+        return builds, platforms
+
+    @Command('try', category='testing', description=AUTOTRY_HELP_MSG)
+    @CommandArgument('paths', nargs='*', help='Paths to search for tests to run on try.')
+    @CommandArgument('-v', dest='verbose', action='store_true', default=True,
+                     help='Print detailed information about the resulting test selection '
+                          'and commands performed.')
+    @CommandArgument('-p', dest='platforms', required='AUTOTRY_PLATFORM_HINT' not in os.environ,
+                     help='Platforms to run. (required if not found in the environment)')
+    @CommandArgument('-u', dest='tests',
+                     help='Test jobs to run. These will be use in place of test jobs '
+                          'determined by test paths, if any.')
+    @CommandArgument('--extra', dest='extra_tests',
+                     help='Additional tests to run. These will be added to test jobs '
+                          'determined by test paths, if any.')
+    @CommandArgument('-b', dest='builds', default='do',
+                     help='Build types to run (d for debug, o for optimized)')
+    @CommandArgument('--tag', dest='tags', action='append',
+                     help='Restrict tests to the given tag (may be specified multiple times)')
+    @CommandArgument('--no-push', dest='push', action='store_false',
+                     help='Do not push to try as a result of running this command (if '
+                          'specified this command will only print calculated try '
+                          'syntax and selection info).')
+    def autotry(self, builds=None, platforms=None, paths=None, verbose=None, extra_tests=None,
+                push=None, tags=None, tests=None):
+
+        from mozbuild.testing import TestResolver
+        from mozbuild.controller.building import BuildDriver
+
+        print("mach try is under development, please file bugs blocking 1149670.")
+
+        builds, platforms = self.validate_args(paths, tests, builds, platforms)
+        resolver = self._spawn(TestResolver)
+
+        at = AutoTry(self.topsrcdir, resolver, self._mach_context)
+        if at.find_uncommited_changes():
+            print('ERROR please commit changes before continuing')
+            sys.exit(1)
+
+        driver = self._spawn(BuildDriver)
+        driver.install_tests(remove=False)
+
+        manifests_by_flavor = at.manifests_by_flavor(paths)
+
+        if not manifests_by_flavor and not tests:
+            print("No tests were found when attempting to resolve paths:\n\n\t%s" %
+                  paths)
+            sys.exit(1)
+
+        all_manifests = set()
+        for m in manifests_by_flavor.values():
+            all_manifests |= m
+        all_manifests = list(all_manifests)
+
+        msg = at.calc_try_syntax(platforms, manifests_by_flavor.keys(), tests,
+                                 extra_tests, builds, all_manifests, tags)
+
+        if verbose:
+            print('Tests from the following manifests will be selected: ')
+            pprint.pprint(manifests_by_flavor)
+
+        if verbose:
+            print('The following try message was calculated:\n\n\t%s\n' % msg)
+
+        if push:
+            at.push_to_try(msg, verbose)
+
+        return
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/testing/tools/autotry/autotry.py
@@ -0,0 +1,134 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import sys
+import os
+import itertools
+import subprocess
+import which
+
+from collections import defaultdict
+
+
+TRY_SYNTAX_TMPL = """
+try: -b %s -p %s -u %s -t none %s %s
+"""
+
+class AutoTry(object):
+
+    test_flavors = [
+        'browser-chrome',
+        'chrome',
+        'devtools-chrome',
+        'mochitest',
+        'xpcshell',
+        'reftest',
+        'crashtest',
+    ]
+
+    def __init__(self, topsrcdir, resolver, mach_context):
+        self.topsrcdir = topsrcdir
+        self.resolver = resolver
+        self.mach_context = mach_context
+
+        if os.path.exists(os.path.join(self.topsrcdir, '.hg')):
+            self._use_git = False
+        else:
+            self._use_git = True
+
+    def manifests_by_flavor(self, paths):
+        manifests_by_flavor = defaultdict(set)
+
+        if not paths:
+            return dict(manifests_by_flavor)
+
+        tests = list(self.resolver.resolve_tests(paths=paths,
+                                                 cwd=self.mach_context.cwd))
+        for t in tests:
+            if t['flavor'] in AutoTry.test_flavors:
+                flavor = t['flavor']
+                if 'subsuite' in t and t['subsuite'] == 'devtools':
+                    flavor = 'devtools-chrome'
+                manifest = os.path.relpath(t['manifest'], self.topsrcdir)
+                manifests_by_flavor[flavor].add(manifest)
+
+        return dict(manifests_by_flavor)
+
+    def calc_try_syntax(self, platforms, flavors, tests, extra_tests, builds,
+                        manifests, tags):
+
+        # Maps from flavors to the try syntax snippets implied by that flavor.
+        # TODO: put selected tests under their own builder/label to avoid
+        # confusion reading results on treeherder.
+        flavor_suites = {
+            'mochitest': ['mochitest-1', 'mochitest-e10s-1'],
+            'xpcshell': ['xpcshell'],
+            'chrome': ['mochitest-o'],
+            'browser-chrome': ['mochitest-browser-chrome-1',
+                               'mochitest-e10s-browser-chrome-1'],
+            'devtools-chrome': ['mochitest-dt',
+                                'mochitest-e10s-devtools-chrome'],
+            'crashtest': ['crashtest', 'crashtest-e10s'],
+            'reftest': ['reftest', 'reftest-e10s'],
+        }
+
+        if tags:
+            tags = ' '.join('--tag %s' % t for t in tags)
+        else:
+            tags = ''
+
+        if not tests:
+            tests = ','.join(itertools.chain(*(flavor_suites[f] for f in flavors)))
+            if extra_tests:
+                tests += ',%s' % (extra_tests)
+
+        manifests = ' '.join(manifests)
+        if manifests:
+            manifests = '--try-test-paths %s' % manifests
+        return TRY_SYNTAX_TMPL % (builds, platforms, tests, manifests, tags)
+
+    def _run_git(self, *args):
+        args = ['git'] + list(args)
+        ret = subprocess.call(args)
+        if ret:
+            print('ERROR git command %s returned %s' %
+                  (args, ret))
+            sys.exit(1)
+
+    def _git_push_to_try(self, msg):
+        self._run_git('commit', '--allow-empty', '-m', msg)
+        self._run_git('push', 'hg::ssh://hg.mozilla.org/try',
+                      '+HEAD:refs/heads/branches/default/tip')
+        self._run_git('reset', 'HEAD~')
+
+    def push_to_try(self, msg, verbose):
+        if not self._use_git:
+            try:
+                hg_args = ['hg', 'push-to-try', '-m', msg]
+                subprocess.check_call(hg_args, stderr=subprocess.STDOUT)
+            except subprocess.CalledProcessError as e:
+                print('ERROR hg command %s returned %s' % (hg_args, e.returncode))
+                print('The "push-to-try" hg extension is required to push from '
+                      'hg to try with the autotry command.\n\nIt can be installed '
+                      'by running ./mach mercurial-setup')
+                sys.exit(1)
+        else:
+            try:
+                which.which('git-cinnabar')
+                self._git_push_to_try(msg)
+            except which.WhichError:
+                print('ERROR git-cinnabar is required to push from git to try with'
+                      'the autotry command.\n\nMore information can by found at '
+                      'https://github.com/glandium/git-cinnabar')
+                sys.exit(1)
+
+    def find_uncommited_changes(self):
+        if self._use_git:
+            stat = subprocess.check_output(['git', 'status', '-z'])
+            return any(len(entry.strip()) and entry.strip()[0] in ('A', 'M', 'D')
+                       for entry in stat.split('\0'))
+        else:
+            stat = subprocess.check_output(['hg', 'status'])
+            return any(len(entry.strip()) and entry.strip()[0] in ('A', 'M', 'R')
+                       for entry in stat.splitlines())