Bug 1183232 - Allow selecting tests and suites with just a tag from mach try. r=ahal,gps
authorChris Manchester <cmanchester@mozilla.com>
Thu, 06 Aug 2015 10:19:31 -0700
changeset 256676 cab252737a1240f3b6c4e248e8c9a2a7884ff36c
parent 256675 96544133a324b9dc3c4cf512982a346dbb656935
child 256677 9126c5fcc788873196c778e089f12b52f4b7d8fc
push id29187
push usercbook@mozilla.com
push dateFri, 07 Aug 2015 11:13:32 +0000
treeherdermozilla-central@3e51753a099f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersahal, gps
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1183232 - Allow selecting tests and suites with just a tag from mach try. r=ahal,gps
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -67,17 +67,18 @@ class TestMetadata(object):
         """Obtain all tests having the specified flavor.
         This is a generator of dicts describing each test.
         for path in sorted(self._tests_by_flavor.get(flavor, [])):
             yield self._tests_by_path[path]
-    def resolve_tests(self, paths=None, flavor=None, subsuite=None, under_path=None):
+    def resolve_tests(self, paths=None, flavor=None, subsuite=None, under_path=None,
+                      tags=None):
         """Resolve tests from an identifier.
         This is a generator of dicts describing each test.
         ``paths`` can be an iterable of values to use to identify tests to run.
         If an entry is a known test file, tests associated with that file are
         returned (there may be multiple configurations for a single file). If
         an entry is a directory, or a prefix of a directory containing tests,
@@ -89,27 +90,36 @@ class TestMetadata(object):
         test's installed dir.
         If ``flavor`` is a string, it will be used to filter returned tests
         to only be the flavor specified. A flavor is something like
         If ``subsuite`` is a string, it will be used to filter returned tests
         to only be in the subsuite specified.
+        If ``tags`` are specified, they will be used to filter returned tests
+        to only those with a matching tag.
+        if tags:
+            tags = set(tags)
         def fltr(tests):
             for test in tests:
                 if flavor:
                    if (flavor == 'devtools' and test.get('flavor') != 'browser-chrome') or \
                       (flavor != 'devtools' and test.get('flavor') != flavor):
                 if subsuite and test.get('subsuite') != subsuite:
+                if tags and not (tags & set(test.get('tags', '').split())):
+                    continue
                 if under_path \
                     and not test['file_relpath'].startswith(under_path):
                 # Make a copy so modifications don't change the source.
                 yield dict(test)
         paths = paths or []
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -382,19 +382,19 @@ class JsapiTestsCommand(MachCommandBase)
         jsapi_tests_result = subprocess.call(jsapi_tests_cmd)
         return jsapi_tests_result
 class PushToTry(MachCommandBase):
-    def validate_args(self, paths, tests, builds, platforms):
-        if not len(paths) and not tests:
-            print("Paths or tests must be specified as an argument to autotry.")
+    def validate_args(self, paths, tests, tags, builds, platforms):
+        if not any([len(paths), tests, tags]):
+            print("Paths, tests, or tags must be specified.")
         if platforms is None:
             platforms = os.environ['AUTOTRY_PLATFORM_HINT']
         for p in paths:
             p = os.path.normpath(os.path.abspath(p))
             if not p.startswith(self.topsrcdir):
@@ -426,59 +426,61 @@ class PushToTry(MachCommandBase):
     @CommandArgument('--tag', dest='tags', action='append',
                      help='Restrict tests to the given tag (may be specified multiple times)')
     @CommandArgument('--no-push', dest='push', action='store_false',
                      help='Do not push to try as a result of running this command (if '
                           'specified this command will only print calculated try '
                           'syntax and selection info).')
     def autotry(self, builds=None, platforms=None, paths=None, verbose=None,
                 extra_tests=None, push=None, tags=None, tests=None):
-        """Autotry is in beta, please file bugs blocking 1149670.
+        """mach try is under development, please file bugs blocking 1149670.
         Pushes the specified tests to try. The simplest way to specify tests is
         by using the -u argument, which will behave as usual for try syntax.
         This command also provides a mechanism to select test jobs and tests
         within a job by path based on tests present in the tree under that
         path. Mochitests, xpcshell tests, and reftests are eligible for
         selection by this mechanism. Selected tests will be run in a single
         chunk of the relevant suite, at this time in chunk 1.
         Specifying platforms is still required with the -p argument (a default
         is taken from the AUTOTRY_PLATFORM_HINT environment variable if set).
         Tests may be further filtered by passing one or more --tag to the
-        command.
+        command. If one or more --tag is specified with out paths or -u,
+        tests with the given tags will be run in a single chunk of
+        applicable suites.
         To run suites in addition to those determined from the tree, they
         can be passed to the --extra arguent.
         The command requires either its own mercurial extension ("push-to-try",
         installable from mach mercurial-setup) or a git repo using git-cinnabar
         (available at https://github.com/glandium/git-cinnabar).
         from mozbuild.testing import TestResolver
         from mozbuild.controller.building import BuildDriver
         from autotry import AutoTry
         import pprint
         print("mach try is under development, please file bugs blocking 1149670.")
-        builds, platforms = self.validate_args(paths, tests, builds, platforms)
+        builds, platforms = self.validate_args(paths, tests, tags, builds, platforms)
         resolver = self._spawn(TestResolver)
         at = AutoTry(self.topsrcdir, resolver, self._mach_context)
         if at.find_uncommited_changes():
             print('ERROR please commit changes before continuing')
         driver = self._spawn(BuildDriver)
-        manifests_by_flavor = at.manifests_by_flavor(paths)
+        manifests_by_flavor = at.resolve_manifests(paths=paths, tags=tags)
         if not manifests_by_flavor and not tests:
             print("No tests were found when attempting to resolve paths:\n\n\t%s" %
         all_manifests = set()
         for m in manifests_by_flavor.values():
--- a/testing/tools/autotry/autotry.py
+++ b/testing/tools/autotry/autotry.py
@@ -32,24 +32,23 @@ class AutoTry(object):
         self.resolver = resolver
         self.mach_context = mach_context
         if os.path.exists(os.path.join(self.topsrcdir, '.hg')):
             self._use_git = False
             self._use_git = True
-    def manifests_by_flavor(self, paths):
+    def resolve_manifests(self, paths=None, tags=None):
+        assert tags or paths
+        tests = list(self.resolver.resolve_tests(tags=tags,
+                                                 paths=paths,
+                                                 cwd=self.mach_context.cwd))
         manifests_by_flavor = defaultdict(set)
-        if not paths:
-            return dict(manifests_by_flavor)
-        tests = list(self.resolver.resolve_tests(paths=paths,
-                                                 cwd=self.mach_context.cwd))
         for t in tests:
             if t['flavor'] in AutoTry.test_flavors:
                 flavor = t['flavor']
                 if 'subsuite' in t and t['subsuite'] == 'devtools':
                     flavor = 'devtools-chrome'
                 manifest = os.path.relpath(t['manifest'], self.topsrcdir)