Bug 1184405 - Use file metadata from files changed in the current branch in mach try when no other arguments are present. r=jgraham
authorChris Manchester <cmanchester@mozilla.com>
Wed, 30 Sep 2015 16:38:20 -0700
changeset 298828 fd0373aafc35148a3bfd46f2a51cfc539c0a7194
parent 298827 d8494bbabea767e50ec326c48847e252945f4dbd
child 298829 a50cbda36a49c011f672bb59d1e15df21101cfd8
push id5392
push userraliiev@mozilla.com
push dateMon, 14 Dec 2015 20:08:23 +0000
treeherdermozilla-beta@16ce8562a975 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgraham
bugs1184405
milestone44.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1184405 - Use file metadata from files changed in the current branch in mach try when no other arguments are present. r=jgraham
testing/mach_commands.py
testing/tools/autotry/autotry.py
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -570,22 +570,25 @@ class PushToTry(MachCommandBase):
             if defaults is None:
                 print("No saved configuration called %s found in autotry.ini" % kwargs["load"],
                       file=sys.stderr)
 
             for key, value in kwargs.iteritems():
                 if value in (None, []) and key in defaults:
                     kwargs[key] = defaults[key]
 
-        builds, platforms, tests, talos, paths, tags, extra_args = self.validate_args(**kwargs)
-
         if kwargs["push"] and at.find_uncommited_changes():
             print('ERROR please commit changes before continuing')
             sys.exit(1)
 
+        if not any(kwargs[item] for item in ("paths", "tests", "tags")):
+            kwargs["paths"], kwargs["tags"] = at.find_paths_and_tags(kwargs["verbose"])
+
+        builds, platforms, tests, talos, paths, tags, extra_args = self.validate_args(**kwargs)
+
         if paths or tags:
             driver = self._spawn(BuildDriver)
             driver.install_tests(remove=False)
 
             paths = [os.path.relpath(os.path.normpath(os.path.abspath(item)), self.topsrcdir)
                      for item in paths]
             paths_by_flavor = at.paths_by_flavor(paths=paths, tags=tags)
 
--- a/testing/tools/autotry/autotry.py
+++ b/testing/tools/autotry/autotry.py
@@ -223,27 +223,50 @@ class AutoTry(object):
                                                  tags=tags))
 
         for t in tests:
             if t['flavor'] in self.flavor_suites:
                 flavor = t['flavor']
                 if 'subsuite' in t and t['subsuite'] == 'devtools':
                     flavor = 'devtools-chrome'
 
-                for path in paths:
-                    if flavor in ["crashtest", "reftest"]:
-                        manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
-                        if manifest_relpath.startswith(path):
-                            paths_by_flavor[flavor].add(manifest_relpath)
-                    else:
-                        if t['file_relpath'].startswith(path):
-                            paths_by_flavor[flavor].add(path)
+                if flavor in ['crashtest', 'reftest']:
+                    manifest_relpath = os.path.relpath(t['manifest'], self.topsrcdir)
+                    paths_by_flavor[flavor].add(os.path.dirname(manifest_relpath))
+                elif 'dir_relpath' in t:
+                    paths_by_flavor[flavor].add(t['dir_relpath'])
+                else:
+                    file_relpath = os.path.relpath(t['path'], self.topsrcdir)
+                    dir_relpath = os.path.dirname(file_relpath)
+                    paths_by_flavor[flavor].add(dir_relpath)
+
+        for flavor, path_set in paths_by_flavor.items():
+            paths_by_flavor[flavor] = self.deduplicate_prefixes(path_set, paths)
 
         return dict(paths_by_flavor)
 
+    def deduplicate_prefixes(self, path_set, input_paths):
+        # Removes paths redundant to test selection in the given path set.
+        # If a path was passed on the commandline that is the prefix of a
+        # path in our set, we only need to include the specified prefix to
+        # run the intended tests (every test in "layout/base" will run if
+        # "layout" is passed to the reftest harness).
+        removals = set()
+        additions = set()
+
+        for path in path_set:
+            full_path = path
+            while path:
+                path, _ = os.path.split(path)
+                if path in input_paths:
+                    removals.add(full_path)
+                    additions.add(path)
+
+        return additions | (path_set - removals)
+
     def remove_duplicates(self, paths_by_flavor, tests):
         rv = {}
         for item in paths_by_flavor:
             if self.flavor_suites[item] not in tests:
                 rv[item] = paths_by_flavor[item].copy()
         return rv
 
     def calc_try_syntax(self, platforms, tests, talos, builds, paths_by_flavor, tags,
@@ -322,8 +345,60 @@ class AutoTry(object):
         if self._use_git:
             stat = subprocess.check_output(['git', 'status', '-z'])
             return any(len(entry.strip()) and entry.strip()[0] in ('A', 'M', 'D')
                        for entry in stat.split('\0'))
         else:
             stat = subprocess.check_output(['hg', 'status'])
             return any(len(entry.strip()) and entry.strip()[0] in ('A', 'M', 'R')
                        for entry in stat.splitlines())
+
+    def find_paths_and_tags(self, verbose):
+        paths, tags = set(), set()
+        changed_files = self.find_changed_files()
+        if changed_files:
+            if verbose:
+                print("Pushing tests based on modifications to the "
+                      "following files:\n\t%s" % "\n\t".join(changed_files))
+
+            from mozbuild.frontend.reader import (
+                BuildReader,
+                EmptyConfig,
+            )
+
+            config = EmptyConfig(self.topsrcdir)
+            reader = BuildReader(config)
+            files_info = reader.files_info(changed_files)
+
+            for path, info in files_info.items():
+                paths |= info.test_files
+                tags |= info.test_tags
+
+            if verbose:
+                if paths:
+                    print("Pushing tests based on the following patterns:\n\t%s" %
+                          "\n\t".join(paths))
+                if tags:
+                    print("Pushing tests based on the following tags:\n\t%s" %
+                          "\n\t".join(tags))
+        return paths, tags
+
+
+    def find_changed_files(self):
+        """Finds files changed in a local source tree (hg only for now)."""
+        if self._use_git:
+            # Getting changed files on the current branch with rev-list and contains
+            # will not work: subsequent commits on mozilla-central frequently have
+            # non-increasing dates, breaking both.
+            # (see http://thread.gmane.org/gmane.comp.version-control.git/269560/)
+            # Git support will be added in bug 1203686.
+            return []
+
+        hg_args = [
+            'hg', 'log', '-r',
+            # Include everything from the current commit to the last
+            # public ancestor.
+            '::. and not public()',
+            '--template',
+            '{join(files, "\n")}\n',
+        ]
+
+        return subprocess.check_output(hg_args).splitlines()