Bug 1531886 - [tryselect] Use consistent 'run' method to kickstart all selectors, r=gbrown
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 06 Mar 2019 17:34:44 +0000
changeset 520558 bee5219e28b23a0108073d70fe9a6854557416c6
parent 520557 730431cf28dbafda9afa0fb3664e2b86ddb6d449
child 520559 ba832d587ae117729d39b1040f1f87e2655b31ac
push id10862
push userffxbld-merge
push dateMon, 11 Mar 2019 13:01:11 +0000
treeherdermozilla-beta@a2e7f5c935da [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1531886
milestone67.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1531886 - [tryselect] Use consistent 'run' method to kickstart all selectors, r=gbrown This allows us to refactor mach_commands.py so we can call self.handle_presets implicitly. This in turn gives us a future place to add shared code and makes adding new selectors easier. Differential Revision: https://phabricator.services.mozilla.com/D22021
tools/tryselect/mach_commands.py
tools/tryselect/selectors/again.py
tools/tryselect/selectors/chooser/__init__.py
tools/tryselect/selectors/coverage.py
tools/tryselect/selectors/empty.py
tools/tryselect/selectors/fuzzy.py
tools/tryselect/selectors/release.py
tools/tryselect/selectors/syntax.py
tools/tryselect/test/test_again.py
--- a/tools/tryselect/mach_commands.py
+++ b/tools/tryselect/mach_commands.py
@@ -142,16 +142,23 @@ class TrySelect(MachCommandBase):
                     defaults[k] = v
                 else:
                     nondefaults[k] = v
 
             kwargs = merge(defaults, preset, nondefaults)
 
         return kwargs
 
+    def run(self, **kwargs):
+        if 'preset' in self.parser.common_groups:
+            kwargs = self.handle_presets(**kwargs)
+
+        mod = importlib.import_module('tryselect.selectors.{}'.format(self.subcommand))
+        return mod.run(**kwargs)
+
     @Command('try',
              category='ci',
              description='Push selected tasks to the try server',
              parser=generic_parser)
     def try_default(self, argv=None, **kwargs):
         """Push selected tests to the try server.
 
         The |mach try| command is a frontend for scheduling tasks to
@@ -214,25 +221,25 @@ class TrySelect(MachCommandBase):
           word$: exact suffix match (line must end with literal "word")
           !word: exact negation match (line must not contain literal "word")
           'a | 'b: OR operator (joins two exact match operators together)
 
         For example:
 
           ^start 'exact | !ignore fuzzy end$
         """
-        from tryselect.selectors.fuzzy import run_fuzzy_try
         if kwargs.get('save') and not kwargs.get('query'):
             # If saving preset without -q/--query, allow user to use the
             # interface to build the query.
             kwargs_copy = kwargs.copy()
             kwargs_copy['push'] = False
-            kwargs['query'] = run_fuzzy_try(**kwargs_copy)
+            kwargs_copy['save'] = None
+            kwargs['query'] = self.run(save_query=True, **kwargs_copy)
 
-        return run_fuzzy_try(**self.handle_presets(**kwargs))
+        return self.run(**kwargs)
 
     @SubCommand('try',
                 'chooser',
                 description='Schedule tasks by selecting them from a web '
                             'interface.',
                 parser=get_parser('chooser'))
     def try_chooser(self, **kwargs):
         """Push tasks selected from a web interface to try.
@@ -240,43 +247,40 @@ class TrySelect(MachCommandBase):
         This selector will build the taskgraph and spin up a dynamically
         created 'trychooser-like' web-page on the localhost. After a selection
         has been made, pressing the 'Push' button will automatically push the
         selection to try.
         """
         self._activate_virtualenv()
         self.virtualenv_manager.install_pip_package('flask')
 
-        from tryselect.selectors.chooser import run_try_chooser
-        return run_try_chooser(**kwargs)
+        return self.run(**kwargs)
 
     @SubCommand('try',
                 'again',
                 description='Schedule a previously generated (non try syntax) '
                             'push again.',
                 parser=get_parser('again'))
     def try_again(self, **kwargs):
-        from tryselect.selectors.again import run_try_again
-        return run_try_again(**kwargs)
+        return self.run(**kwargs)
 
     @SubCommand('try',
                 'empty',
                 description='Push to try without scheduling any tasks.',
                 parser=get_parser('empty'))
     def try_empty(self, **kwargs):
         """Push to try, running no builds or tests
 
         This selector does not prompt you to run anything, it just pushes
         your patches to try, running no builds or tests by default. After
         the push finishes, you can manually add desired jobs to your push
         via Treeherder's Add New Jobs feature, located in the per-push
         menu.
         """
-        from tryselect.selectors.empty import run_empty_try
-        return run_empty_try(**kwargs)
+        return self.run(**kwargs)
 
     @SubCommand('try',
                 'syntax',
                 description='Select tasks on try using try syntax',
                 parser=get_parser('syntax'))
     def try_syntax(self, **kwargs):
         """Push the current tree to try, with the specified syntax.
 
@@ -310,47 +314,41 @@ class TrySelect(MachCommandBase):
         scheduled. If no platform is selected a default is taken from
         the AUTOTRY_PLATFORM_HINT environment variable, if set.
 
         The command requires either its own mercurial extension ("push-to-try",
         installable from mach vcs-setup) or a git repo using git-cinnabar
         (installable from mach vcs-setup --git).
 
         """
-        from tryselect.selectors.syntax import AutoTry
-
-        kwargs = self.handle_presets(**kwargs)
         try:
             if self.substs.get("MOZ_ARTIFACT_BUILDS"):
                 kwargs['local_artifact_build'] = True
         except BuildEnvironmentNotFoundException:
             # If we don't have a build locally, we can't tell whether
             # an artifact build is desired, but we still want the
             # command to succeed, if possible.
             pass
 
         config_status = os.path.join(self.topobjdir, 'config.status')
         if (kwargs['paths'] or kwargs['tags']) and not config_status:
             print(CONFIG_ENVIRONMENT_NOT_FOUND)
             sys.exit(1)
 
-        at = AutoTry(self.topsrcdir, self._mach_context)
-        return at.run(**kwargs)
+        return self.run(**kwargs)
 
     @SubCommand('try',
                 'coverage',
                 description='Select tasks on try using coverage data',
                 parser=get_parser('coverage'))
     def try_coverage(self, **kwargs):
         """Select which tasks to use using coverage data.
         """
-        from tryselect.selectors.coverage import run_coverage_try
-        return run_coverage_try(**kwargs)
+        return self.run(**kwargs)
 
     @SubCommand('try',
                 'release',
                 description='Push the current tree to try, configured for a staging release.',
                 parser=get_parser('release'))
     def try_release(self, **kwargs):
         """Push the current tree to try, configured for a staging release.
         """
-        from tryselect.selectors.release import run_try_release
-        return run_try_release(**kwargs)
+        return self.run(**kwargs)
--- a/tools/tryselect/selectors/again.py
+++ b/tools/tryselect/selectors/again.py
@@ -60,17 +60,17 @@ def migrate_old_history():
             os.makedirs(history_dir)
 
         shutil.move(path, history_path)
 
     if os.path.isdir(old_history_dir):
         shutil.rmtree(old_history_dir)
 
 
-def run_try_again(index=0, purge=False, list_configs=False, message='{msg}', **pushargs):
+def run(index=0, purge=False, list_configs=False, message='{msg}', **pushargs):
     # TODO: Remove after January 1st, 2020.
     migrate_old_history()
 
     if purge:
         os.remove(history_path)
         return
 
     if not os.path.isfile(history_path):
--- a/tools/tryselect/selectors/chooser/__init__.py
+++ b/tools/tryselect/selectors/chooser/__init__.py
@@ -17,19 +17,19 @@ here = os.path.abspath(os.path.dirname(_
 
 class ChooserParser(BaseTryParser):
     name = 'chooser'
     arguments = []
     common_groups = ['push', 'task']
     templates = ['artifact', 'env', 'rebuild', 'chemspill-prio', 'gecko-profile']
 
 
-def run_try_chooser(update=False, query=None, templates=None, full=False, parameters=None,
-                    save=False, preset=None, mod_presets=False, push=True, message='{msg}',
-                    **kwargs):
+def run(update=False, query=None, templates=None, full=False, parameters=None,
+        save=False, preset=None, mod_presets=False, push=True, message='{msg}',
+        **kwargs):
     from .app import create_application
     check_working_directory(push)
 
     tg = generate_tasks(parameters, full, root=vcs.path)
     app = create_application(tg)
 
     if os.environ.get('WERKZEUG_RUN_MAIN') == 'true':
         # we are in the reloader process, don't open the browser or do any try stuff
--- a/tools/tryselect/selectors/coverage.py
+++ b/tools/tryselect/selectors/coverage.py
@@ -338,19 +338,17 @@ def filter_tasks_by_chunks(tasks, chunks
 
 def is_opt_task(task):
     '''True if the task runs on a supported platform and build type combination.
     This is used to remove -ccov/asan/pgo tasks, along with all /debug tasks.
     '''
     return any(platform in task for platform in OPT_TASK_PATTERNS)
 
 
-def run_coverage_try(templates={}, full=False, parameters=None,
-                     push=True, message='{msg}', **kwargs):
-
+def run(templates={}, full=False, parameters=None, push=True, message='{msg}', **kwargs):
     download_coverage_mapping(vcs.base_ref)
 
     changed_sources = vcs.get_outgoing_files()
     test_files, test_chunks = find_tests(changed_sources)
     if not test_files and not test_chunks:
         print('ERROR Could not find any tests or chunks to run.')
         return 1
 
--- a/tools/tryselect/selectors/empty.py
+++ b/tools/tryselect/selectors/empty.py
@@ -8,12 +8,12 @@ from ..cli import BaseTryParser
 from ..push import push_to_try
 
 
 class EmptyParser(BaseTryParser):
     name = 'empty'
     common_groups = ['push']
 
 
-def run_empty_try(message='{msg}', push=True, **kwargs):
+def run(message='{msg}', push=True, **kwargs):
     msg = 'No try selector specified, use "Add New Jobs" to select tasks.'
     return push_to_try('empty', message.format(msg=msg), [], push=push,
                        closed_tree=kwargs["closed_tree"])
--- a/tools/tryselect/selectors/fuzzy.py
+++ b/tools/tryselect/selectors/fuzzy.py
@@ -103,28 +103,28 @@ class FuzzyParser(BaseTryParser):
           'default': False,
           'help': "Update fzf before running.",
           }],
     ]
     common_groups = ['push', 'task', 'preset']
     templates = ['artifact', 'path', 'env', 'rebuild', 'chemspill-prio', 'gecko-profile']
 
 
-def run(cmd, cwd=None):
+def run_cmd(cmd, cwd=None):
     is_win = platform.system() == 'Windows'
     return subprocess.call(cmd, cwd=cwd, shell=True if is_win else False)
 
 
 def run_fzf_install_script(fzf_path):
     if platform.system() == 'Windows':
         cmd = ['bash', '-c', './install --bin']
     else:
         cmd = ['./install', '--bin']
 
-    if run(cmd, cwd=fzf_path):
+    if run_cmd(cmd, cwd=fzf_path):
         print(FZF_INSTALL_FAILED)
         sys.exit(1)
 
 
 def fzf_bootstrap(update=False):
     """Bootstrap fzf if necessary and return path to the executable.
 
     The bootstrap works by cloning the fzf repository and running the included
@@ -139,17 +139,17 @@ def fzf_bootstrap(update=False):
     if update and not os.path.isdir(fzf_path):
         print("fzf installed somewhere other than {}, please update manually".format(fzf_path))
         sys.exit(1)
 
     def get_fzf():
         return find_executable('fzf', os.path.join(fzf_path, 'bin'))
 
     if update:
-        ret = run(['git', 'pull'], cwd=fzf_path)
+        ret = run_cmd(['git', 'pull'], cwd=fzf_path)
         if ret:
             print("Update fzf failed.")
             sys.exit(1)
 
         run_fzf_install_script(fzf_path)
         return get_fzf()
 
     if os.path.isdir(fzf_path):
@@ -198,18 +198,18 @@ def run_fzf(cmd, tasks):
         selected = out[1:]
     return query, selected
 
 
 def filter_target_task(task):
     return not any(re.search(pattern, task) for pattern in TARGET_TASK_FILTERS)
 
 
-def run_fuzzy_try(update=False, query=None, templates=None, full=False, parameters=None,
-                  save=False, push=True, message='{msg}', paths=None, **kwargs):
+def run(update=False, query=None, templates=None, full=False, parameters=None, save_query=False,
+        push=True, message='{msg}', paths=None, **kwargs):
     fzf = fzf_bootstrap(update)
 
     if not fzf:
         print(FZF_NOT_FOUND)
         return 1
 
     check_working_directory(push)
     tg = generate_tasks(parameters, full, root=vcs.path)
@@ -256,17 +256,17 @@ def run_fuzzy_try(update=False, query=No
         if tasks:
             queries.append(query)
             selected.update(tasks)
 
     if not selected:
         print("no tasks selected")
         return
 
-    if save:
+    if save_query:
         return queries
 
     # build commit message
     msg = "Fuzzy"
     args = []
     if paths:
         args.append("paths={}".format(':'.join(paths)))
     if query:
--- a/tools/tryselect/selectors/release.py
+++ b/tools/tryselect/selectors/release.py
@@ -54,21 +54,17 @@ class ReleaseParser(BaseTryParser):
     ]
     common_groups = ['push']
 
     def __init__(self, *args, **kwargs):
         super(ReleaseParser, self).__init__(*args, **kwargs)
         self.set_defaults(migrations=[])
 
 
-def run_try_release(
-    version, migrations, limit_locales, tasks,
-    push=True, message='{msg}', **kwargs
-):
-
+def run(version, migrations, limit_locales, tasks, push=True, message='{msg}', **kwargs):
     app_version = attr.evolve(version, beta_number=None, is_esr=False)
 
     files_to_change = {
         'browser/config/version.txt': '{}\n'.format(app_version),
         'browser/config/version_display.txt': '{}\n'.format(version),
         'config/milestone.txt': '{}\n'.format(app_version),
     }
 
--- a/tools/tryselect/selectors/syntax.py
+++ b/tools/tryselect/selectors/syntax.py
@@ -8,17 +8,17 @@ import os
 import re
 import sys
 from collections import defaultdict
 
 import mozpack.path as mozpath
 from moztest.resolve import TestResolver
 
 from ..cli import BaseTryParser
-from ..push import push_to_try
+from ..push import build, push_to_try
 
 here = os.path.abspath(os.path.dirname(__file__))
 
 
 class SyntaxParser(BaseTryParser):
     name = 'syntax'
     arguments = [
         [['paths'],
@@ -305,20 +305,19 @@ class AutoTry(object):
         "marionette-e10s",
         "mochitests",
         "reftest",
         "robocop",
         "web-platform-tests",
         "xpcshell",
     ]
 
-    def __init__(self, topsrcdir, mach_context):
-        self.topsrcdir = topsrcdir
+    def __init__(self):
+        self.topsrcdir = build.topsrcdir
         self._resolver = None
-        self.mach_context = mach_context
 
     @property
     def resolver(self):
         if self._resolver is None:
             self._resolver = TestResolver.from_environment(cwd=here)
         return self._resolver
 
     @classmethod
@@ -601,8 +600,13 @@ class AutoTry(object):
             for flavor, paths in paths_by_flavor.iteritems():
                 print("%s: %s" % (flavor, ",".join(paths)))
 
         if kwargs["verbose"]:
             print('The following try syntax was calculated:\n%s' % msg)
 
         push_to_try('syntax', kwargs["message"].format(msg=msg), push=kwargs['push'],
                     closed_tree=kwargs["closed_tree"])
+
+
+def run(**kwargs):
+    at = AutoTry()
+    return at.run(**kwargs)
--- a/tools/tryselect/test/test_again.py
+++ b/tools/tryselect/test/test_again.py
@@ -27,17 +27,17 @@ def test_try_again(monkeypatch):
         assert len(fh.readlines()) == 1
 
     def fake_push_to_try(*args, **kwargs):
         return args, kwargs
 
     monkeypatch.setattr(push, 'push_to_try', fake_push_to_try)
     reload(again)
 
-    args, kwargs = again.run_try_again()
+    args, kwargs = again.run()
 
     assert args[0] == 'again'
     assert args[1] == 'Fuzzy message'
 
     try_task_config = kwargs.pop('try_task_config')
     assert sorted(try_task_config.get('tasks')) == sorted(['foo', 'bar'])
     assert try_task_config.get('templates') == {
         'artifact': True,
@@ -48,13 +48,13 @@ def test_try_again(monkeypatch):
         assert len(fh.readlines()) == 1
 
 
 def test_no_push_does_not_generate_history(tmpdir):
     assert not os.path.isfile(push.history_path)
 
     push.push_to_try('fuzzy', 'Fuzzy', ['foo', 'bar'], {'artifact': True}, push=False)
     assert not os.path.isfile(push.history_path)
-    assert again.run_try_again() == 1
+    assert again.run() == 1
 
 
 if __name__ == '__main__':
     mozunit.main()