Bug 1509048 - [tryselect] Pass full taskgraph into selectors, r=marco
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Fri, 23 Nov 2018 18:17:33 +0000
changeset 507089 306f3bd4760aa5224173f6b6d9c138a4f856384f
parent 507088 8ab07484e939f28ef182671a0cac6e444efa06bf
child 507090 a95519247ae7f4ec260d0009ee7b5af0841513d5
push id1905
push userffxbld-merge
push dateMon, 21 Jan 2019 12:33:13 +0000
treeherdermozilla-release@c2fca1944d8c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmarco
bugs1509048
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1509048 - [tryselect] Pass full taskgraph into selectors, r=marco Currently selectors that generate the taskgraph receive a list of task labels back. While this works, it isn't very robust and limits the selectors in what they can accomplish. For example, it means we need to use regexes to map test suites to tasks. If we had the full taskgraph, we could use filter functions instead. There are also new selectors in the works which will need this (like ./mach try chooser). Finally the try syntax selector would also need this if we ever migrate it to use the 'try_task_config.json' format. Differential Revision: https://phabricator.services.mozilla.com/D12570
tools/tryselect/selectors/coverage.py
tools/tryselect/selectors/fuzzy.py
tools/tryselect/tasks.py
tools/tryselect/test/setup.sh
--- a/tools/tryselect/selectors/coverage.py
+++ b/tools/tryselect/selectors/coverage.py
@@ -349,17 +349,18 @@ def run_coverage_try(templates={}, full=
     download_coverage_mapping(vcs.base_ref)
 
     changed_sources = vcs.get_outgoing_files()
     test_files, test_chunks = find_tests(changed_sources)
     if not test_files and not test_chunks:
         print('ERROR Could not find any tests or chunks to run.')
         return 1
 
-    all_tasks = generate_tasks(parameters, full, root=build.topsrcdir)
+    tg = generate_tasks(parameters, full, root=build.topsrcdir)
+    all_tasks = tg.tasks.keys()
 
     tasks_by_chunks = filter_tasks_by_chunks(all_tasks, test_chunks)
     tasks_by_path = filter_tasks_by_paths(all_tasks, test_files)
     tasks = filter(is_opt_task, set(tasks_by_path + tasks_by_chunks))
 
     if not tasks:
         print('ERROR Did not find any matching tasks after filtering.')
         return 1
--- a/tools/tryselect/selectors/fuzzy.py
+++ b/tools/tryselect/selectors/fuzzy.py
@@ -1,16 +1,17 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 import os
 import platform
+import re
 import subprocess
 import sys
 from distutils.spawn import find_executable
 
 from mozboot.util import get_state_dir
 from mozterm import Terminal
 from six import string_types
 
@@ -18,16 +19,23 @@ from .. import preset as pset
 from ..cli import BaseTryParser
 from ..tasks import generate_tasks, filter_tasks_by_paths
 from ..push import check_working_directory, push_to_try, vcs
 
 terminal = Terminal()
 
 here = os.path.abspath(os.path.dirname(__file__))
 
+# Some tasks show up in the target task set, but are either special cases
+# or uncommon enough that they should only be selectable with --full.
+TARGET_TASK_FILTERS = (
+    '.*-ccov\/.*',
+)
+
+
 FZF_NOT_FOUND = """
 Could not find the `fzf` binary.
 
 The `mach try fuzzy` command depends on fzf. Please install it following the
 appropriate instructions for your platform:
 
     https://github.com/junegunn/fzf#installation
 
@@ -181,30 +189,38 @@ def run_fzf(cmd, tasks):
     selected = []
     query = None
     if out:
         query = out[0]
         selected = out[1:]
     return query, selected
 
 
+def filter_target_task(task):
+    return not any(re.search(pattern, task) for pattern in TARGET_TASK_FILTERS)
+
+
 def run_fuzzy_try(update=False, query=None, templates=None, full=False, parameters=None,
                   save=False, preset=None, mod_presets=False, push=True, message='{msg}',
                   paths=None, **kwargs):
     if mod_presets:
         return getattr(pset, mod_presets)(section='fuzzy')
 
     fzf = fzf_bootstrap(update)
 
     if not fzf:
         print(FZF_NOT_FOUND)
         return 1
 
     check_working_directory(push)
-    all_tasks = generate_tasks(parameters, full, root=vcs.path)
+    tg = generate_tasks(parameters, full, root=vcs.path)
+    all_tasks = sorted(tg.tasks.keys())
+
+    if not full:
+        all_tasks = filter(filter_target_task, all_tasks)
 
     if paths:
         all_tasks = filter_tasks_by_paths(all_tasks, paths)
         if not all_tasks:
             return 1
 
     key_shortcuts = [k + ':' + v for k, v in fzf_shortcuts.iteritems()]
     base_cmd = [
--- a/tools/tryselect/tasks.py
+++ b/tools/tryselect/tasks.py
@@ -1,87 +1,85 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import absolute_import, print_function, unicode_literals
 
+import glob
 import hashlib
+import json
 import os
 import re
 import shutil
 import sys
 
 from mozboot.util import get_state_dir
 from mozbuild.base import MozbuildObject
 from mozpack.files import FileFinder
 from moztest.resolve import TestResolver, get_suite_definition
 
 import taskgraph
 from taskgraph.generator import TaskGraphGenerator
 from taskgraph.parameters import (
     ParameterMismatch,
     load_parameters_file,
 )
+from taskgraph.taskgraph import TaskGraph
 
 here = os.path.abspath(os.path.dirname(__file__))
 build = MozbuildObject.from_environment(cwd=here)
 
 
 PARAMETER_MISMATCH = """
 ERROR - The parameters being used to generate tasks differ from those defined
 in your working copy:
 
     {}
 
 To fix this, either rebase onto the latest mozilla-central or pass in
 -p/--parameters. For more information on how to define parameters, see:
 https://firefox-source-docs.mozilla.org/taskcluster/taskcluster/mach.html#parameters
 """
 
-# Some tasks show up in the target task set, but are either special cases
-# or uncommon enough that they should only be selectable with --full.
-TARGET_TASK_FILTERS = (
-    '.*-ccov\/.*',
-)
-
 
 def invalidate(cache, root):
     if not os.path.isfile(cache):
         return
 
     tc_dir = os.path.join(root, 'taskcluster')
     tmod = max(os.path.getmtime(os.path.join(tc_dir, p)) for p, _ in FileFinder(tc_dir))
     cmod = os.path.getmtime(cache)
 
     if tmod > cmod:
         os.remove(cache)
 
 
-def filter_target_task(task):
-    return not any(re.search(pattern, task) for pattern in TARGET_TASK_FILTERS)
-
-
 def generate_tasks(params, full, root):
     params = params or "project=mozilla-central"
 
     # Try to delete the old taskgraph cache directory.
     old_cache_dir = os.path.join(get_state_dir()[0], 'cache', 'taskgraph')
     if os.path.isdir(old_cache_dir):
         shutil.rmtree(old_cache_dir)
 
     root_hash = hashlib.sha256(os.path.abspath(root)).hexdigest()
     cache_dir = os.path.join(get_state_dir()[0], 'cache', root_hash, 'taskgraph')
-    attr = 'full_task_set' if full else 'target_task_set'
+
+    # Cleanup old cache files
+    for path in glob.glob(os.path.join(cache_dir, '*_set')):
+        os.remove(path)
+
+    attr = 'full_task_graph' if full else 'target_task_graph'
     cache = os.path.join(cache_dir, attr)
 
     invalidate(cache, root)
     if os.path.isfile(cache):
         with open(cache, 'r') as fh:
-            return fh.read().splitlines()
+            return TaskGraph.from_json(json.load(fh))[1]
 
     if not os.path.isdir(cache_dir):
         os.makedirs(cache_dir)
 
     print("Task configuration changed, generating {}".format(attr.replace('_', ' ')))
     try:
         params = load_parameters_file(params, strict=False, overrides={'try_mode': 'try_select'})
         params.check()
@@ -90,26 +88,22 @@ def generate_tasks(params, full, root):
         sys.exit(1)
 
     taskgraph.fast = True
     cwd = os.getcwd()
     os.chdir(build.topsrcdir)
 
     root = os.path.join(root, 'taskcluster', 'ci')
     tg = getattr(TaskGraphGenerator(root_dir=root, parameters=params), attr)
-    labels = [label for label in tg.graph.visit_postorder()]
-
-    if not full:
-        labels = filter(filter_target_task, labels)
 
     os.chdir(cwd)
 
     with open(cache, 'w') as fh:
-        fh.write('\n'.join(labels))
-    return labels
+        json.dump(tg.to_json(), fh)
+    return tg
 
 
 def filter_tasks_by_paths(tasks, paths):
     resolver = TestResolver.from_environment(cwd=here)
     run_suites, run_tests = resolver.resolve_metadata(paths)
     flavors = set([(t['flavor'], t.get('subsuite')) for t in run_tests])
 
     task_regexes = set()
--- a/tools/tryselect/test/setup.sh
+++ b/tools/tryselect/test/setup.sh
@@ -8,26 +8,86 @@ default=syntax
 EOF
 
 calculate_hash='import hashlib, os, sys
 print hashlib.sha256(os.path.abspath(sys.argv[1])).hexdigest()'
 roothash=$(python -c "$calculate_hash" "$topsrcdir")
 cachedir=$MOZBUILD_STATE_PATH/cache/$roothash/taskgraph
 mkdir -p $cachedir
 
-cat > $cachedir/target_task_set << EOF
-test/foo-opt
-test/foo-debug
-build-baz
+cat > $cachedir/target_task_graph << EOF
+{
+  "test/foo-opt": {
+    "kind": "test",
+    "label": "test/foo-opt",
+    "attributes": {},
+    "task": {},
+    "optimization": {},
+    "dependencies": {}
+  },
+  "test/foo-debug": {
+    "kind": "test",
+    "label": "test/foo-debug",
+    "attributes": {},
+    "task": {},
+    "optimization": {},
+    "dependencies": {}
+  },
+  "build-baz": {
+    "kind": "build",
+    "label": "build-baz",
+    "attributes": {},
+    "task": {},
+    "optimization": {},
+    "dependencies": {}
+  }
+}
 EOF
 
-cat > $cachedir/full_task_set << EOF
-test/foo-opt
-test/foo-debug
-test/bar-opt
-test/bar-debug
-build-baz
+cat > $cachedir/full_task_graph << EOF
+{
+  "test/foo-opt": {
+    "kind": "test",
+    "label": "test/foo-opt",
+    "attributes": {},
+    "task": {},
+    "optimization": {},
+    "dependencies": {}
+  },
+  "test/foo-debug": {
+    "kind": "test",
+    "label": "test/foo-debug",
+    "attributes": {},
+    "task": {},
+    "optimization": {},
+    "dependencies": {}
+  },
+  "test/bar-opt": {
+    "kind": "test",
+    "label": "test/bar-opt",
+    "attributes": {},
+    "task": {},
+    "optimization": {},
+    "dependencies": {}
+  },
+  "test/bar-debug": {
+    "kind": "test",
+    "label": "test/bar-debug",
+    "attributes": {},
+    "task": {},
+    "optimization": {},
+    "dependencies": {}
+  },
+  "build-baz": {
+    "kind": "build",
+    "label": "build-baz",
+    "attributes": {},
+    "task": {},
+    "optimization": {},
+    "dependencies": {}
+  }
+}
 EOF
 
 # set mtime to the future so we don't re-generate tasks
 find $cachedir -type f -exec touch -d "next day" {} +
 
 export testargs="--no-push --no-artifact"