Bug 1387135 - Add an 'artifact' try_task_config template to schedule artifact builds, r?dustin draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Tue, 15 Aug 2017 11:37:40 -0400
changeset 647699 845ccd6227ac83f360ccf2561c6c36b8c06904f6
parent 647698 c8d3ac4b2c55a177eb98b19b41b8b96b85e819a2
child 647700 08b48d1d0d5605f843e9fc71c8c958b59b41b6c5
push id74515
push userahalberstadt@mozilla.com
push dateWed, 16 Aug 2017 20:27:32 +0000
reviewersdustin
bugs1387135
milestone57.0a1
Bug 1387135 - Add an 'artifact' try_task_config template to schedule artifact builds, r?dustin This will change all build symbols to 'Ba' and set the USE_ARTIFACTS=1 environment variable. Mozharness will detect this env to decide whether to perform an artifact build or not. MozReview-Commit-ID: J8HVZzOt4mX
taskcluster/taskgraph/templates/artifact.yml
taskcluster/taskgraph/test/test_morph.py
testing/mozharness/scripts/fx_desktop_build.py
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/templates/artifact.yml
@@ -0,0 +1,22 @@
+---
+$if: task.tags['kind'] == "build"
+then:
+    $merge:
+        - $eval: task
+        - extra:
+              $merge:
+                  - $eval: task.extra
+                  - treeherder:
+                        $merge:
+                            - $eval: task.extra.treeherder
+                            - symbol: Ba
+        - payload:
+              $merge:
+                  - $eval: task.payload
+                  - env:
+                        $merge:
+                            - $eval: task.payload.env
+                            - USE_ARTIFACT:
+                                  $eval: input.artifact.enabled
+else:
+    $eval: task
--- a/taskcluster/taskgraph/test/test_morph.py
+++ b/taskcluster/taskgraph/test/test_morph.py
@@ -9,17 +9,28 @@ import unittest
 from taskgraph import morph
 from taskgraph.graph import Graph
 from taskgraph.taskgraph import TaskGraph
 from taskgraph.task import Task
 
 from mozunit import main
 
 
-class TestIndexTask(unittest.TestCase):
+class MorphTestCase(unittest.TestCase):
+
+    def make_taskgraph(self, tasks):
+        label_to_taskid = {k: k + '-tid' for k in tasks}
+        for label, task_id in label_to_taskid.iteritems():
+            tasks[label].task_id = task_id
+        graph = Graph(nodes=set(tasks), edges=set())
+        taskgraph = TaskGraph(tasks, graph)
+        return taskgraph, label_to_taskid
+
+
+class TestIndexTask(MorphTestCase):
 
     def test_make_index_tasks(self):
         task_def = {
             'routes': [
                 "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.es-MX",
                 "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.fy-NL",
                 "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sk",
                 "index.gecko.v2.mozilla-central.latest.firefox-l10n.linux64-opt.sl",
@@ -69,19 +80,69 @@ class TestIndexTask(unittest.TestCase):
 
         self.assertEqual(index_task.task['payload']['command'][0], 'insert-indexes.js')
         self.assertEqual(index_task.task['payload']['env']['TARGET_TASKID'], 'a-tid')
 
         # check the scope summary
         self.assertEqual(index_task.task['scopes'],
                          ['index:insert-task:gecko.v2.mozilla-central.*'])
 
-    def make_taskgraph(self, tasks):
-        label_to_taskid = {k: k + '-tid' for k in tasks}
-        for label, task_id in label_to_taskid.iteritems():
-            tasks[label].task_id = task_id
-        graph = Graph(nodes=set(tasks), edges=set())
-        taskgraph = TaskGraph(tasks, graph)
-        return taskgraph, label_to_taskid
+
+class TestApplyJSONeTemplates(MorphTestCase):
+
+    tasks = [
+        Task(kind='build', label='a', attributes={}, task={
+            'extra': {
+                'treeherder': {
+                    'group': 'tc',
+                    'symbol': 'B'
+                }
+            },
+            'payload': {
+                'env': {
+                    'FOO': 'BAR'
+                }
+            },
+            'tags': {
+                'kind': 'build'
+            }
+        }),
+        Task(kind='test', label='b', attributes={}, task={
+            'extra': {
+                'treeherder': {
+                    'group': 'tc',
+                    'symbol': 't'
+                }
+            },
+            'payload': {
+                'env': {
+                    'FOO': 'BAR'
+                }
+            },
+            'tags': {
+                'kind': 'test'
+            }
+        }),
+    ]
+
+    def test_template_artifact(self):
+        tg, label_to_taskid = self.make_taskgraph({
+            t.label: t for t in self.tasks
+        })
+
+        fn = morph.apply_jsone_templates({'artifact': {'enabled': 1}})
+        morphed = fn(tg, label_to_taskid)[0]
+
+        self.assertEqual(len(morphed.tasks), 2)
+
+        for t in morphed.tasks.values():
+            if t.kind == 'build':
+                self.assertEqual(t.task['extra']['treeherder']['group'], 'tc')
+                self.assertEqual(t.task['extra']['treeherder']['symbol'], 'Ba')
+                self.assertEqual(t.task['payload']['env']['USE_ARTIFACT'], 1)
+            else:
+                self.assertEqual(t.task['extra']['treeherder']['group'], 'tc')
+                self.assertEqual(t.task['extra']['treeherder']['symbol'], 't')
+                self.assertNotIn('USE_ARTIFACT', t.task['payload']['env'])
 
 
 if __name__ == '__main__':
     main()
--- a/testing/mozharness/scripts/fx_desktop_build.py
+++ b/testing/mozharness/scripts/fx_desktop_build.py
@@ -120,28 +120,28 @@ class FxDesktopBuild(BuildScript, TryToo
                     platform_for_log_url += '-pgo'
                 # postrun.py uses stage_platform buildbot prop as part of the log url
                 self.set_buildbot_property('stage_platform',
                                            platform_for_log_url,
                                            write_to_file=True)
             else:
                 self.fatal("'stage_platform' not determined and is required in your config")
 
-        if self.try_message_has_flag('artifact'):
+        if self.try_message_has_flag('artifact') or os.environ.get('USE_ARTIFACT'):
             # Not all jobs that look like builds can be made into artifact
             # builds (for example, various SAN builds will not make sense as
             # artifact builds).  By default, only a vanilla debug or opt build
             # will be replaced by an artifact build.
             #
             # In addition, some jobs want to specify their artifact equivalent.
             # Use `artifact_flag_build_variant_in_try` to specify that variant.
             #
             # This is temporary, until we find a way to introduce an "artifact
             # build dimension" like "opt"/"debug" into the CI configurations.
-            self.info('Artifact build requested in try syntax.')
+            self.info('Artifact build requested by try push.')
 
             variant = None
 
             if 'artifact_flag_build_variant_in_try' in c:
                 variant = c['artifact_flag_build_variant_in_try']
                 if not variant:
                     self.info('Build variant has falsy `artifact_flag_build_variant_in_try`; '
                               'ignoring artifact build request and performing original build.')