Bug 1502749: [taskgraph] Use the correct index rank for index-tasks morphs; r=bstack
authorTom Prince <mozilla@hocat.ca>
Sun, 28 Oct 2018 21:54:12 +0000
changeset 443265 ccef78189e7bb02a2f9687760b5a48accdc7eaff
parent 443264 f117f841519e338db9f54e6b89fb5772156e1ccd
child 443266 fcadaf0b5611d25760f6f5f09198152418d7a965
push id34950
push usercsabou@mozilla.com
push dateMon, 29 Oct 2018 04:16:25 +0000
treeherdermozilla-central@d2e24bdf0648 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbstack
bugs1502749
milestone65.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1502749: [taskgraph] Use the correct index rank for index-tasks morphs; r=bstack When a task gets too routes to be indexed by the queue, we create a new dependent task that generates the index tasks. This task didn't set the index rank, so if a task gets new routes that push it over the limit, it would stop updating the old index, as it would be using an index of 0. Differential Revision: https://phabricator.services.mozilla.com/D10016
taskcluster/docker/index-task/insert-indexes.js
taskcluster/taskgraph/morph.py
taskcluster/taskgraph/test/test_morph.py
--- a/taskcluster/docker/index-task/insert-indexes.js
+++ b/taskcluster/docker/index-task/insert-indexes.js
@@ -10,31 +10,37 @@ let index = new taskcluster.Index({
 // Create queue instance for fetching taskId
 let queue = new taskcluster.Queue({
     delayFactor:    750,  // Good solid delay for background process
     retries:        8,    // A few extra retries for robustness
 });
 
 // Load input
 let taskId = process.env.TARGET_TASKID;
+let rank = parseInt(process.env.INDEX_RANK, 10);
 let namespaces = process.argv.slice(2);
 
 // Validate input
 if (!taskId) {
   console.log("Expected target task as environment variable: TARGET_TASKID");
   process.exit(1);
 }
 
+if (isNaN(rank)) {
+  console.log("Expected index rank as environment variable: INDEX_RANK");
+  process.exit(1);
+}
+
 // Fetch task definition to get expiration and then insert into index
 queue.task(taskId).then(task => task.expires).then(expires => {
   return Promise.all(namespaces.map(namespace => {
-    console.log("Inserting %s into index under: %s", taskId, namespace);
+    console.log("Inserting %s into index (rank %d) under: %s", taskId, rank, namespace);
     return index.insertTask(namespace, {
       taskId,
-      rank: 0,
+      rank,
       data: {},
       expires,
     });
   }));
 }).then(() => {
   console.log("indexing successfully completed.");
   process.exit(0);
 }).catch(err => {
--- a/taskcluster/taskgraph/morph.py
+++ b/taskcluster/taskgraph/morph.py
@@ -78,17 +78,17 @@ def derive_misc_task(task, purpose, imag
                 'path': 'public/image.tar.zst',
                 'taskId': image_taskid,
                 'type': 'task-image',
             },
             'features': {
                 'taskclusterProxy': True,
             },
             'maxRunTime': 600,
-        }
+        },
     }
 
     # only include the docker-image dependency here if it is actually in the
     # taskgraph (has not been optimized).  It is included in
     # task_def['dependencies'] unconditionally.
     dependencies = {'parent': task.task_id}
     if image_taskid in taskgraph.tasks:
         dependencies['docker-image'] = image_taskid
@@ -127,17 +127,20 @@ def make_index_task(parent_task, taskgra
             match = summ_re.match(scope)
             if match:
                 scope = match.group(1) + '*'
                 break
         scopes.add(scope)
     task.task['scopes'] = sorted(scopes)
 
     task.task['payload']['command'] = ['insert-indexes.js'] + index_paths
-    task.task['payload']['env'] = {"TARGET_TASKID": parent_task.task_id}
+    task.task['payload']['env'] = {
+        'TARGET_TASKID': parent_task.task_id,
+        'INDEX_RANK': parent_task.task.get('extra', {}).get('index', {}).get('rank', 0),
+    }
     return task
 
 
 def add_index_tasks(taskgraph, label_to_taskid):
     """
     The TaskCluster queue only allows 10 routes on a task, but we have tasks
     with many more routes, for purposes of indexing. This graph morph adds
     "index tasks" that depend on such tasks and do the index insertions
--- a/taskcluster/taskgraph/test/test_morph.py
+++ b/taskcluster/taskgraph/test/test_morph.py
@@ -54,37 +54,41 @@ def test_make_index_tasks(make_taskgraph
             "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.fy-NL",
             "index.gecko.v2.mozilla-central.revision."
             "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sk",
             "index.gecko.v2.mozilla-central.revision."
             "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.sl",
             "index.gecko.v2.mozilla-central.revision."
             "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.uk",
             "index.gecko.v2.mozilla-central.revision."
-            "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN"
+            "b5d8b27a753725c1de41ffae2e338798f3b5cacd.firefox-l10n.linux64-opt.zh-CN",
         ],
         'deadline': 'soon',
         'metadata': {
             'description': 'desc',
             'owner': 'owner@foo.com',
             'source': 'https://source',
         },
+        'extra': {
+            'index': {'rank': 1540722354},
+        },
     }
     task = Task(kind='test', label='a', attributes={}, task=task_def)
     docker_task = Task(kind='docker-image', label='build-docker-image-index-task',
                        attributes={}, task={})
     taskgraph, label_to_taskid = make_taskgraph({
         task.label: task,
         docker_task.label: docker_task,
     })
 
     index_task = morph.make_index_task(task, taskgraph, label_to_taskid)
 
     assert index_task.task['payload']['command'][0] == 'insert-indexes.js'
     assert index_task.task['payload']['env']['TARGET_TASKID'] == 'a-tid'
+    assert index_task.task['payload']['env']['INDEX_RANK'] == 1540722354
 
     # check the scope summary
     assert index_task.task['scopes'] == ['index:insert-task:gecko.v2.mozilla-central.*']
 
 
 TASKS = [
     {
         'kind': 'build',