Bug 1286809 - Provide mechanism to run a subset of tasks in nss-try r=franziskus
authorTim Taubert <ttaubert@mozilla.com>
Tue, 26 Jul 2016 07:01:27 +0200
changeset 12406 e8327de7628eabf2cecea779227f8aa840acf462
parent 12405 49ef294a7ceeaf1390b56684260ceeea13dd4e7b
child 12407 f92631b5152c31558b24c5093c88e65fe938c3ad
push id1417
push userttaubert@mozilla.com
push dateTue, 26 Jul 2016 05:03:49 +0000
reviewersfranziskus
bugs1286809
Bug 1286809 - Provide mechanism to run a subset of tasks in nss-try r=franziskus
.taskcluster.yml
automation/taskcluster/docker/setup.sh
automation/taskcluster/graph/build.js
automation/taskcluster/graph/linux/_build_base.yml
automation/taskcluster/graph/linux/_test_base.yml
automation/taskcluster/graph/tools/_build_base.yml
automation/taskcluster/graph/try_syntax.js
automation/taskcluster/scripts/extend_task_graph.sh
--- a/.taskcluster.yml
+++ b/.taskcluster.yml
@@ -52,22 +52,23 @@ tasks:
       tags:
         createdForUser: {{owner}}
 
       routes:
         - "tc-treeherder-stage.v2.{{project}}.{{revision}}.{{pushlog_id}}"
         - "tc-treeherder.v2.{{project}}.{{revision}}.{{pushlog_id}}"
 
       payload:
-        image: "ttaubert/nss-ci:0.0.17"
+        image: "ttaubert/nss-ci:0.0.18"
 
         env:
           TC_OWNER: {{owner}}
           TC_SOURCE: {{{source}}}
           TC_PROJECT: {{project}}
+          TC_COMMENT: '{{comment}}'
           NSS_PUSHLOG_ID: '{{pushlog_id}}'
           NSS_HEAD_REPOSITORY: '{{{url}}}'
           NSS_HEAD_REVISION: '{{revision}}'
 
         maxRunTime: 1800
 
         command:
           - bash
--- a/automation/taskcluster/docker/setup.sh
+++ b/automation/taskcluster/docker/setup.sh
@@ -53,16 +53,13 @@ update-alternatives --install /usr/bin/g
 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-6 20
 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-6 20
 update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-5 30
 update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-5 30
 
 locale-gen en_US.UTF-8
 dpkg-reconfigure locales
 
-# Install required Node modules.
-su -c "npm install flatmap js-yaml merge slugid" worker
-
 # Cleanup.
 rm -rf ~/.ccache ~/.cache
 apt-get clean
 apt-get autoclean
 rm $0
--- a/automation/taskcluster/graph/build.js
+++ b/automation/taskcluster/graph/build.js
@@ -3,23 +3,24 @@
  * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
 
 var fs = require("fs");
 var path = require("path");
 var merge = require("merge");
 var yaml = require("js-yaml");
 var slugid = require("slugid");
 var flatmap = require("flatmap");
+var try_syntax = require("./try_syntax");
 
 // Default values for debugging.
 var TC_OWNER = process.env.TC_OWNER || "{{tc_owner}}";
 var TC_SOURCE = process.env.TC_SOURCE || "{{tc_source}}";
 var TC_PROJECT = process.env.TC_PROJECT || "{{tc_project}}";
+var TC_COMMENT = process.env.TC_COMMENT || "{{tc_comment}}";
 var NSS_PUSHLOG_ID = process.env.NSS_PUSHLOG_ID || "{{nss_pushlog_id}}";
-var NSS_HEAD_REPOSITORY = process.env.NSS_HEAD_REPOSITORY || "{{nss_head_repo}}";
 var NSS_HEAD_REVISION = process.env.NSS_HEAD_REVISION || "{{nss_head_rev}}";
 
 // Register custom YAML types.
 var YAML_SCHEMA = yaml.Schema.create([
   // Point in time at $now + x hours.
   new yaml.Type('!from_now', {
     kind: "scalar",
 
@@ -38,17 +39,17 @@ var YAML_SCHEMA = yaml.Schema.create([
   new yaml.Type('!env', {
     kind: "scalar",
 
     resolve: function (data) {
       return true;
     },
 
     construct: function (data) {
-      return process.env[data];
+      return process.env[data] || "{{" + data.toLowerCase() + "}}";
     }
   })
 ]);
 
 // Parse a given YAML file.
 function parseYamlFile(file, fallback) {
   // Return fallback if the file doesn't exist.
   if (!fs.existsSync(file) && fallback) {
@@ -168,10 +169,15 @@ function generatePlatformTasks(platform)
   });
 }
 
 // Construct the task graph.
 var graph = {
   tasks: flatmap(["linux", "windows", "tools"], generatePlatformTasks)
 };
 
+// Filter tasks when try syntax is given.
+if (TC_PROJECT == "nss-try") {
+  graph.tasks = try_syntax.filterTasks(graph.tasks, TC_COMMENT);
+}
+
 // Output the final graph.
 process.stdout.write(JSON.stringify(graph, null, 2));
--- a/automation/taskcluster/graph/linux/_build_base.yml
+++ b/automation/taskcluster/graph/linux/_build_base.yml
@@ -9,17 +9,17 @@ task:
   schedulerId: task-graph-scheduler
 
   metadata:
     owner: !env TC_OWNER
     source: !env TC_SOURCE
 
   payload:
     maxRunTime: 3600
-    image: ttaubert/nss-ci:0.0.17
+    image: ttaubert/nss-ci:0.0.18
 
     artifacts:
       public:
         type: directory
         path: /home/worker/artifacts
         expires: !from_now 24
 
     command:
--- a/automation/taskcluster/graph/linux/_test_base.yml
+++ b/automation/taskcluster/graph/linux/_test_base.yml
@@ -9,14 +9,14 @@ task:
   schedulerId: task-graph-scheduler
 
   metadata:
     owner: !env TC_OWNER
     source: !env TC_SOURCE
 
   payload:
     maxRunTime: 3600
-    image: ttaubert/nss-ci:0.0.17
+    image: ttaubert/nss-ci:0.0.18
 
     command:
       - "/bin/bash"
       - "-c"
       - "bin/checkout.sh && nss/automation/taskcluster/scripts/run_tests.sh"
--- a/automation/taskcluster/graph/tools/_build_base.yml
+++ b/automation/taskcluster/graph/tools/_build_base.yml
@@ -9,17 +9,17 @@ task:
   schedulerId: task-graph-scheduler
 
   metadata:
     owner: !env TC_OWNER
     source: !env TC_SOURCE
 
   payload:
     maxRunTime: 3600
-    image: ttaubert/nss-ci:0.0.17
+    image: ttaubert/nss-ci:0.0.18
 
     env:
       NSS_HEAD_REPOSITORY: !env NSS_HEAD_REPOSITORY
       NSS_HEAD_REVISION: !env NSS_HEAD_REVISION
 
   extra:
     treeherder:
       build:
new file mode 100644
--- /dev/null
+++ b/automation/taskcluster/graph/try_syntax.js
@@ -0,0 +1,136 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+var intersect = require("intersect");
+var parse_args = require("minimist");
+
+function parseOptions(opts) {
+  opts = parse_args(opts.split(/\s+/), {
+    default: {build: "do", platform: "all", unittests: "none", tools: "none"},
+    alias: {b: "build", p: "platform", u: "unittests", t: "tools"},
+    string: ["build", "platform", "unittests", "tools"]
+  });
+
+  // Parse build types (d=debug, o=opt).
+  var builds = intersect(opts.build.split(""), ["d", "o"]);
+
+  // If the given value is nonsense default to debug and opt builds.
+  if (builds.length == 0) {
+    builds = ["d", "o"];
+  }
+
+  // Parse platforms.
+  var allPlatforms = ["linux", "linux64", "linux64-asan", "win64"];
+  var platforms = intersect(opts.platform.split(/\s*,\s*/), allPlatforms);
+
+  // If the given value is nonsense or "none" default to all platforms.
+  if (platforms.length == 0 && opts.platform != "none") {
+    platforms = allPlatforms;
+  }
+
+  // Parse unit tests.
+  var allUnitTests = ["crmf", "chains", "cipher", "db", "ec", "fips", "gtest",
+                      "lowhash", "merge", "sdr", "smime", "tools", "ssl"];
+  var unittests = intersect(opts.unittests.split(/\s*,\s*/), allUnitTests);
+
+  // If the given value is "all" run all tests.
+  // If it's nonsense then don't run any tests.
+  if (opts.unittests == "all") {
+    unittests = allUnitTests;
+  } else if (unittests.length == 0) {
+    unittests = [];
+  }
+
+  // Parse tools.
+  var allTools = ["clang-format", "scan-build"];
+  var tools = intersect(opts.tools.split(/\s*,\s*/), allTools);
+
+  // If the given value is "all" run all tools.
+  // If it's nonsense then don't run any tools.
+  if (opts.tools == "all") {
+    tools = allTools;
+  } else if (tools.length == 0) {
+    tools = [];
+  }
+
+  return {
+    builds: builds,
+    platforms: platforms,
+    unittests: unittests,
+    tools: tools
+  };
+}
+
+function filterTasks(tasks, comment) {
+  // Check for try syntax in changeset comment.
+  var match = comment.match(/^\s*try:\s*(.*)\s*$/);
+  if (!match) {
+    return tasks;
+  }
+
+  var opts = parseOptions(match[1]);
+
+  return tasks.filter(function (task) {
+    var env = task.task.payload.env || {};
+    var th = task.task.extra.treeherder;
+    var symbol = th.symbol.toLowerCase();
+    var machine = th.machine.platform;
+    var coll = th.collection || {};
+    var found;
+
+    // Never include memleak builds, they'll go away soon.
+    if (coll.memleak) {
+      return false;
+    }
+
+    // Filter tools. We can immediately return here as those
+    // are not affected by platform or build type selectors.
+    if (machine == "nss-tools") {
+      return opts.tools.some(function (tool) {
+        return symbol.startsWith(tool);
+      });
+    }
+
+    // Filter unit tests.
+    if (env.NSS_TESTS && env.TC_PARENT_TASK_ID) {
+      found = opts.unittests.some(function (test) {
+        return symbol.startsWith(test);
+      });
+
+      if (!found) {
+        return false;
+      }
+    }
+
+    // Filter by platform.
+    found = opts.platforms.some(function (platform) {
+      var aliases = {
+        "linux": "linux32",
+        "linux64-asan": "linux64",
+        "win64": "windows2012-64"
+      };
+
+      // Check the platform name.
+      var keep = machine == (aliases[platform] || platform);
+
+      // Additional check for LSan.
+      if (platform == "linux64-asan") {
+        keep &= coll.asan || coll.lsan;
+      }
+
+      return keep;
+    });
+
+    if (!found) {
+      return false;
+    }
+
+    // Finally, filter by build type.
+    var isDebug = coll.debug || coll.asan || coll.lsan;
+    return (isDebug && opts.builds.indexOf("d") > -1) ||
+           (!isDebug && opts.builds.indexOf("o") > -1);
+  });
+}
+
+module.exports.filterTasks = filterTasks;
--- a/automation/taskcluster/scripts/extend_task_graph.sh
+++ b/automation/taskcluster/scripts/extend_task_graph.sh
@@ -4,10 +4,13 @@ set -v -e -x
 
 if [ $(id -u) = 0 ]; then
     # Drop privileges by re-running this script.
     exec su worker $0
 fi
 
 mkdir -p /home/worker/artifacts
 
+# Install Node.JS dependencies.
+npm install flatmap js-yaml merge slugid minimist intersect
+
 # Build the task graph definition.
 nodejs nss/automation/taskcluster/graph/build.js > /home/worker/artifacts/graph.json