Bug 1535484 - Add proper logs for node-devtools test suite r=jlast
☠☠ backed out by 5003c4244ea2 ☠ ☠
authorJulian Descottes <jdescottes@mozilla.com>
Fri, 10 May 2019 14:31:49 +0000
changeset 535288 f005c5114e062c9ad0a5552784a97b56488f81a4
parent 535287 1760c6913c094a2364658cbd73131383e452d890
child 535289 5003c4244ea22c6a1337b3716bc641cb17891638
push id2082
push userffxbld-merge
push dateMon, 01 Jul 2019 08:34:18 +0000
treeherdermozilla-release@2fb19d0466d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlast
bugs1535484
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1535484 - Add proper logs for node-devtools test suite r=jlast Differential Revision: https://phabricator.services.mozilla.com/D30630
devtools/client/bin/devtools-node-test-runner.js
taskcluster/ci/source-test/node.yml
new file mode 100644
--- /dev/null
+++ b/devtools/client/bin/devtools-node-test-runner.js
@@ -0,0 +1,114 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+
+/* global __dirname, process */
+
+"use strict";
+
+/**
+ * This is a test runner dedicated to run DevTools node tests continuous integration
+ * platforms. It will parse the logs to output errors compliant with treeherder tooling.
+ *
+ * See taskcluster/ci/source-test/node.yml for the definition of the task running those
+ * tests on try.
+ */
+
+const { execFileSync } = require("child_process");
+const { chdir } = require("process");
+const path = require("path");
+
+// Supported node test suites for DevTools
+const TEST_TYPES = {
+  JEST: "jest",
+  MOCHA: "mocha",
+};
+
+const SUITES = {
+  "aboutdebugging-new": {
+    path: "../aboutdebugging-new/test/jest",
+    type: TEST_TYPES.JEST,
+  },
+  "framework": {
+    path: "../framework/test/jest",
+    type: TEST_TYPES.JEST,
+  },
+  "netmonitor": {
+    path: "../netmonitor/test",
+    type: TEST_TYPES.JEST,
+  },
+  "webconsole": {
+    path: "../webconsole/test",
+    type: TEST_TYPES.MOCHA,
+  },
+};
+
+function execOut(...args) {
+  let out;
+  let err;
+  try {
+    out = execFileSync(...args);
+  } catch (e) {
+    out = e.stdout;
+    err = e.stderr;
+  }
+  return { out: out.toString(), err: err && err.toString() };
+}
+
+function getErrors(suite, out, err) {
+  switch (SUITES[suite].type) {
+    case TEST_TYPES.JEST:
+      // jest errors are logged in the `err` buffer.
+      return parseErrorsFromLogs(err, / {4}✕\s*(.*)/);
+    case TEST_TYPES.MOCHA:
+      // mocha errors are logged in the `out` buffer, and will either log Error or
+      // TypeError depending on the error detected.
+      return parseErrorsFromLogs(out, / {4}((?:Type)?Error\:.*)/);
+    default:
+      throw new Error("Unsupported suite type: " + SUITES[suite].type);
+  }
+}
+
+function parseErrorsFromLogs(text, regExp) {
+  text = text || "";
+  const globalRegexp = new RegExp(regExp, "g");
+  const matches = text.match(globalRegexp) || [];
+  return matches.map(m => m.match(regExp)[1]);
+}
+
+function runTests() {
+  console.log("[devtools-node-test-runner] Extract suite argument");
+  const suiteArg = process.argv.find(arg => arg.includes("suite="));
+  const suite = suiteArg.split("=")[1];
+  if (!SUITES[suite]) {
+    throw new Error("Invalid suite argument to devtools-node-test-runner: " + suite);
+  }
+
+  console.log("[devtools-node-test-runner] Found test suite: " + suite);
+  const testPath = path.join(__dirname, SUITES[suite].path);
+  chdir(testPath);
+
+  console.log("[devtools-node-test-runner] Run `yarn` in test folder");
+  execOut("yarn");
+
+  console.log(`TEST START | ${SUITES[suite].type} | ${suite}`);
+
+  console.log("[devtools-node-test-runner] Run `yarn test` in test folder");
+  const { out, err } = execOut("yarn", ["test"]);
+
+  if (err) {
+    console.log("[devtools-node-test-runner] Error log");
+    console.log(err);
+  }
+
+  console.log("[devtools-node-test-runner] Parse errors from the test logs");
+  const errors = getErrors(suite, out, err) || [];
+  for (const error of errors) {
+    console.log(`TEST-UNEXPECTED-FAIL | ${SUITES[suite].type} | ${suite} | ${error}`);
+  }
+  return errors.length === 0;
+}
+
+const success = runTests();
+
+process.exitCode = success ? 0 : 1;
--- a/taskcluster/ci/source-test/node.yml
+++ b/taskcluster/ci/source-test/node.yml
@@ -23,39 +23,32 @@ debugger-tests:
             - 'devtools/client/debugger/**'
 
 devtools-tests:
     description: devtools node-based tests (for instance jest)
     platform: linux64/opt
     treeherder:
         symbol: node(devtools)
         kind: test
-        tier: 3
+        tier: 1
     worker-type: t-linux-xlarge
     worker:
         docker-image: {in-tree: "lint"}
         max-run-time: 1800
     run:
         using: run-task
         cache-dotcache: true
         command: >
             cd /builds/worker/checkouts/gecko/ &&
             npm install &&
-            cd /builds/worker/checkouts/gecko/devtools/client/aboutdebugging-new/test/jest &&
-            yarn &&
-            yarn test &&
-            cd /builds/worker/checkouts/gecko/devtools/client/framework/test/jest &&
-            yarn &&
-            yarn test &&
-            cd /builds/worker/checkouts/gecko/devtools/client/webconsole/test &&
-            yarn &&
-            yarn test &&
-            cd /builds/worker/checkouts/gecko/devtools/client/netmonitor/test &&
-            yarn &&
-            yarn test
+            cd /builds/worker/checkouts/gecko/devtools/client/bin/ &&
+            node devtools-node-test-runner.js --suite=aboutdebugging-new &&
+            node devtools-node-test-runner.js --suite=framework &&
+            node devtools-node-test-runner.js --suite=netmonitor &&
+            node devtools-node-test-runner.js --suite=webconsole
     when:
         files-changed:
             - 'devtools/client/aboutdebugging-new/src/components/**'
             - 'devtools/client/framework/components/**'
             - 'devtools/client/webconsole/**'
             - 'devtools/client/netmonitor/**'
 
 eslint-plugin-mozilla: