Bug 1535484 - Add proper logs for node-devtools test suite r=jlast
☠☠ backed out by cacd53838ee6 ☠ ☠
authorJulian Descottes <jdescottes@mozilla.com>
Fri, 10 May 2019 16:42:02 +0000
changeset 535307 865ec5a708ba94d700672df65d5ab5ea81c01898
parent 535306 f798e09dc294fd2c86f67489ac5b1bbf5195f07d
child 535308 186144ac4762b3b64311290a7b9c5ecf1addea19
push id2082
push userffxbld-merge
push dateMon, 01 Jul 2019 08:34:18 +0000
treeherdermozilla-release@2fb19d0466d2 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjlast
bugs1535484
milestone68.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1535484 - Add proper logs for node-devtools test suite r=jlast Differential Revision: https://phabricator.services.mozilla.com/D30630
devtools/client/bin/devtools-node-test-runner.js
devtools/client/webconsole/test/components/evaluation-result.test.js
taskcluster/ci/source-test/node.yml
new file mode 100644
--- /dev/null
+++ b/devtools/client/bin/devtools-node-test-runner.js
@@ -0,0 +1,114 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+
+/* global __dirname, process */
+
+"use strict";
+
+/**
+ * This is a test runner dedicated to run DevTools node tests continuous integration
+ * platforms. It will parse the logs to output errors compliant with treeherder tooling.
+ *
+ * See taskcluster/ci/source-test/node.yml for the definition of the task running those
+ * tests on try.
+ */
+
+const { execFileSync } = require("child_process");
+const { chdir } = require("process");
+const path = require("path");
+
+// Supported node test suites for DevTools
+const TEST_TYPES = {
+  JEST: "jest",
+  MOCHA: "mocha",
+};
+
+const SUITES = {
+  "aboutdebugging-new": {
+    path: "../aboutdebugging-new/test/jest",
+    type: TEST_TYPES.JEST,
+  },
+  "framework": {
+    path: "../framework/test/jest",
+    type: TEST_TYPES.JEST,
+  },
+  "netmonitor": {
+    path: "../netmonitor/test",
+    type: TEST_TYPES.JEST,
+  },
+  "webconsole": {
+    path: "../webconsole/test",
+    type: TEST_TYPES.MOCHA,
+  },
+};
+
+function execOut(...args) {
+  let out;
+  let err;
+  try {
+    out = execFileSync(...args);
+  } catch (e) {
+    out = e.stdout;
+    err = e.stderr;
+  }
+  return { out: out.toString(), err: err && err.toString() };
+}
+
+function getErrors(suite, out, err) {
+  switch (SUITES[suite].type) {
+    case TEST_TYPES.JEST:
+      // jest errors are logged in the `err` buffer.
+      return parseErrorsFromLogs(err, / {4}✕\s*(.*)/);
+    case TEST_TYPES.MOCHA:
+      // mocha errors are logged in the `out` buffer, and will either log Error or
+      // TypeError depending on the error detected.
+      return parseErrorsFromLogs(out, / {4}((?:Type)?Error\:.*)/);
+    default:
+      throw new Error("Unsupported suite type: " + SUITES[suite].type);
+  }
+}
+
+function parseErrorsFromLogs(text, regExp) {
+  text = text || "";
+  const globalRegexp = new RegExp(regExp, "g");
+  const matches = text.match(globalRegexp) || [];
+  return matches.map(m => m.match(regExp)[1]);
+}
+
+function runTests() {
+  console.log("[devtools-node-test-runner] Extract suite argument");
+  const suiteArg = process.argv.find(arg => arg.includes("suite="));
+  const suite = suiteArg.split("=")[1];
+  if (!SUITES[suite]) {
+    throw new Error("Invalid suite argument to devtools-node-test-runner: " + suite);
+  }
+
+  console.log("[devtools-node-test-runner] Found test suite: " + suite);
+  const testPath = path.join(__dirname, SUITES[suite].path);
+  chdir(testPath);
+
+  console.log("[devtools-node-test-runner] Run `yarn` in test folder");
+  execOut("yarn");
+
+  console.log(`TEST START | ${SUITES[suite].type} | ${suite}`);
+
+  console.log("[devtools-node-test-runner] Run `yarn test` in test folder");
+  const { out, err } = execOut("yarn", ["test"]);
+
+  if (err) {
+    console.log("[devtools-node-test-runner] Error log");
+    console.log(err);
+  }
+
+  console.log("[devtools-node-test-runner] Parse errors from the test logs");
+  const errors = getErrors(suite, out, err) || [];
+  for (const error of errors) {
+    console.log(`TEST-UNEXPECTED-FAIL | ${SUITES[suite].type} | ${suite} | ${error}`);
+  }
+  return errors.length === 0;
+}
+
+const success = runTests();
+
+process.exitCode = success ? 0 : 1;
--- a/devtools/client/webconsole/test/components/evaluation-result.test.js
+++ b/devtools/client/webconsole/test/components/evaluation-result.test.js
@@ -92,36 +92,37 @@ describe("EvaluationResult component:", 
     const message = stubPreparedMessages.get("cd(document)");
     const wrapper = render(EvaluationResult({ message, serviceContainer }));
 
     expect(wrapper.find(".message-body").text()).toBe(
       "Cannot cd() to the given window. Invalid argument."
     );
   });
 
-  it("displays a [Learn more] link", () => {
-    const store = setupStore();
-
-    const message = stubPreparedMessages.get("asdf()");
+  // TODO: Regressed by Bug 1230194, disabled in Bug 1535484. Filed Bug 1550791 to fix it.
+  // it("displays a [Learn more] link", () => {
+  //   const store = setupStore();
 
-    serviceContainer.openLink = sinon.spy();
-    const wrapper = mount(
-      Provider({ store }, EvaluationResult({ message, serviceContainer }))
-    );
+  //   const message = stubPreparedMessages.get("asdf()");
+
+  //   serviceContainer.openLink = sinon.spy();
+  //   const wrapper = mount(
+  //     Provider({ store }, EvaluationResult({ message, serviceContainer }))
+  //   );
 
-    const url =
-      "https://developer.mozilla.org/docs/Web/JavaScript/Reference/Errors/Not_defined";
-    const learnMore = wrapper.find(".learn-more-link");
-    expect(learnMore.length).toBe(1);
-    expect(learnMore.prop("title")).toBe(url);
+  //   const url =
+  //     "https://developer.mozilla.org/docs/Web/JavaScript/Reference/Errors/Not_defined";
+  //   const learnMore = wrapper.find(".learn-more-link");
+  //   expect(learnMore.length).toBe(1);
+  //   expect(learnMore.prop("title")).toBe(url);
 
-    learnMore.simulate("click");
-    const call = serviceContainer.openLink.getCall(0);
-    expect(call.args[0]).toEqual(message.exceptionDocURL);
-  });
+  //   learnMore.simulate("click");
+  //   const call = serviceContainer.openLink.getCall(0);
+  //   expect(call.args[0]).toEqual(message.exceptionDocURL);
+  // });
 
   it("has the expected indent", () => {
     const message = stubPreparedMessages.get("new Date(0)");
 
     const indent = 10;
     // We need to wrap the ConsoleApiElement in a Provider in order for the
     // ObjectInspector to work.
     let wrapper = render(
--- a/taskcluster/ci/source-test/node.yml
+++ b/taskcluster/ci/source-test/node.yml
@@ -23,39 +23,32 @@ debugger-tests:
             - 'devtools/client/debugger/**'
 
 devtools-tests:
     description: devtools node-based tests (for instance jest)
     platform: linux64/opt
     treeherder:
         symbol: node(devtools)
         kind: test
-        tier: 3
+        tier: 1
     worker-type: t-linux-xlarge
     worker:
         docker-image: {in-tree: "lint"}
         max-run-time: 1800
     run:
         using: run-task
         cache-dotcache: true
         command: >
             cd /builds/worker/checkouts/gecko/ &&
             npm install &&
-            cd /builds/worker/checkouts/gecko/devtools/client/aboutdebugging-new/test/jest &&
-            yarn &&
-            yarn test &&
-            cd /builds/worker/checkouts/gecko/devtools/client/framework/test/jest &&
-            yarn &&
-            yarn test &&
-            cd /builds/worker/checkouts/gecko/devtools/client/webconsole/test &&
-            yarn &&
-            yarn test &&
-            cd /builds/worker/checkouts/gecko/devtools/client/netmonitor/test &&
-            yarn &&
-            yarn test
+            cd /builds/worker/checkouts/gecko/devtools/client/bin/ &&
+            node devtools-node-test-runner.js --suite=aboutdebugging-new &&
+            node devtools-node-test-runner.js --suite=framework &&
+            node devtools-node-test-runner.js --suite=netmonitor &&
+            node devtools-node-test-runner.js --suite=webconsole
     when:
         files-changed:
             - 'devtools/client/aboutdebugging-new/src/components/**'
             - 'devtools/client/framework/components/**'
             - 'devtools/client/webconsole/**'
             - 'devtools/client/netmonitor/**'
 
 eslint-plugin-mozilla: