Bug 1567924 - add newtab/messaging unit testing to taskcluster r=Mardak,ahal
authorDan Mosedale <dmose@mozilla.org>
Thu, 29 Aug 2019 14:06:38 +0000
changeset 554391 ee1b60bbb5103a129918f2301696a51d80bbdd16
parent 554390 3be18873536a3f403c9a0cf30f06bc939103a787
child 554392 9f021f1c28e0fe557be7b2768d41e64456a403d2
push id2165
push userffxbld-merge
push dateMon, 14 Oct 2019 16:30:58 +0000
treeherdermozilla-release@0eae18af659f [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersMardak, ahal
bugs1567924
milestone70.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1567924 - add newtab/messaging unit testing to taskcluster r=Mardak,ahal Adds unit tests for the Home page code and Messaging System code in browser/component/newtab into treeherder/try automation. Currently at Tier 2, should move to Tier 1 before long. Differential Revision: https://phabricator.services.mozilla.com/D43562
browser/components/newtab/bin/try-runner.js
browser/components/newtab/karma.mc.config.js
taskcluster/ci/source-test/node.yml
new file mode 100644
--- /dev/null
+++ b/browser/components/newtab/bin/try-runner.js
@@ -0,0 +1,144 @@
+/* eslint-disable no-console */
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at <http://mozilla.org/MPL/2.0/>. */
+
+/*
+ * A small test runner/reporter for node-based tests,
+ * which are run via taskcluster node(debugger).
+ *
+ * Forked from
+ * https://searchfox.org/mozilla-central/source/devtools/client/debugger/bin/try-runner.js
+ */
+
+const { execFileSync } = require("child_process");
+const { readFileSync } = require("fs");
+const path = require("path");
+
+function logErrors(tool, errors) {
+  for (const error of errors) {
+    console.log(`TEST-UNEXPECTED-FAIL ${tool} | ${error}`);
+  }
+  return errors;
+}
+
+function execOut(...args) {
+  let out;
+  let err;
+
+  try {
+    out = execFileSync(...args, {
+      silent: false,
+    });
+  } catch (e) {
+    // For debugging on (eg) try server...
+    //
+    // if (e) {
+    //   logErrors("execOut", ["execFileSync returned exception: ", e]);
+    // }
+
+    out = e && e.stdout;
+    err = e && e.stderr;
+  }
+  return { out: out && out.toString(), err: err && err.toString() };
+}
+
+function logStart(name) {
+  console.log(`TEST START | ${name}`);
+}
+
+function karma() {
+  logStart("karma");
+
+  const { out } = execOut("npm", [
+    "run",
+    "testmc:unit",
+    // , "--", "--log-level", "--verbose",
+    // to debug the karma integration, uncomment the above line
+  ]);
+
+  // karma spits everything to stdout, not stderr, so if nothing came back on
+  // stdout, give up now.
+  if (!out) {
+    return false;
+  }
+
+  let jsonContent;
+  try {
+    // Note that this will be overwritten at each run, but that shouldn't
+    // matter.
+    jsonContent = readFileSync(path.join("logs", "karma-run-results.json"));
+  } catch (ex) {
+    console.error("exception reading karma-run-results.json: ", ex);
+    return false;
+  }
+
+  const results = JSON.parse(jsonContent);
+
+  const failed = results.summary.failed === 0;
+
+  let errors = [];
+  // eslint-disable-next-line guard-for-in
+  for (let testArray in results.result) {
+    let failedTests = Array.from(results.result[testArray]).filter(
+      test => !test.success && !test.skipped
+    );
+
+    let errs = failedTests.map(test => {
+      return `${test.suite.join(":")} ${test.description}: ${test.log[0]}`;
+    });
+
+    errors = errors.concat(errs);
+  }
+
+  logErrors("karma", errors);
+  return failed;
+}
+
+function sasslint() {
+  logStart("sasslint");
+  const { out } = execOut("npm", [
+    "run",
+    "--silent",
+    "lint:sasslint",
+    "--",
+    "--format",
+    "json",
+  ]);
+
+  if (!out.length) {
+    return true;
+  }
+
+  let fileObjects = JSON.parse(out);
+  let filesWithIssues = fileObjects.filter(
+    file => file.warningCount || file.errorCount
+  );
+
+  let errs = [];
+  let errorString;
+  filesWithIssues.forEach(file => {
+    file.messages.forEach(messageObj => {
+      errorString = `${file.filePath}(${messageObj.line}, ${
+        messageObj.column
+      }): ${messageObj.message} (${messageObj.ruleId})`;
+      errs.push(errorString);
+    });
+  });
+
+  const errors = logErrors("sasslint", errs);
+  return errors.length === 0;
+}
+
+const karmaPassed = karma();
+const sasslintPassed = sasslint();
+
+const success = karmaPassed && sasslintPassed;
+
+console.log({
+  karmaPassed,
+  sasslintPassed,
+});
+
+process.exitCode = success ? 0 : 1;
+console.log("CODE", process.exitCode);
--- a/browser/components/newtab/karma.mc.config.js
+++ b/browser/components/newtab/karma.mc.config.js
@@ -43,17 +43,25 @@ module.exports = function(config) {
     frameworks: [
       "chai", // require("chai") require("karma-chai")
       "mocha", // require("mocha") require("karma-mocha")
       "sinon", // require("sinon") require("karma-sinon")
     ],
     reporters: [
       "coverage-istanbul", // require("karma-coverage")
       "mocha", // require("karma-mocha-reporter")
+
+      // for bin/try-runner.js to parse the output easily
+      "json", // require("karma-json-reporter")
     ],
+    jsonReporter: {
+      // So this doesn't get interleaved with other karma output
+      stdout: false,
+      outputFile: path.join("logs", "karma-run-results.json"),
+    },
     coverageIstanbulReporter: {
       reports: ["html", "text-summary"],
       dir: PATHS.coverageReportingPath,
       // This will make karma fail if coverage reporting is less than the minimums here
       thresholds: !isTDD && {
         each: {
           statements: 100,
           lines: 100,
--- a/taskcluster/ci/source-test/node.yml
+++ b/taskcluster/ci/source-test/node.yml
@@ -1,12 +1,43 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 ---
+newtab-unit-tests:
+    description: newtab unit tests
+    platform: linux64/opt
+    treeherder:
+        symbol: node(newtab)
+        kind: test
+        tier: 2
+    worker-type: t-linux-xlarge
+    worker:
+        docker-image: {in-tree: "desktop1604-test"}
+        max-run-time: 1800
+    require-build: true
+    fetches:
+        build:
+            - target.tar.bz2
+    run:
+        using: run-task
+        cache-dotcache: true
+        # We fetch our own Firefox nightly because the version of firefox
+        # currently in the desktop1604-test docker image is too old to
+        # pass our unit tests.
+        cwd: '{checkout}'
+        command: >
+            export FIREFOX_BIN=$MOZ_FETCHES_DIR/firefox/firefox &&
+            cd /builds/worker/checkouts/gecko/browser/components/newtab &&
+            npm install &&
+            node bin/try-runner.js
+    when:
+        files-changed:
+            - "browser/components/newtab/**"
+
 debugger-tests:
     description: devtools debugger unit tests and flow type checks
     platform: linux64/opt
     treeherder:
         symbol: node(debugger)
         kind: test
         tier: 1
     worker-type: t-linux-xlarge