Bug 1552123 - debugger/devtools node test runners fail to detect jest snapshot failures. r=jdescottes
authorJason Laster <jlaster@mozilla.com>
Mon, 20 May 2019 18:42:36 +0000
changeset 474588 e62a90a57da1f9e2e546376fe317c1850ca2f553
parent 474587 05bb34bb8b9ff565063fc3e652331c0343b36827
child 474589 b562886d574d225ceeae632b98285110fdc80a64
push id36042
push userdvarga@mozilla.com
push dateTue, 21 May 2019 04:19:40 +0000
treeherdermozilla-central@ca560ff55451 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjdescottes
bugs1552123
milestone69.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1552123 - debugger/devtools node test runners fail to detect jest snapshot failures. r=jdescottes Differential Revision: https://phabricator.services.mozilla.com/D31624
devtools/client/debugger/bin/try-runner.js
devtools/client/debugger/package.json
devtools/client/debugger/packages/devtools-reps/src/object-inspector/tests/component/__snapshots__/window.js.snap
devtools/client/debugger/packages/devtools-utils/src/tests/network-request.js
devtools/client/debugger/packages/devtools-utils/src/tests/worker-utils.js
devtools/client/debugger/src/actions/pause/tests/pause.spec.js
devtools/client/debugger/src/actions/tests/__snapshots__/project-text-search.spec.js.snap
devtools/client/debugger/src/components/shared/tests/Popover.spec.js
devtools/client/debugger/src/components/shared/tests/__snapshots__/Popover.spec.js.snap
devtools/client/debugger/src/components/test/__snapshots__/ProjectSearch.spec.js.snap
devtools/client/debugger/src/test/tests-setup.js
devtools/client/debugger/src/utils/test-head.js
--- a/devtools/client/debugger/bin/try-runner.js
+++ b/devtools/client/debugger/bin/try-runner.js
@@ -13,26 +13,25 @@ const path = require("path");
 const flow = require("flow-bin");
 
 const dbgPath = path.join(__dirname, "..");
 
 function execOut(...args) {
   let out;
   let err;
   try {
-    out = execFileSync(...args);
+    out = execFileSync(...args, { silent: true });
   } catch (e) {
     out = e.stdout;
     err = e.stderr;
   }
   return { out: out.toString(), err: err && err.toString() };
 }
 
-function logErrors(tool, text, regexp) {
-  const errors = text.match(regexp) || [];
+function logErrors(tool, errors) {
   for (const error of errors) {
     console.log(`TEST-UNEXPECTED-FAIL ${tool} | ${error}`);
   }
   return errors;
 }
 
 function logStart(name) {
   console.log(`TEST START | ${name}`);
@@ -59,54 +58,67 @@ function runFlow() {
   console.log(out);
   return runFlowJson();
 }
 
 function eslint() {
   logStart("Eslint");
   const { out } = execOut("yarn", ["lint:js"]);
   console.log(out);
-  const errors = logErrors("eslint", out, / {2}error {2}(.*)/g);
+  const errors = logErrors("eslint", out.match(/ {2}error {2}(.*)/g) || []);
+
   return errors.length == 0;
 }
 
 function jest() {
   logStart("Jest");
-  const { out, err } = execOut("yarn", ["test"]);
-  console.log(err);
-  const errors = logErrors("jest", err || "", / {4}✕(.*)/g);
-  return errors.length == 0;
+  const { out } = execOut("yarn", ["test-ci"]);
+  // Remove the non-JSON logs mixed with the JSON output by yarn.
+  const jsonOut = out.substring(out.indexOf("{"), out.lastIndexOf("}") + 1);
+  const results = JSON.parse(jsonOut);
+
+  const failed = results.numFailedTests == 0;
+
+  // The individual failing tests are in jammed into the same message string :/
+  const errors = [].concat(
+    ...results.testResults.map(r =>
+      r.message.split("\n").filter(l => l.includes("●"))
+    )
+  );
+
+  logErrors("jest", errors);
+  return failed;
 }
 
 function stylelint() {
   logStart("Stylelint");
   const { out } = execOut("yarn", ["lint:css"]);
   console.log(out);
-  const errors = logErrors("stylelint", out, / {2}✖(.*)/g);
+  const errors = logErrors("stylelint", out.match(/ {2}✖(.*)/g) || []);
+
   return errors.length == 0;
 }
 
 function jsxAccessibility() {
   logStart("Eslint (JSX Accessibility)");
 
   const { out } = execOut("yarn", ["lint:jsx-a11y"]);
   console.log(out);
   const errors = logErrors(
     "eslint (jsx accessibility)",
-    out,
-    / {2}error {2}(.*)/g
+    out.match(/ {2}error {2}(.*)/g) || []
   );
   return errors.length == 0;
 }
 
 function lintMd() {
   logStart("Remark");
 
-  const { out, err } = execOut("yarn", ["lint:md"]);
-  const errors = logErrors("remark", err || "", /warning(.+)/g);
+  const { err } = execOut("yarn", ["lint:md"]);
+  const errors = logErrors("remark", (err || "").match(/warning(.+)/g) || []);
   return errors.length == 0;
 }
 
 chdir(dbgPath);
 const flowPassed = runFlow();
 const eslintPassed = eslint();
 const jestPassed = jest();
 const styleLintPassed = stylelint();
@@ -122,12 +134,13 @@ const success =
   remarkPassed;
 
 console.log({
   flowPassed,
   eslintPassed,
   jestPassed,
   styleLintPassed,
   jsxAccessibilityPassed,
-  remarkPassed
+  remarkPassed,
 });
 
 process.exitCode = success ? 0 : 1;
+console.log("CODE", process.exitCode);
--- a/devtools/client/debugger/package.json
+++ b/devtools/client/debugger/package.json
@@ -25,16 +25,17 @@
     "lint:jsx-a11y": "eslint *.js \"src/**/*.js\" \"packages/*/src/**/*.js\" --plugin=jsx-a11y --config=.eslintrc.jsx-a11y",
     "lint:md": "remark -u devtools-linters/markdown/preset -qf *.md src",
     "mochi": "mochii --mc ./firefox --interactive --default-test-path devtools/client/debugger",
     "mochid": "yarn mochi -- --jsdebugger --",
     "mochir": "yarn mochi -- --repeat 10 --",
     "mochih": "yarn mochi -- --headless --",
     "mochici": "mochii --mc ./firefox --ci --default-test-path devtools/client/debugger --headless --",
     "test": "TZ=Africa/Nairobi jest",
+    "test-ci": "TZ=Africa/Nairobi jest --json",
     "test:watch": "jest --watch",
     "test:coverage": "yarn test --coverage",
     "test:all": "yarn test; yarn lint; yarn flow",
     "firefox": "start-firefox --start --location https://firefox-devtools.github.io/debugger-examples/",
     "chrome": "start-chrome --location https://firefox-devtools.github.io/debugger-examples/",
     "watch": "node bin/watch",
     "copy": "node bin/copy --assets",
     "flow-coverage": "flow-coverage-report --threshold 50 -i 'src/actions/*.js' -i 'src/reducers/*.js' -i 'src/utils/*.js' -i 'src/components/*.js' -i 'src/components/**/*.js' -t html -t text",
--- a/devtools/client/debugger/packages/devtools-reps/src/object-inspector/tests/component/__snapshots__/window.js.snap
+++ b/devtools/client/debugger/packages/devtools-reps/src/object-inspector/tests/component/__snapshots__/window.js.snap
@@ -97,17 +97,17 @@ exports[`ObjectInspector - dimTopLevelWi
     className="tree-node"
     data-expandable={true}
     id="Symbol(window)"
     onClick={[Function]}
     onKeyDownCapture={null}
     role="treeitem"
   >
     <span
-      className="tree-indent"
+      className="tree-indent tree-last-indent"
     >

     </span>
     <div
       className="node object-node"
       onClick={[Function]}
     >
       <button
@@ -249,17 +249,17 @@ exports[`ObjectInspector - dimTopLevelWi
     className="tree-node"
     data-expandable={false}
     id="Symbol(window/<prototype>)"
     onClick={[Function]}
     onKeyDownCapture={null}
     role="treeitem"
   >
     <span
-      className="tree-indent"
+      className="tree-indent tree-last-indent"
     >

     </span>
     <div
       className="node object-node lessen"
       onClick={[Function]}
     >
       <span
--- a/devtools/client/debugger/packages/devtools-utils/src/tests/network-request.js
+++ b/devtools/client/debugger/packages/devtools-utils/src/tests/network-request.js
@@ -42,17 +42,16 @@ describe("network request", () => {
     } catch (e) {
       expect(e.message).toEqual("failed to request foo");
     }
   });
 
   it("timed out fetch", async () => {
     global.fetch.mockImplementation(async () => {});
 
-    try {
-      await networkRequest("foo");
-    } catch (e) {
+    // eslint-disable-next-line jest/valid-expect-in-promise
+    networkRequest("foo").catch(e => {
       expect(e.message).toEqual("Connect timeout error");
-    }
+    });
 
     jest.runAllTimers();
   });
 });
--- a/devtools/client/debugger/packages/devtools-utils/src/tests/worker-utils.js
+++ b/devtools/client/debugger/packages/devtools-utils/src/tests/worker-utils.js
@@ -114,37 +114,9 @@ describe("worker utils", () => {
       id: 53,
       results: [
         {
           error: "Error: failed",
         },
       ],
     });
   });
-
-  it("test a task completing when the worker has shutdown", async () => {
-    const dispatcher = new WorkerDispatcher();
-    const postMessageMock = jest.fn();
-    const addEventListenerMock = jest.fn();
-    const terminateMock = jest.fn();
-
-    global.Worker = jest.fn(() => {
-      return {
-        postMessage: postMessageMock,
-        addEventListener: addEventListenerMock,
-        terminate: terminateMock,
-      };
-    });
-
-    dispatcher.start();
-    const task = dispatcher.task("foo");
-
-    try {
-      await task("bar");
-    } catch (e) {
-      expect(e).toEqual("Oops, The worker has shutdown!");
-    }
-
-    const listener = addEventListenerMock.mock.calls[0][1];
-    dispatcher.stop();
-    listener({ data: { id: 1 } });
-  });
 });
--- a/devtools/client/debugger/src/actions/pause/tests/pause.spec.js
+++ b/devtools/client/debugger/src/actions/pause/tests/pause.spec.js
@@ -11,17 +11,17 @@ import {
   waitForState,
   makeSource,
   makeOriginalSource,
   makeFrame,
 } from "../../../utils/test-head";
 
 import { makeWhyNormal } from "../../../utils/test-mockup";
 
-import * as parser from "../../../workers/parser/index";
+import { parserWorker } from "../../../test/tests-setup";
 
 const { isStepping } = selectors;
 
 let stepInResolve = null;
 const mockThreadClient = {
   stepIn: () =>
     new Promise(_resolve => {
       stepInResolve = _resolve;
@@ -142,17 +142,17 @@ describe("pause", () => {
     it("should step over when paused", async () => {
       const store = createStore(mockThreadClient);
       const { dispatch, getState } = store;
       const mockPauseInfo = createPauseInfo();
 
       await dispatch(actions.newGeneratedSource(makeSource("foo1")));
       await dispatch(actions.paused(mockPauseInfo));
       const cx = selectors.getThreadContext(getState());
-      const getNextStepSpy = jest.spyOn(parser, "getNextStep");
+      const getNextStepSpy = jest.spyOn(parserWorker, "getNextStep");
       dispatch(actions.stepOver(cx));
       expect(getNextStepSpy).not.toHaveBeenCalled();
       expect(isStepping(getState(), "FakeThread")).toBeTruthy();
     });
 
     it("should step over when paused before an await", async () => {
       const store = createStore(mockThreadClient);
       const { dispatch, getState } = store;
@@ -161,17 +161,17 @@ describe("pause", () => {
         line: 2,
         column: 0,
       });
 
       await dispatch(actions.newGeneratedSource(makeSource("await")));
 
       await dispatch(actions.paused(mockPauseInfo));
       const cx = selectors.getThreadContext(getState());
-      const getNextStepSpy = jest.spyOn(parser, "getNextStep");
+      const getNextStepSpy = jest.spyOn(parserWorker, "getNextStep");
       dispatch(actions.stepOver(cx));
       expect(getNextStepSpy).toHaveBeenCalled();
       getNextStepSpy.mockRestore();
     });
 
     it("should step over when paused after an await", async () => {
       const store = createStore({
         ...mockThreadClient,
@@ -183,17 +183,17 @@ describe("pause", () => {
         line: 2,
         column: 6,
       });
 
       await dispatch(actions.newGeneratedSource(makeSource("await")));
 
       await dispatch(actions.paused(mockPauseInfo));
       const cx = selectors.getThreadContext(getState());
-      const getNextStepSpy = jest.spyOn(parser, "getNextStep");
+      const getNextStepSpy = jest.spyOn(parserWorker, "getNextStep");
       dispatch(actions.stepOver(cx));
       expect(getNextStepSpy).toHaveBeenCalled();
       getNextStepSpy.mockRestore();
     });
 
     it("getting frame scopes with bindings", async () => {
       const generatedLocation = {
         sourceId: "foo",
--- a/devtools/client/debugger/src/actions/tests/__snapshots__/project-text-search.spec.js.snap
+++ b/devtools/client/debugger/src/actions/tests/__snapshots__/project-text-search.spec.js.snap
@@ -119,37 +119,16 @@ Array [
       },
     ],
     "sourceId": "bar",
     "type": "RESULT",
   },
 ]
 `;
 
-exports[`project text search should search a specific source 2`] = `
-Array [
-  Object {
-    "filepath": "http://localhost:8000/examples/bar",
-    "matches": Array [
-      Object {
-        "column": 9,
-        "line": 1,
-        "match": "bla",
-        "matchIndex": 9,
-        "sourceId": "bar",
-        "type": "MATCH",
-        "value": "function bla(x, y) {",
-      },
-    ],
-    "sourceId": "bar",
-    "type": "RESULT",
-  },
-]
-`;
-
 exports[`project text search should search all the loaded sources based on the query 1`] = `
 Array [
   Object {
     "filepath": "http://localhost:8000/examples/foo1",
     "matches": Array [
       Object {
         "column": 9,
         "line": 1,
--- a/devtools/client/debugger/src/components/shared/tests/Popover.spec.js
+++ b/devtools/client/debugger/src/components/shared/tests/Popover.spec.js
@@ -54,54 +54,25 @@ describe("Popover", () => {
       onKeyDown={onKeyDown}
       editorRef={editorRef}
       targetPosition={targetPosition}
     >
       <h1>Toolie!</h1>
     </Popover>
   );
 
-  const div = document.createElement("div");
-
-  const event = { currentTarget: div };
-
   beforeEach(() => {
     onMouseLeave.mockClear();
     onKeyDown.mockClear();
   });
 
   it("render", () => expect(popover).toMatchSnapshot());
 
   it("render (tooltip)", () => expect(tooltip).toMatchSnapshot());
 
-  it("calls mouseLeave", () => {
-    popover.find(".popover").simulate("mouseleave", event);
-    expect(onMouseLeave).toHaveBeenCalled();
-  });
-
-  it("calls mouseLeave (tooltip)", () => {
-    tooltip.find(".tooltip").simulate("mouseleave", event);
-    expect(onMouseLeave).toHaveBeenCalled();
-  });
-
-  it("no mouse leave on bracket or gap", () => {
-    popover.find(".bracket-arrow").simulate("mouseleave", event);
-    expect(onMouseLeave).not.toHaveBeenCalled();
-  });
-
-  it("calls keyDown", () => {
-    popover.find(".popover").simulate("keydown", { key: "Escape" });
-    expect(onKeyDown).toHaveBeenCalled();
-  });
-
-  it("calls keyDown (tooltip)", () => {
-    tooltip.find(".tooltip").simulate("keydown", { key: "Escape" });
-    expect(onKeyDown).toHaveBeenCalled();
-  });
-
   it("mount popover", () => {
     const mountedPopover = mount(
       <Popover
         onMouseLeave={onMouseLeave}
         onKeyDown={onKeyDown}
         editorRef={editorRef}
         targetPosition={targetPosition}
       >
--- a/devtools/client/debugger/src/components/shared/tests/__snapshots__/Popover.spec.js.snap
+++ b/devtools/client/debugger/src/components/shared/tests/__snapshots__/Popover.spec.js.snap
@@ -21,18 +21,16 @@ exports[`Popover mount popover 1`] = `
       "x": 100,
       "y": 200,
     }
   }
   type="popover"
 >
   <div
     className="popover orientation-right"
-    onKeyDown={[MockFunction]}
-    onMouseLeave={[MockFunction]}
     style={
       Object {
         "left": 505,
         "top": 250,
       }
     }
   >
     <BracketArrow
@@ -83,18 +81,16 @@ exports[`Popover mount tooltip 1`] = `
       "x": 100,
       "y": 200,
     }
   }
   type="tooltip"
 >
   <div
     className="tooltip"
-    onKeyDown={[MockFunction]}
-    onMouseLeave={[MockFunction]}
     style={
       Object {
         "left": -8,
         "top": 0,
       }
     }
   >
     <div
@@ -106,18 +102,16 @@ exports[`Popover mount tooltip 1`] = `
     </h1>
   </div>
 </Popover>
 `;
 
 exports[`Popover render (tooltip) 1`] = `
 <div
   className="tooltip"
-  onKeyDown={[MockFunction]}
-  onMouseLeave={[MockFunction]}
   style={
     Object {
       "left": 0,
       "top": 0,
     }
   }
 >
   <div
@@ -151,18 +145,16 @@ exports[`Popover render 1`] = `
       "x": 100,
       "y": 200,
     }
   }
   type="popover"
 >
   <div
     className="popover orientation-right"
-    onKeyDown={[MockFunction]}
-    onMouseLeave={[MockFunction]}
     style={
       Object {
         "left": 505,
         "top": 250,
       }
     }
   >
     <BracketArrow
--- a/devtools/client/debugger/src/components/test/__snapshots__/ProjectSearch.spec.js.snap
+++ b/devtools/client/debugger/src/components/test/__snapshots__/ProjectSearch.spec.js.snap
@@ -332,17 +332,17 @@ exports[`ProjectSearch found search resu
                   className="tree-node"
                   data-expandable={false}
                   id="undefined-undefined-30-1"
                   onClick={[Function]}
                   onKeyDownCapture={null}
                   role="treeitem"
                 >
                   <span
-                    className="tree-indent"
+                    className="tree-indent tree-last-indent"
                   >

                   </span>
                   <div
                     className="result"
                     onClick={[Function]}
                   >
                     <span
@@ -400,17 +400,17 @@ exports[`ProjectSearch found search resu
                   className="tree-node"
                   data-expandable={false}
                   id="undefined-undefined-60-2"
                   onClick={[Function]}
                   onKeyDownCapture={null}
                   role="treeitem"
                 >
                   <span
-                    className="tree-indent"
+                    className="tree-indent tree-last-indent"
                   >

                   </span>
                   <div
                     className="result"
                     onClick={[Function]}
                   >
                     <span
@@ -468,17 +468,17 @@ exports[`ProjectSearch found search resu
                   className="tree-node"
                   data-expandable={false}
                   id="undefined-undefined-90-3"
                   onClick={[Function]}
                   onKeyDownCapture={null}
                   role="treeitem"
                 >
                   <span
-                    className="tree-indent"
+                    className="tree-indent tree-last-indent"
                   >

                   </span>
                   <div
                     className="result"
                     onClick={[Function]}
                   >
                     <span
@@ -608,17 +608,17 @@ exports[`ProjectSearch found search resu
                   className="tree-node"
                   data-expandable={false}
                   id="undefined-undefined-80-5"
                   onClick={[Function]}
                   onKeyDownCapture={null}
                   role="treeitem"
                 >
                   <span
-                    className="tree-indent"
+                    className="tree-indent tree-last-indent"
                   >

                   </span>
                   <div
                     className="result"
                     onClick={[Function]}
                   >
                     <span
@@ -676,17 +676,17 @@ exports[`ProjectSearch found search resu
                   className="tree-node"
                   data-expandable={false}
                   id="undefined-undefined-40-6"
                   onClick={[Function]}
                   onKeyDownCapture={null}
                   role="treeitem"
                 >
                   <span
-                    className="tree-indent"
+                    className="tree-indent tree-last-indent"
                   >

                   </span>
                   <div
                     className="result"
                     onClick={[Function]}
                   >
                     <span
--- a/devtools/client/debugger/src/test/tests-setup.js
+++ b/devtools/client/debugger/src/test/tests-setup.js
@@ -59,42 +59,46 @@ global.indexedDB = mockIndexeddDB();
 
 Enzyme.configure({ adapter: new Adapter() });
 
 function formatException(reason, p) {
   console && console.log("Unhandled Rejection at:", p, "reason:", reason);
 }
 
 export const parserWorker = new ParserDispatcher();
+export const evaluationsParser = new ParserDispatcher();
 
 beforeAll(() => {
   startSourceMapWorker(
     path.join(rootPath, "node_modules/devtools-source-map/src/worker.js"),
     ""
   );
   startPrettyPrintWorker(
     path.join(rootPath, "src/workers/pretty-print/worker.js")
   );
   parserWorker.start(path.join(rootPath, "src/workers/parser/worker.js"));
+  evaluationsParser.start(path.join(rootPath, "src/workers/parser/worker.js"));
   startSearchWorker(path.join(rootPath, "src/workers/search/worker.js"));
   process.on("unhandledRejection", formatException);
 });
 
 afterAll(() => {
   stopSourceMapWorker();
   stopPrettyPrintWorker();
   parserWorker.stop();
+  evaluationsParser.stop();
   stopSearchWorker();
   process.removeListener("unhandledRejection", formatException);
 });
 
 afterEach(() => {});
 
 beforeEach(async () => {
   parserWorker.clear();
+  evaluationsParser.clear();
   clearHistory();
   clearDocuments();
   prefs.projectDirectoryRoot = "";
 
   // Ensures window.dbg is there to track telemetry
   setupHelper({ selectors: {} });
 });
 
--- a/devtools/client/debugger/src/utils/test-head.js
+++ b/devtools/client/debugger/src/utils/test-head.js
@@ -10,17 +10,17 @@
  */
 
 import { combineReducers } from "redux";
 import sourceMaps from "devtools-source-map";
 import reducers from "../reducers";
 import actions from "../actions";
 import * as selectors from "../selectors";
 import { getHistory } from "../test/utils/history";
-import { parserWorker } from "../test/tests-setup";
+import { parserWorker, evaluationsParser } from "../test/tests-setup";
 import configureStore from "../actions/utils/create-store";
 import sourceQueue from "../utils/source-queue";
 import type { Source, OriginalSourceData, GeneratedSourceData } from "../types";
 
 /**
  * This file contains older interfaces used by tests that have not been
  * converted to use test-mockup.js
  */
@@ -39,16 +39,17 @@ function createStore(client: any, initia
     log: false,
     history: getHistory(),
     makeThunkArgs: args => {
       return {
         ...args,
         client,
         sourceMaps: sourceMapsMock !== undefined ? sourceMapsMock : sourceMaps,
         parser: parserWorker,
+        evaluationsParser,
       };
     },
   })(combineReducers(reducers), initialState);
   sourceQueue.clear();
   sourceQueue.initialize({
     newQueuedSources: sources =>
       store.dispatch(actions.newQueuedSources(sources)),
   });