Bug 1301984 - Add code coverage to mochitest-plain tests. r=jmaher
authorGreg Mierzwinski <gmierz2@outlook.com>
Mon, 20 Feb 2017 22:55:59 -0500
changeset 373342 1dcde4d265a4a7fdd9ad86bd216fd057670c74cb
parent 373298 073f05a135ccf1a0170f0b5e4fdcf63902ca713e
child 373343 2e2093189f5549660f99d09e0d5519865a2ede4f
push id10863
push userjlorenzo@mozilla.com
push dateMon, 06 Mar 2017 23:02:23 +0000
treeherdermozilla-aurora@0931190cd725 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1301984
milestone54.0a1
Bug 1301984 - Add code coverage to mochitest-plain tests. r=jmaher This patch allows the use of the flag '--jscov-dir-prefix' for mochitest plain tests to enable code coverage collection with the JS Debugger. It also enables the mochitest-plain tests for the linux64-jsdcov build platform. MozReview-Commit-ID: 6RqMEZ1I0D7
taskcluster/ci/test/test-sets.yml
taskcluster/ci/test/tests.yml
testing/mochitest/runtests.py
testing/mochitest/tests/SimpleTest/TestRunner.js
testing/mochitest/tests/SimpleTest/setup.js
testing/mozharness/configs/unittests/linux_unittest.py
testing/mozharness/scripts/desktop_unittest.py
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -91,16 +91,17 @@ ccov-code-coverage-tests:
     - mochitest
     - mochitest-browser-chrome
     - mochitest-devtools-chrome
     - reftest
     - web-platform-tests
     - xpcshell
 
 jsdcov-code-coverage-tests:
+    - mochitest
     - mochitest-browser-chrome
     - mochitest-devtools-chrome
     - xpcshell
 
 ##
 # Test sets still being greened up in various ways
 
 windows-vm-tests:
--- a/taskcluster/ci/test/tests.yml
+++ b/taskcluster/ci/test/tests.yml
@@ -349,38 +349,44 @@ marionette:
                         windows.*:
                             - marionette/windows_taskcluster_config.py
                         default:
                             - marionette/prod_config.py
                             - remove_executables.py
 
 mochitest:
     description: "Mochitest plain run"
-    suite: mochitest/plain-chunked
+    suite:
+        by-test-platform:
+            linux64-jsdcov/opt: mochitest/plain-chunked-coverage
+            default: mochitest/plain-chunked
     treeherder-symbol: tc-M()
     loopback-video: true
     instance-size:
         by-test-platform:
+            linux64-jsdcov/opt: xlarge
             android.*: xlarge
             default: legacy # Bug 1281241: migrating to m3.large instances
     chunks:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 32
             android.*: 20
             macosx.*: 5
             windows.*: 5
             linux.*: 10
     e10s:
         by-test-platform:
             linux64-ccov/opt: false
+            linux64-jsdcov/opt: false
             android.*: false
             default: both
     max-run-time:
         by-test-platform:
             android-4.3-arm7-api-15/debug: 10800
+            linux64-jsdcov/opt: 10800
             default: 5400
     allow-software-gl-layers: false
     mozharness:
         by-test-platform:
             android.*:
                 script: android_emulator_unittest.py
                 no-read-buildbot-config: true
                 config:
@@ -397,17 +403,21 @@ mochitest:
                             - unittests/win_taskcluster_unittest.py
                         macosx.*:
                             - remove_executables.py
                             - unittests/mac_unittest.py
                         linux.*:
                             - unittests/linux_unittest.py
                             - remove_executables.py
                 extra-options:
-                    - --mochitest-suite=plain-chunked
+                    by-test-platform:
+                        linux64-jsdcov/opt:
+                            - --mochitest-suite=plain-chunked-coverage
+                        default:
+                            - --mochitest-suite=plain-chunked
 
 mochitest-a11y:
     description: "Mochitest a11y run"
     suite: mochitest/a11y
     treeherder-symbol: tc-M(a11y)
     loopback-video: true
     e10s: false
     docker-image: {"in-tree": "desktop1604-test"}
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -973,16 +973,18 @@ class MochitestDesktop(object):
                     encodeURIComponent(
                         options.dumpOutputDirectory))
             if options.dumpAboutMemoryAfterTest:
                 self.urlOpts.append("dumpAboutMemoryAfterTest=true")
             if options.dumpDMDAfterTest:
                 self.urlOpts.append("dumpDMDAfterTest=true")
             if options.debugger:
                 self.urlOpts.append("interactiveDebugger=true")
+            if options.jscov_dir_prefix:
+                self.urlOpts.append("jscovDirPrefix=%s" % options.jscov_dir_prefix)
 
     def normflavor(self, flavor):
         """
         In some places the string 'browser-chrome' is expected instead of
         'browser' and 'mochitest' instead of 'plain'. Normalize the flavor
         strings for those instances.
         """
         # TODO Use consistent flavor strings everywhere and remove this
--- a/testing/mochitest/tests/SimpleTest/TestRunner.js
+++ b/testing/mochitest/tests/SimpleTest/TestRunner.js
@@ -103,16 +103,22 @@ TestRunner._structuredFormatter = new St
 
 /**
  * Make sure the tests don't hang indefinitely.
 **/
 TestRunner._numTimeouts = 0;
 TestRunner._currentTestStartTime = new Date().valueOf();
 TestRunner._timeoutFactor = 1;
 
+/**
+* Used to collect code coverage with the js debugger.
+*/
+TestRunner.jscovDirPrefix = "";
+var coverageCollector = {};
+
 TestRunner._checkForHangs = function() {
   function reportError(win, msg) {
     if ("SimpleTest" in win) {
       win.SimpleTest.ok(false, msg);
     } else if ("W3CTest" in win) {
       win.W3CTest.logFailure(msg);
     }
   }
@@ -347,16 +353,22 @@ TestRunner.getParameterInfo = function()
  *
 **/
 TestRunner.runTests = function (/*url...*/) {
     TestRunner.structuredLogger.info("SimpleTest START");
     TestRunner.originalTestURL = $("current-test").innerHTML;
 
     SpecialPowers.registerProcessCrashObservers();
 
+    // Initialize code coverage
+    if (TestRunner.jscovDirPrefix != "") {
+        var CoverageCollector = SpecialPowers.Cu.import("resource://testing-common/CoverageUtils.jsm", {}).CoverageCollector;
+        coverageCollector = new CoverageCollector(TestRunner.jscovDirPrefix);
+    }
+
     TestRunner._urls = flattenArguments(arguments);
 
     var singleTestRun = this._urls.length <= 1 && TestRunner.repeat <= 1;
     TestRunner.showTestReport = singleTestRun;
     var frame = $('testframe');
     frame.src = "";
     if (singleTestRun) {
         // Can't use document.body because this runs in a XUL doc as well...
@@ -476,18 +488,22 @@ TestRunner.runNextTest = function() {
           // Loops are finished
           if (TestRunner.logEnabled) {
             TestRunner.structuredLogger.info("TEST-INFO | Ran " + TestRunner._currentLoop + " Loops");
             TestRunner.structuredLogger.info("SimpleTest FINISHED");
           }
 
           if (TestRunner.onComplete)
             TestRunner.onComplete();
-       }
-       TestRunner.generateFailureList();
+        }
+        TestRunner.generateFailureList();
+
+        if (TestRunner.jscovDirPrefix != "") {
+          coverageCollector.finalize();
+        }
     }
 };
 
 TestRunner.expectChildProcessCrash = function() {
     TestRunner._expectingProcessCrash = true;
 };
 
 /**
@@ -500,16 +516,21 @@ TestRunner.testFinished = function(tests
         !TestRunner._loopIsRestarting) {
         TestRunner.structuredLogger.testEnd(TestRunner.currentTestURL,
                                             "ERROR",
                                             "OK",
                                             "called finish() multiple times");
         TestRunner.updateUI([{ result: false }]);
         return;
     }
+
+    if (TestRunner.jscovDirPrefix != "") {
+        coverageCollector.recordTestCoverage(TestRunner.currentTestURL);
+    }
+
     TestRunner._lastTestFinished = TestRunner._currentTest;
     TestRunner._loopIsRestarting = false;
 
     // TODO : replace this by a function that returns the mem data as an object
     // that's dumped later with the test_end message
     MemoryStats.dump(TestRunner._currentTest,
                      TestRunner.currentTestURL,
                      TestRunner.dumpOutputDirectory,
--- a/testing/mochitest/tests/SimpleTest/setup.js
+++ b/testing/mochitest/tests/SimpleTest/setup.js
@@ -139,16 +139,20 @@ if (params.dumpAboutMemoryAfterTest) {
 if (params.dumpDMDAfterTest) {
   TestRunner.dumpDMDAfterTest = true;
 }
 
 if (params.interactiveDebugger) {
   TestRunner.interactiveDebugger = true;
 }
 
+if (params.jscovDirPrefix) {
+  TestRunner.jscovDirPrefix = params.jscovDirPrefix;
+}
+
 if (params.maxTimeouts) {
   TestRunner.maxTimeouts = params.maxTimeouts;
 }
 
 // Log things to the console if appropriate.
 TestRunner.logger.addListener("dumpListener", consoleLevel + "", function(msg) {
   dump(msg.info.join(' ') + "\n");
 });
--- a/testing/mozharness/configs/unittests/linux_unittest.py
+++ b/testing/mozharness/configs/unittests/linux_unittest.py
@@ -178,16 +178,17 @@ config = {
         "valgrind-plain": ["--valgrind=/usr/bin/valgrind",
                            "--valgrind-supp-files=" + VALGRIND_SUPP_ARCH +
                                "," + VALGRIND_SUPP_CROSS_ARCH,
                            "--timeout=900", "--max-timeouts=50"],
         "plain": [],
         "plain-gpu": ["--subsuite=gpu"],
         "plain-clipboard": ["--subsuite=clipboard"],
         "plain-chunked": ["--chunk-by-dir=4"],
+        "plain-chunked-coverage": ["--chunk-by-dir=4", "--timeout=1200"],
         "mochitest-media": ["--subsuite=media"],
         "chrome": ["--flavor=chrome"],
         "chrome-gpu": ["--flavor=chrome", "--subsuite=gpu"],
         "chrome-clipboard": ["--flavor=chrome", "--subsuite=clipboard"],
         "chrome-chunked": ["--flavor=chrome", "--chunk-by-dir=4"],
         "browser-chrome": ["--flavor=browser"],
         "browser-chrome-gpu": ["--flavor=browser", "--subsuite=gpu"],
         "browser-chrome-clipboard": ["--flavor=browser", "--subsuite=clipboard"],
--- a/testing/mozharness/scripts/desktop_unittest.py
+++ b/testing/mozharness/scripts/desktop_unittest.py
@@ -394,17 +394,18 @@ class DesktopUnittest(TestingMixin, Merc
 
             # set pluginsPath
             abs_res_plugins_dir = os.path.join(abs_res_dir, 'plugins')
             str_format_values['test_plugin_path'] = abs_res_plugins_dir
 
             if suite_category not in c["suite_definitions"]:
                 self.fatal("'%s' not defined in the config!")
 
-            if suite in ('browser-chrome-coverage', 'xpcshell-coverage', 'mochitest-devtools-chrome-coverage'):
+            if suite in ('browser-chrome-coverage', 'xpcshell-coverage',
+                         'mochitest-devtools-chrome-coverage', 'plain-chunked-coverage'):
                 base_cmd.append('--jscov-dir-prefix=%s' %
                                 dirs['abs_blob_upload_dir'])
 
             options = c["suite_definitions"][suite_category]["options"]
             if options:
                 for option in options:
                     option = option % str_format_values
                     if not option.endswith('None'):