Bug 938019 - Run mochitests from manifests. r=ted, r=gps, a=NPOTB
authorBill McCloskey <wmccloskey@mozilla.com>
Tue, 18 Mar 2014 08:03:51 -0700
changeset 192922 f2d71607466d1559abe1cedb93126bdf8cc6053c
parent 192921 0f85c1a7a5ca79b6acca12cc8c49bd5f73249f39
child 192923 e5d7e5709d98611429fc302befb90948494db043
push id474
push userasasaki@mozilla.com
push dateMon, 02 Jun 2014 21:01:02 +0000
treeherdermozilla-release@967f4cf1b31c [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted, gps, NPOTB
bugs938019
milestone30.0a2
Bug 938019 - Run mochitests from manifests. r=ted, r=gps, a=NPOTB
python/mozbuild/mozbuild/frontend/emitter.py
python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/common.ini
python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/mochitest.ini
python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/moz.build
python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/test_foo.html
python/mozbuild/mozbuild/test/frontend/test_emitter.py
testing/mochitest/mach_commands.py
testing/mochitest/manifestLibrary.js
testing/mochitest/mochitest_options.py
testing/mochitest/runtests.py
testing/mochitest/runtestsb2g.py
testing/mochitest/runtestsremote.py
testing/mozbase/manifestdestiny/manifestparser/manifestparser.py
testing/mozbase/manifestdestiny/tests/no-tests.ini
testing/mozbase/manifestdestiny/tests/test_manifestparser.py
--- a/python/mozbuild/mozbuild/frontend/emitter.py
+++ b/python/mozbuild/mozbuild/frontend/emitter.py
@@ -561,19 +561,22 @@ class TreeMetadataEmitter(LoggingMixin):
                     (mozpath.join(out_dir, test['relpath']), True)
 
                 process_support_files(test)
 
             if not filtered:
                 # If there are no tests, look for support-files under DEFAULT.
                 process_support_files(defaults)
 
-            # We also copy the manifest into the output directory.
-            out_path = mozpath.join(out_dir, mozpath.basename(manifest_path))
-            obj.installs[path] = (out_path, False)
+            # We also copy manifests into the output directory,
+            # including manifests from [include:foo] directives.
+            for mpath in m.manifests():
+                mpath = mozpath.normpath(mpath)
+                out_path = mozpath.join(out_dir, mozpath.basename(mpath))
+                obj.installs[mpath] = (out_path, False)
 
             # Some manifests reference files that are auto generated as
             # part of the build or shouldn't be installed for some
             # reason. Here, we prune those files from the install set.
             # FUTURE we should be able to detect autogenerated files from
             # other build metadata. Once we do that, we can get rid of this.
             for f in defaults.get('generated-files', '').split():
                 # We re-raise otherwise the stack trace isn't informative.
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/common.ini
@@ -0,0 +1,1 @@
+[test_foo.html]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/mochitest.ini
@@ -0,0 +1,4 @@
+[DEFAULT]
+install-to-subdir = subdir
+
+[include:common.ini]
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/moz.build
@@ -0,0 +1,4 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+MOCHITEST_MANIFESTS += ['mochitest.ini']
new file mode 100644
--- /dev/null
+++ b/python/mozbuild/mozbuild/test/frontend/data/test-manifest-install-includes/test_foo.html
@@ -0,0 +1,1 @@
+<html></html>
--- a/python/mozbuild/mozbuild/test/frontend/test_emitter.py
+++ b/python/mozbuild/mozbuild/test/frontend/test_emitter.py
@@ -361,16 +361,34 @@ class TestEmitterBasic(unittest.TestCase
         expected = [
             mozpath.normpath(mozpath.join(o.install_prefix, "subdir/subdir.ini")),
             mozpath.normpath(mozpath.join(o.install_prefix, "subdir/support.txt")),
             mozpath.normpath(mozpath.join(o.install_prefix, "subdir/test_foo.html")),
         ]
         paths = sorted([v[0] for v in o.installs.values()])
         self.assertEqual(paths, expected)
 
+    def test_test_manifest_install_includes(self):
+        """Ensure that any [include:foo.ini] are copied to the objdir."""
+        reader = self.reader('test-manifest-install-includes')
+
+        objs = self.read_topsrcdir(reader)
+        self.assertEqual(len(objs), 1)
+        o = objs[0]
+        self.assertEqual(len(o.installs), 3)
+        self.assertEqual(o.manifest_relpath, "mochitest.ini")
+        self.assertEqual(o.manifest_obj_relpath, "subdir/mochitest.ini")
+        expected = [
+            mozpath.normpath(mozpath.join(o.install_prefix, "subdir/common.ini")),
+            mozpath.normpath(mozpath.join(o.install_prefix, "subdir/mochitest.ini")),
+            mozpath.normpath(mozpath.join(o.install_prefix, "subdir/test_foo.html")),
+        ]
+        paths = sorted([v[0] for v in o.installs.values()])
+        self.assertEqual(paths, expected)
+
     def test_test_manifest_keys_extracted(self):
         """Ensure all metadata from test manifests is extracted."""
         reader = self.reader('test-manifest-keys-extracted')
 
         objs = [o for o in self.read_topsrcdir(reader)
                 if isinstance(o, TestManifest)]
 
         self.assertEqual(len(objs), 6)
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -315,17 +315,16 @@ class MochitestRunner(MozbuildObject):
         options.jsdebugger = jsdebugger
         options.debugOnFailure = debug_on_failure
         options.startAt = start_at
         options.endAt = end_at
         options.e10s = e10s
         options.dumpAboutMemoryAfterTest = dump_about_memory_after_test
         options.dumpDMDAfterTest = dump_dmd_after_test
         options.dumpOutputDirectory = dump_output_directory
-        mozinfo.update({"e10s": e10s}) # for test manifest parsing.
 
         options.failureFile = failure_file_path
         if install_extension != None:
             options.extensionsToInstall = [os.path.join(self.topsrcdir,install_extension)]
 
         for k, v in kwargs.iteritems():
             setattr(options, k, v)
 
--- a/testing/mochitest/manifestLibrary.js
+++ b/testing/mochitest/manifestLibrary.js
@@ -14,17 +14,18 @@ function parseTestManifest(testManifest,
     return;
   }
 
   // For mochitest-chrome and mochitest-browser-chrome harnesses, we 
   // define tests as links[testname] = true.
   // For mochitest-plain, we define lists as an array of testnames.
   for (var obj of testManifest['tests']) {
     var path = obj['path'];
-    if (obj.disabled) {
+    // Note that obj.disabled may be "". We still want to skip in that case.
+    if ("disabled" in obj) {
       dump("TEST-SKIPPED | " + path + " | " + obj.disabled + "\n");
       continue;
     }
     if (params.testRoot != 'tests' && params.testRoot !== undefined) {
       links[params.baseurl + '/' + params.testRoot + '/' + path] = true
     } else {
       paths.push(params.testPrefix + path);
     }
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -399,16 +399,18 @@ class MochitestOptions(optparse.OptionPa
         for option, value in self.mochitest_options:
             self.add_option(*option, **value)
         addCommonOptions(self)
         self.set_usage(self.__doc__)
 
     def verifyOptions(self, options, mochitest):
         """ verify correct options and cleanup paths """
 
+        mozinfo.update({"e10s": options.e10s}) # for test manifest parsing.
+
         if options.app is None:
             if build_obj is not None:
                 options.app = build_obj.get_binary_path()
             else:
                 self.error("could not find the application path, --appname must be specified")
 
         if options.totalChunks is not None and options.thisChunk is None:
             self.error("thisChunk must be specified when totalChunks is specified")
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -357,57 +357,147 @@ class MochitestUtilsMixin(object):
         self.urlOpts.append("debugOnFailure=true")
       if options.dumpOutputDirectory:
         self.urlOpts.append("dumpOutputDirectory=%s" % encodeURIComponent(options.dumpOutputDirectory))
       if options.dumpAboutMemoryAfterTest:
         self.urlOpts.append("dumpAboutMemoryAfterTest=true")
       if options.dumpDMDAfterTest:
         self.urlOpts.append("dumpDMDAfterTest=true")
 
+  def getTestFlavor(self, options):
+    if options.browserChrome:
+      return "browser-chrome"
+    elif options.chrome:
+      return "chrome"
+    elif options.a11y:
+      return "a11y"
+    elif options.webapprtChrome:
+      return "webapprt-chrome"
+    else:
+      return "mochitest"
+
+  # This check can be removed when bug 983867 is fixed.
+  def isTest(self, options, filename):
+    allow_js_css = False
+    if options.browserChrome:
+      allow_js_css = True
+      testPattern = re.compile(r"browser_.+\.js")
+    elif options.chrome or options.a11y:
+      testPattern = re.compile(r"(browser|test)_.+\.(xul|html|js|xhtml)")
+    elif options.webapprtContent:
+      testPattern = re.compile(r"webapprt_")
+    elif options.webapprtChrome:
+      allow_js_css = True
+      testPattern = re.compile(r"browser_")
+    else:
+      testPattern = re.compile(r"test_")
+
+    if not allow_js_css and (".js" in filename or ".css" in filename):
+      return False
+
+    pathPieces = filename.split("/")
+
+    return (testPattern.match(pathPieces[-1]) and
+            not re.search(r'\^headers\^$', filename))
+
+  def getTestPath(self, options):
+    if options.ipcplugins:
+      return "dom/plugins/test"
+    else:
+      return options.testPath
+
+  def getTestRoot(self, options):
+    if options.browserChrome:
+      if options.immersiveMode:
+        return 'metro'
+      return 'browser'
+    elif options.a11y:
+      return 'a11y'
+    elif options.webapprtChrome:
+      return 'webapprtChrome'
+    elif options.chrome:
+      return 'chrome'
+    return self.TEST_PATH
+
+  def buildTestURL(self, options):
+    testHost = "http://mochi.test:8888"
+    testPath = self.getTestPath(options)
+    testURL = "/".join([testHost, self.TEST_PATH, testPath])
+    if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, testPath)) and options.repeat > 0:
+      testURL = "/".join([testHost, self.TEST_PATH, os.path.dirname(testPath)])
+    if options.chrome or options.a11y:
+      testURL = "/".join([testHost, self.CHROME_PATH])
+    elif options.browserChrome:
+      testURL = "about:blank"
+    return testURL
+
   def buildTestPath(self, options):
     """ Build the url path to the specific test harness and test file or directory
         Build a manifest of tests to run and write out a json file for the harness to read
     """
+    manifest = None
+
+    testRoot = self.getTestRoot(options)
+    testRootAbs = os.path.abspath(testRoot)
     if options.manifestFile and os.path.isfile(options.manifestFile):
-      manifest = TestManifest(strict=False)
-      manifest.read(options.manifestFile)
+      manifestFileAbs = os.path.abspath(options.manifestFile)
+      assert manifestFileAbs.startswith(testRootAbs)
+      manifest = TestManifest([options.manifestFile], strict=False)
+    else:
+      masterName = self.getTestFlavor(options) + '.ini'
+      masterPath = os.path.join(testRoot, masterName)
+      if os.path.exists(masterPath):
+        manifest = TestManifest([masterPath], strict=False)
+
+    if manifest:
+      # Python 2.6 doesn't allow unicode keys to be used for keyword
+      # arguments. This gross hack works around the problem until we
+      # rid ourselves of 2.6.
+      info = {}
+      for k, v in mozinfo.info.items():
+        if isinstance(k, unicode):
+          k = k.encode('ascii')
+        info[k] = v
+
       # Bug 883858 - return all tests including disabled tests
-      tests = manifest.active_tests(disabled=True, **mozinfo.info)
-      # We need to ensure we match on a complete directory name matching the
-      # test root, and not a substring somewhere else in the path.
-      test_root = os.path.sep + self.getTestRoot(options) + os.path.sep
+      tests = manifest.active_tests(disabled=True, **info)
       paths = []
+      testPath = self.getTestPath(options)
       for test in tests:
-        tp = test['path'].split(test_root, 1)[1].replace('\\', '/').strip('/')
+        pathAbs = os.path.abspath(test['path'])
+        assert pathAbs.startswith(testRootAbs)
+        tp = pathAbs[len(testRootAbs):].replace('\\', '/').strip('/')
 
         # Filter out tests if we are using --test-path
-        if options.testPath and not tp.startswith(options.testPath):
+        if testPath and not tp.startswith(testPath):
+          continue
+
+        if not self.isTest(options, tp):
+          print 'Warning: %s from manifest %s is not a valid test' % (test['name'], test['manifest'])
           continue
 
         testob = {'path': tp}
         if test.has_key('disabled'):
           testob['disabled'] = test['disabled']
         paths.append(testob)
 
+      # Sort tests so they are run in a deterministic order.
+      def path_sort(ob1, ob2):
+        path1 = ob1['path'].split('/')
+        path2 = ob2['path'].split('/')
+        return cmp(path1, path2)
+
+      paths.sort(path_sort)
+
       # Bug 883865 - add this functionality into manifestDestiny
       with open('tests.json', 'w') as manifestFile:
         manifestFile.write(json.dumps({'tests': paths}))
       options.manifestFile = 'tests.json'
 
-    testHost = "http://mochi.test:8888"
-    testURL = ("/").join([testHost, self.TEST_PATH, options.testPath])
-    if os.path.isfile(os.path.join(self.oldcwd, os.path.dirname(__file__), self.TEST_PATH, options.testPath)) and options.repeat > 0:
-       testURL = ("/").join([testHost, self.TEST_PATH, os.path.dirname(options.testPath)])
-    if options.chrome or options.a11y:
-       testURL = ("/").join([testHost, self.CHROME_PATH])
-    elif options.browserChrome:
-      testURL = "about:blank"
-    elif options.ipcplugins:
-      testURL = ("/").join([testHost, self.TEST_PATH, "dom/plugins/test"])
-    return testURL
+    return self.buildTestURL(options)
 
   def startWebSocketServer(self, options, debuggerInfo):
     """ Launch the websocket server """
     if options.webServer != '127.0.0.1':
       return
 
     self.wsserver = WebSocketServer(options, SCRIPT_DIR, debuggerInfo)
     self.wsserver.start()
@@ -1293,29 +1383,16 @@ class Mochitest(MochitestUtilsMixin):
 
       content += '"' + opt + '": '
       content += jsonString(val)
     content += "}"
 
     with open(os.path.join(options.profilePath, "testConfig.js"), "w") as config:
       config.write(content)
 
-  def getTestRoot(self, options):
-    if (options.browserChrome):
-      if (options.immersiveMode):
-        return 'metro'
-      return 'browser'
-    elif (options.a11y):
-      return 'a11y'
-    elif (options.webapprtChrome):
-      return 'webapprtChrome'
-    elif (options.chrome):
-      return 'chrome'
-    return self.TEST_PATH
-
   def installExtensionFromPath(self, options, path, extensionID = None):
     """install an extension to options.profilePath"""
 
     # TODO: currently extensionID is unused; see
     # https://bugzilla.mozilla.org/show_bug.cgi?id=914267
     # [mozprofile] make extensionID a parameter to install_from_path
     # https://github.com/mozilla/mozbase/blob/master/mozprofile/mozprofile/addons.py#L169
 
--- a/testing/mochitest/runtestsb2g.py
+++ b/testing/mochitest/runtestsb2g.py
@@ -60,16 +60,20 @@ class B2GMochitest(MochitestUtilsMixin):
             self.SERVER_STARTUP_TIMEOUT = 90
 
     def setup_common_options(self, options):
         test_url = self.buildTestPath(options)
         if len(self.urlOpts) > 0:
             test_url += "?" + "&".join(self.urlOpts)
         self.test_script_args.append(test_url)
 
+    def buildTestPath(self, options):
+        # Skip over the manifest building that happens on desktop.
+        return self.buildTestURL(options)
+
     def build_profile(self, options):
         # preferences
         prefs = {}
         for path in self.preferences:
             prefs.update(Preferences.read_prefs(path))
 
         for v in options.extraPrefs:
             thispref = v.split("=", 1)
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -374,16 +374,24 @@ class MochiRemote(Mochitest):
             except devicemanager.DMError:
                 log.error("Automation Error: Unable to copy profile to device.")
                 raise
 
         options.profilePath = self.remoteProfile
         options.logFile = self.localLog
         return retVal
 
+    def buildTestPath(self, options):
+        if options.robocopIni != "":
+            # Skip over manifest building if we just want to run
+            # robocop tests.
+            return self.buildTestURL(options)
+        else:
+            return super(MochiRemote, self).buildTestPath(options)
+
     def installChromeFile(self, filename, options):
         parts = options.app.split('/')
         if (parts[0] == options.app):
           return "NO_CHROME_ON_DROID"
         path = '/'.join(parts[:-1])
         manifest = path + "/chrome/" + os.path.basename(filename)
         try:
             self._dm.pushFile(filename, manifest)
--- a/testing/mozbase/manifestdestiny/manifestparser/manifestparser.py
+++ b/testing/mozbase/manifestdestiny/manifestparser/manifestparser.py
@@ -598,17 +598,19 @@ class ManifestParser(object):
         # return the tests
         return tests
 
     def manifests(self, tests=None):
         """
         return manifests in order in which they appear in the tests
         """
         if tests is None:
-            tests = self.tests
+            # Make sure to return all the manifests, even ones without tests.
+            return self.manifest_defaults.keys()
+
         manifests = []
         for test in tests:
             manifest = test.get('manifest')
             if not manifest:
                 continue
             if manifest not in manifests:
                 manifests.append(manifest)
         return manifests
new file mode 100644
--- /dev/null
+++ b/testing/mozbase/manifestdestiny/tests/no-tests.ini
@@ -0,0 +1,2 @@
+[DEFAULT]
+foo = bar
--- a/testing/mozbase/manifestdestiny/tests/test_manifestparser.py
+++ b/testing/mozbase/manifestdestiny/tests/test_manifestparser.py
@@ -195,10 +195,22 @@ class TestManifestParser(unittest.TestCa
 
         parser = ManifestParser()
         manifest = os.path.join(here, 'just-defaults.ini')
         parser.read(manifest)
         self.assertEqual(len(parser.tests), 0)
         self.assertTrue(manifest in parser.manifest_defaults)
         self.assertEquals(parser.manifest_defaults[manifest]['foo'], 'bar')
 
+    def test_manifest_list(self):
+        """
+        Ensure a manifest with just a DEFAULT section still returns
+        itself from the manifests() method.
+        """
+
+        parser = ManifestParser()
+        manifest = os.path.join(here, 'no-tests.ini')
+        parser.read(manifest)
+        self.assertEqual(len(parser.tests), 0)
+        self.assertTrue(len(parser.manifests()) == 1)
+
 if __name__ == '__main__':
     unittest.main()