Bug 996240 - Harness changes for devtools subsuite. r=jmaher, a=NPOTB
authorMichael Ratcliffe <mratcliffe@mozilla.com>
Tue, 25 Mar 2014 16:52:53 +0000
changeset 191900 6680a86d1ebceec2ad44b22db53ee5be28f1ce3c
parent 191899 8d6733e63feda2e0859c979545f6a5f739cde650
child 191901 76622a1dc44e73f8317397ba4589ef75e2b5f72c
push id3503
push userraliiev@mozilla.com
push dateMon, 28 Apr 2014 18:51:11 +0000
treeherdermozilla-beta@c95ac01e332e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher, NPOTB
bugs996240
milestone30.0a2
Bug 996240 - Harness changes for devtools subsuite. r=jmaher, a=NPOTB
python/mozbuild/mozbuild/testing.py
testing/mach_commands.py
testing/mochitest/mach_commands.py
testing/mochitest/mochitest_options.py
testing/mochitest/runtests.py
testing/mozbase/manifestdestiny/manifestparser/manifestparser.py
testing/mozbase/manifestdestiny/tests/test_convert_directory.py
testing/mozbase/manifestdestiny/tests/test_manifestparser.py
testing/testsuite-targets.mk
--- a/python/mozbuild/mozbuild/testing.py
+++ b/python/mozbuild/mozbuild/testing.py
@@ -85,17 +85,19 @@ class TestMetadata(object):
         test's installed dir.
 
         If ``flavor`` is a string, it will be used to filter returned tests
         to only be the flavor specified. A flavor is something like
         ``xpcshell``.
         """
         def fltr(tests):
             for test in tests:
-                if flavor and test.get('flavor') != flavor:
+                if flavor:
+                   if (flavor == 'devtools' and test.get('flavor') != 'browser-chrome') or \
+                      (flavor != 'devtools' and test.get('flavor') != flavor):
                     continue
 
                 if under_path \
                     and not test['file_relpath'].startswith(under_path):
                     continue
 
                 # Make a copy so modifications don't change the source.
                 yield dict(test)
--- a/testing/mach_commands.py
+++ b/testing/mach_commands.py
@@ -70,16 +70,21 @@ TEST_SUITES = {
         'aliases': ('bc', 'BC', 'Bc'),
         'mach_command': 'mochitest-browser',
         'kwargs': {'test_file': None},
     },
     'mochitest-chrome': {
         'mach_command': 'mochitest-chrome',
         'kwargs': {'test_file': None},
     },
+    'mochitest-devtools': {
+        'aliases': ('dt', 'DT', 'Dt'),
+        'mach_command': 'mochitest-browser --subsuite=devtools',
+        'kwargs': {'test_file': None},
+    },
     'mochitest-ipcplugins': {
         'make_target': 'mochitest-ipcplugins',
     },
     'mochitest-plain': {
         'mach_command': 'mochitest-plain',
         'kwargs': {'test_file': None},
     },
     'reftest': {
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -269,27 +269,31 @@ class MochitestRunner(MozbuildObject):
         for handler in remove_handlers:
             logging.getLogger().removeHandler(handler)
 
         runner = mochitest.Mochitest()
 
         opts = mochitest.MochitestOptions()
         options, args = opts.parse_args([])
 
+        options.subsuite = ''
         flavor = suite
 
         # Need to set the suite options before verifyOptions below.
         if suite == 'plain':
             # Don't need additional options for plain.
             flavor = 'mochitest'
         elif suite == 'chrome':
             options.chrome = True
         elif suite == 'browser':
             options.browserChrome = True
             flavor = 'browser-chrome'
+        elif suite == 'devtools':
+            options.browserChrome = True
+            options.subsuite = 'devtools'
         elif suite == 'metro':
             options.immersiveMode = True
             options.browserChrome = True
         elif suite == 'a11y':
             options.a11y = True
         elif suite == 'webapprt-content':
             options.webapprtContent = True
             options.app = self.get_webapp_runtime_path()
@@ -598,16 +602,23 @@ class MachCommands(MachCommandBase):
 
     @Command('mochitest-browser', category='testing',
         conditions=[conditions.is_firefox],
         description='Run a mochitest with browser chrome.')
     @MochitestCommand
     def run_mochitest_browser(self, test_paths, **kwargs):
         return self.run_mochitest(test_paths, 'browser', **kwargs)
 
+    @Command('mochitest-devtools', category='testing',
+        conditions=[conditions.is_firefox],
+        description='Run a devtools mochitest with browser chrome.')
+    @MochitestCommand
+    def run_mochitest_devtools(self, test_paths, **kwargs):
+        return self.run_mochitest(test_paths, 'devtools', **kwargs)
+
     @Command('mochitest-metro', category='testing',
         conditions=[conditions.is_firefox],
         description='Run a mochitest with metro browser chrome.')
     @MochitestCommand
     def run_mochitest_metro(self, test_paths, **kwargs):
         return self.run_mochitest(test_paths, 'metro', **kwargs)
 
     @Command('mochitest-a11y', category='testing',
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -166,16 +166,22 @@ class MochitestOptions(optparse.OptionPa
           "default": "",
         }],
         [["--browser-chrome"],
         { "action": "store_true",
           "dest": "browserChrome",
           "help": "run browser chrome Mochitests",
           "default": False,
         }],
+        [["--subsuite"],
+        { "action": "store",
+          "dest": "subsuite",
+          "help": "subsuite of tests to run",
+          "default": "",
+        }],
         [["--webapprt-content"],
         { "action": "store_true",
           "dest": "webapprtContent",
           "help": "run WebappRT content tests",
           "default": False,
         }],
         [["--webapprt-chrome"],
         { "action": "store_true",
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -303,17 +303,20 @@ class MochitestUtilsMixin(object):
         endAt -- name of test to end at
         timeout -- per-test timeout in seconds
         repeat -- How many times to repeat the test, ie: repeat=1 will run the test twice.
     """
 
     # allow relative paths for logFile
     if options.logFile:
       options.logFile = self.getLogFilePath(options.logFile)
-    if options.browserChrome or options.chrome or options.a11y or options.webapprtChrome:
+
+    # Note that all tests under options.subsuite need to be browser chrome tests.
+    if options.browserChrome or options.chrome or options.subsuite or \
+       options.a11y or options.webapprtChrome:
       self.makeTestConfig(options)
     else:
       if options.autorun:
         self.urlOpts.append("autorun=1")
       if options.timeout:
         self.urlOpts.append("timeout=%d" % options.timeout)
       if options.closeWhenDone:
         self.urlOpts.append("closeWhenDone=1")
@@ -456,19 +459,20 @@ class MochitestUtilsMixin(object):
       # rid ourselves of 2.6.
       info = {}
       for k, v in mozinfo.info.items():
         if isinstance(k, unicode):
           k = k.encode('ascii')
         info[k] = v
 
       # Bug 883858 - return all tests including disabled tests
-      tests = manifest.active_tests(disabled=True, **info)
+      tests = manifest.active_tests(disabled=True, options=options, **info)
       paths = []
       testPath = self.getTestPath(options)
+
       for test in tests:
         pathAbs = os.path.abspath(test['path'])
         assert pathAbs.startswith(testRootAbs)
         tp = pathAbs[len(testRootAbs):].replace('\\', '/').strip('/')
 
         # Filter out tests if we are using --test-path
         if testPath and not tp.startswith(testPath):
           continue
--- a/testing/mozbase/manifestdestiny/manifestparser/manifestparser.py
+++ b/testing/mozbase/manifestdestiny/manifestparser/manifestparser.py
@@ -441,16 +441,19 @@ class ManifestParser(object):
             rootdir = self.rootdir + os.path.sep
 
         # read the configuration
         sections = read_ini(fp=fp, variables=defaults, strict=self.strict)
         self.manifest_defaults[filename] = defaults
 
         # get the tests
         for section, data in sections:
+            subsuite = ''
+            if 'subsuite' in data:
+                subsuite = data['subsuite']
 
             # a file to include
             # TODO: keep track of included file structure:
             # self.manifests = {'manifest.ini': 'relative/path.ini'}
             if section.startswith('include:'):
                 include_file = section.split('include:', 1)[-1]
                 include_file = normalize_path(include_file)
                 if not os.path.isabs(include_file):
@@ -493,16 +496,17 @@ class ManifestParser(object):
                 # When the rootdir is unknown, the relpath needs to be left
                 # unchanged. We use an empty string as rootdir in that case,
                 # which leaves relpath unchanged after slicing.
                 if path.startswith(rootdir):
                     _relpath = path[len(rootdir):]
                 else:
                     _relpath = relpath(path, rootdir)
 
+            test['subsuite'] = subsuite
             test['path'] = path
             test['relpath'] = _relpath
 
             # append the item
             self.tests.append(test)
 
     def read(self, *filenames, **defaults):
         """
@@ -1075,24 +1079,30 @@ class TestManifest(ManifestParser):
                 test.setdefault('disabled', reason)
 
             # mark test as a fail if so indicated
             if fail_tag in test:
                 condition = test[fail_tag]
                 if parse(condition, **values):
                     test['expected'] = 'fail'
 
-    def active_tests(self, exists=True, disabled=True, **values):
+    def active_tests(self, exists=True, disabled=True, options=None, **values):
         """
         - exists : return only existing tests
         - disabled : whether to return disabled tests
         - tags : keys and values to filter on (e.g. `os = linux mac`)
         """
+        tests = [i.copy() for i in self.tests] # shallow copy
 
-        tests = [i.copy() for i in self.tests] # shallow copy
+        # Filter on current subsuite
+        if options:
+            if  options.subsuite:
+                tests = [test for test in tests if options.subsuite == test['subsuite']]
+            else:
+                tests = [test for test in tests if not test['subsuite']]
 
         # mark all tests as passing unless indicated otherwise
         for test in tests:
             test['expected'] = test.get('expected', 'pass')
 
         # ignore tests that do not exist
         if exists:
             tests = [test for test in tests if os.path.exists(test['path'])]
--- a/testing/mozbase/manifestdestiny/tests/test_convert_directory.py
+++ b/testing/mozbase/manifestdestiny/tests/test_convert_directory.py
@@ -42,22 +42,26 @@ class TestDirectoryConversion(unittest.T
         try:
             stub = stub.replace(os.path.sep, "/")
             self.assertTrue(os.path.exists(stub) and os.path.isdir(stub))
 
             # Make a manifest for it
             manifest = convert([stub])
             self.assertEqual(str(manifest),
 """[%(stub)s/bar]
+subsuite = 
 
 [%(stub)s/fleem]
+subsuite = 
 
 [%(stub)s/foo]
+subsuite = 
 
 [%(stub)s/subdir/subfile]
+subsuite = 
 
 """ % dict(stub=stub))
         except:
             raise
         finally:
             shutil.rmtree(stub) # cleanup
 
     def test_convert_directory_manifests_in_place(self):
--- a/testing/mozbase/manifestdestiny/tests/test_manifestparser.py
+++ b/testing/mozbase/manifestdestiny/tests/test_manifestparser.py
@@ -100,17 +100,17 @@ class TestManifestParser(unittest.TestCa
 
         # All of the included tests actually exist:
         self.assertEqual([i['name'] for i in parser.missing()], [])
 
         # Write the output to a manifest:
         buffer = StringIO()
         parser.write(fp=buffer, global_kwargs={'foo': 'bar'})
         self.assertEqual(buffer.getvalue().strip(),
-                         '[DEFAULT]\nfoo = bar\n\n[fleem]\n\n[include/flowers]\nblue = ocean\nred = roses\nyellow = submarine')
+                         '[DEFAULT]\nfoo = bar\n\n[fleem]\nsubsuite = \n\n[include/flowers]\nblue = ocean\nred = roses\nsubsuite = \nyellow = submarine')
 
     def test_copy(self):
         """Test our ability to copy a set of manifests"""
 
         tempdir = tempfile.mkdtemp()
         include_example = os.path.join(here, 'include-example.ini')
         manifest = ManifestParser(manifests=(include_example,))
         manifest.copy(tempdir)
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -15,17 +15,17 @@ endif
 
 # include automation-build.mk to get the path to the binary
 TARGET_DEPTH = $(DEPTH)
 include $(topsrcdir)/build/binary-location.mk
 
 SYMBOLS_PATH := --symbols-path=$(DIST)/crashreporter-symbols
 
 # Usage: |make [TEST_PATH=...] [EXTRA_TEST_ARGS=...] mochitest*|.
-MOCHITESTS := mochitest-plain mochitest-chrome mochitest-a11y mochitest-ipcplugins
+MOCHITESTS := mochitest-plain mochitest-chrome mochitest-devtools mochitest-a11y mochitest-ipcplugins
 mochitest:: $(MOCHITESTS)
 
 ifndef TEST_PACKAGE_NAME
 TEST_PACKAGE_NAME := $(ANDROID_PACKAGE_NAME)
 endif
 
 RUN_MOCHITEST_B2G_DESKTOP = \
   rm -f ./$@.log && \
@@ -140,16 +140,20 @@ mochitest-1 mochitest-2 mochitest-3 moch
 	echo 'mochitest: $* / 5'
 	$(RUN_MOCHITEST) --chunk-by-dir=4 --total-chunks=5 --this-chunk=$*
 	$(CHECK_TEST_ERROR)
 
 mochitest-chrome:
 	$(RUN_MOCHITEST) --chrome
 	$(CHECK_TEST_ERROR)
 
+mochitest-devtools:
+	$(RUN_MOCHITEST) --subsuite=devtools
+	$(CHECK_TEST_ERROR)
+
 mochitest-a11y:
 	$(RUN_MOCHITEST) --a11y
 	$(CHECK_TEST_ERROR)
 
 mochitest-ipcplugins:
 ifeq (Darwin,$(OS_ARCH))
 ifeq (i386,$(TARGET_CPU))
 	$(RUN_MOCHITEST) --setpref=dom.ipc.plugins.enabled.i386.test.plugin=false $(IPCPLUGINS_PATH_ARG)
@@ -533,16 +537,17 @@ stage-marionette: make-stage-dir
           | (cd $(MARIONETTE_DIR)/tests && tar -xf -)
 
 stage-mozbase: make-stage-dir
 	$(MAKE) -C $(DEPTH)/testing/mozbase stage-package
 .PHONY: \
   mochitest \
   mochitest-plain \
   mochitest-chrome \
+  mochitest-devtools \
   mochitest-a11y \
   mochitest-ipcplugins \
   reftest \
   crashtest \
   xpcshell-tests \
   jstestbrowser \
   package-tests \
   make-stage-dir \