Bug 1328830 - Add ability to set prefs from the DEFAULT section of a mochitest manifest, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 20 Jul 2017 09:31:32 -0400
changeset 642659 687cced20fd6ed73b7015c79c37bc2065231d131
parent 642658 fbf29cfa4717abfd141cfa7503d69a0ccee1fc2a
child 642660 5c21cf82263e673720d3bef9b827ae963cc6c2e3
push id72833
push userbmo:emilio+bugs@crisal.io
push dateTue, 08 Aug 2017 16:50:16 +0000
reviewersjmaher
bugs1328830
milestone57.0a1
Bug 1328830 - Add ability to set prefs from the DEFAULT section of a mochitest manifest, r=jmaher This will only work if runByManifest is enabled, otherwise the harness will error out. It's also illegal to set this on an individual test, it must be on the entire manifest. MozReview-Commit-ID: LWYa3Sk1uyW
testing/mochitest/runtests.py
testing/mochitest/tests/python/conftest.py
testing/mochitest/tests/python/python.ini
testing/mochitest/tests/python/test_get_active_tests.py
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -830,16 +830,17 @@ class MochitestDesktop(object):
     def __init__(self, flavor, logger_options, quiet=False):
         update_mozinfo()
         self.flavor = flavor
         self.server = None
         self.wsserver = None
         self.websocketProcessBridge = None
         self.sslTunnel = None
         self.tests_by_manifest = defaultdict(list)
+        self.prefs_by_manifest = defaultdict(set)
         self._active_tests = None
         self._locations = None
 
         self.marionette = None
         self.start_script = None
         self.mozLogs = None
         self.start_script_kwargs = {}
         self.urlOpts = []
@@ -1478,32 +1479,49 @@ toolbar#nav-bar {
             if not self.isTest(options, tp):
                 self.log.warning(
                     'Warning: %s from manifest %s is not a valid test' %
                     (test['name'], test['manifest']))
                 continue
 
             manifest_relpath = os.path.relpath(test['manifest'], manifest_root)
             self.tests_by_manifest[manifest_relpath].append(tp)
+            self.prefs_by_manifest[manifest_relpath].add(test.get('prefs'))
+
+            if 'prefs' in test and not options.runByManifest:
+                self.log.error("parsing {}: runByManifest mode must be enabled to "
+                               "set the `prefs` key".format(manifest_relpath))
+                sys.exit(1)
 
             testob = {'path': tp, 'manifest': manifest_relpath}
             if 'disabled' in test:
                 testob['disabled'] = test['disabled']
             if 'expected' in test:
                 testob['expected'] = test['expected']
             if 'scheme' in test:
                 testob['scheme'] = test['scheme']
             if options.failure_pattern_file:
                 pat_file = os.path.join(os.path.dirname(test['manifest']),
                                         options.failure_pattern_file)
                 patterns = self.getFailurePatterns(pat_file, test['name'])
                 if patterns:
                     testob['expected'] = patterns
             paths.append(testob)
 
+        # The 'prefs' key needs to be set in the DEFAULT section, unfortunately
+        # we can't tell what comes from DEFAULT or not. So to validate this, we
+        # stash all prefs from tests in the same manifest into a set. If the
+        # length of the set > 1, then we know 'prefs' didn't come from DEFAULT.
+        pref_not_default = [m for m, p in self.prefs_by_manifest.iteritems() if len(p) > 1]
+        if pref_not_default:
+            self.log.error("The 'prefs' key must be set in the DEFAULT section of a "
+                           "manifest. Fix the following manifests: {}".format(
+                            '\n'.join(pref_not_default)))
+            sys.exit(1)
+
         def path_sort(ob1, ob2):
             path1 = ob1['path'].split('/')
             path2 = ob2['path'].split('/')
             return cmp(path1, path2)
 
         paths.sort(path_sort)
         if options.dump_tests:
             options.dump_tests = os.path.expanduser(options.dump_tests)
@@ -2438,23 +2456,32 @@ toolbar#nav-bar {
 
         # Until we have all green, this does not run on jetpack*, or a11y (for perf reasons)
         if not options.runByManifest:
             return self.runMochitests(options, [t['path'] for t in tests])
 
         # code for --run-by-manifest
         manifests = set(t['manifest'] for t in tests)
         result = 1  # default value, if no tests are run.
+        origPrefs = options.extraPrefs[:]
         for m in sorted(manifests):
             self.log.info("Running manifest: {}".format(m))
-            tests_in_manifest = [t['path'] for t in tests if t['manifest'] == m]
+
+            prefs = self.prefs_by_manifest[m].pop()
+            options.extraPrefs = origPrefs[:]
+            if prefs:
+                prefs = prefs.strip().split()
+                self.log.info("The following extra prefs will be set:\n  {}".format(
+                    '\n  '.join(prefs)))
+                options.extraPrefs.extend(prefs)
 
             # If we are using --run-by-manifest, we should not use the profile path (if) provided
             # by the user, since we need to create a new directory for each run. We would face
             # problems if we use the directory provided by the user.
+            tests_in_manifest = [t['path'] for t in tests if t['manifest'] == m]
             result = self.runMochitests(options, tests_in_manifest)
 
             # Dump the logging buffer
             self.message_logger.dump_buffered()
 
             if result == -1:
                 break
 
--- a/testing/mochitest/tests/python/conftest.py
+++ b/testing/mochitest/tests/python/conftest.py
@@ -176,16 +176,22 @@ def runtests(setup_harness_root, binary,
 
         result = runtests.run_test_harness(parser, Namespace(**options))
         out = json.loads('[' + ','.join(buf.getvalue().splitlines()) + ']')
         buf.close()
         return result, out
     return inner
 
 
+@pytest.fixture
+def build_obj(setup_harness_root):
+    mochitest_options = pytest.importorskip('mochitest_options')
+    return mochitest_options.build_obj
+
+
 @pytest.fixture(autouse=True)
 def skip_using_mozinfo(request, setup_harness_root):
     """Gives tests the ability to skip based on values from mozinfo.
 
     Example:
         @pytest.mark.skip_mozinfo("!e10s || os == 'linux'")
         def test_foo():
             pass
--- a/testing/mochitest/tests/python/python.ini
+++ b/testing/mochitest/tests/python/python.ini
@@ -1,5 +1,6 @@
 [DEFAULT]
 subsuite = mochitest
 sequential = true
 
 [test_basic_mochitest_plain.py]
+[test_get_active_tests.py]
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/tests/python/test_get_active_tests.py
@@ -0,0 +1,91 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import print_function, unicode_literals
+
+import os
+import sys
+from argparse import Namespace
+
+from manifestparser import TestManifest
+
+import pytest
+
+
+@pytest.fixture
+def get_active_tests(setup_harness_root, parser):
+    runtests = pytest.importorskip('runtests')
+    md = runtests.MochitestDesktop('plain', {'log_tbpl': '-'})
+
+    options = vars(parser.parse_args([]))
+
+    def inner(**kwargs):
+        opts = options.copy()
+        opts.update(kwargs)
+
+        manifest = opts.get('manifestFile')
+        if isinstance(manifest, basestring):
+            md.testRootAbs = os.path.dirname(manifest)
+        elif isinstance(manifest, TestManifest):
+            md.testRootAbs = manifest.rootdir
+
+        md._active_tests = None
+        return md, md.getActiveTests(Namespace(**opts))
+
+    return inner
+
+
+@pytest.fixture
+def create_manifest(tmpdir, build_obj):
+    manifest = tmpdir.join('manifest.ini')
+
+    def inner(string):
+        manifest.write(string)
+        path = unicode(manifest)
+        mobj = TestManifest(manifests=(path,), strict=False)
+        manifest_root = build_obj.topsrcdir if build_obj else mobj.rootdir
+        return os.path.relpath(path, manifest_root), mobj
+    return inner
+
+
+def test_prefs_validation(get_active_tests, create_manifest):
+    manifest_relpath, manifest = create_manifest("""
+[DEFAULT]
+prefs=
+  foo=bar
+  browser.dom.foo=baz
+
+[files/test_pass.html]
+[files/test_fail.html]
+""")
+
+    options = {
+        'runByManifest': True,
+        'manifestFile': manifest,
+    }
+    md, tests = get_active_tests(**options)
+
+    assert len(tests) == 2
+    assert manifest_relpath in md.prefs_by_manifest
+
+    prefs = md.prefs_by_manifest[manifest_relpath]
+    assert len(prefs) == 1
+    assert prefs.pop() == "\nfoo=bar\nbrowser.dom.foo=baz"
+
+    options['runByManifest'] = False
+    with pytest.raises(SystemExit):
+        get_active_tests(**options)
+
+    options['runByManifest'] = True
+    options['manifestFile'] = create_manifest("""
+[files/test_pass.html]
+prefs=foo=bar
+[files/test_fail.html]
+""")[1]
+    with pytest.raises(SystemExit):
+        get_active_tests(**options)
+
+
+if __name__ == '__main__':
+    sys.exit(pytest.main(['--verbose', __file__]))