Bug 1392390 - Create a reftest selftest harness, r?jmaher draft
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 11 Sep 2017 16:08:01 -0400
changeset 663025 84b7628f7e42d0bea7f2e0ea2d69a82ae807edbe
parent 663024 a4b9e1f2edb14016744a834541c14b050af1ae71
child 731070 b1281574a1c7ab7ca8f5bf19d7d7e2872c135f48
push id79294
push userahalberstadt@mozilla.com
push dateTue, 12 Sep 2017 14:34:46 +0000
reviewersjmaher
bugs1392390
milestone57.0a1
Bug 1392390 - Create a reftest selftest harness, r?jmaher This just adds two basic tests, one for a passing test and another for a failing one. In mochitest, we use privileged APIs to also tests crashes, assertions, asan and leaks. But these APIs aren't available to reftests so I'm not sure how we can test these things. I figure it's not worth holding the framework up on this though, I'll file a follow-up to figure out something to do for that. MozReview-Commit-ID: 59TSbsugT5T
layout/tools/reftest/runreftest.py
layout/tools/reftest/selftest/conftest.py
layout/tools/reftest/selftest/files/green.html
layout/tools/reftest/selftest/files/red.html
layout/tools/reftest/selftest/files/reftest-fail.list
layout/tools/reftest/selftest/files/reftest-pass.list
layout/tools/reftest/selftest/python.ini
layout/tools/reftest/selftest/test_reftest_output.py
moz.build
taskcluster/ci/source-test/python.yml
testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -70,16 +70,31 @@ summaryLines = [('Successful', [('pass',
                                 ('failedLoad', 'failed load'),
                                 ('exception', 'exception')]),
                 ('Known problems', [('knownFail', 'known fail'),
                                     ('knownAsserts', 'known asserts'),
                                     ('random', 'random'),
                                     ('skipped', 'skipped'),
                                     ('slow', 'slow')])]
 
+
+def update_mozinfo():
+    """walk up directories to find mozinfo.json update the info"""
+    # TODO: This should go in a more generic place, e.g. mozinfo
+
+    path = SCRIPT_DIRECTORY
+    dirs = set()
+    while path != os.path.expanduser('~'):
+        if path in dirs:
+            break
+        dirs.add(path)
+        path = os.path.split(path)[0]
+    mozinfo.find_and_update_from_json(*dirs)
+
+
 # Python's print is not threadsafe.
 printLock = threading.Lock()
 
 
 class ReftestThread(threading.Thread):
     def __init__(self, cmdargs):
         threading.Thread.__init__(self)
         self.cmdargs = cmdargs
@@ -211,17 +226,17 @@ class ReftestResolver(object):
 class RefTest(object):
     TEST_SEEN_INITIAL = 'reftest'
     TEST_SEEN_FINAL = 'Main app process exited normally'
     use_marionette = True
     oldcwd = os.getcwd()
     resolver_cls = ReftestResolver
 
     def __init__(self):
-        self.update_mozinfo()
+        update_mozinfo()
         self.lastTestSeen = self.TEST_SEEN_INITIAL
         self.haveDumpedScreen = False
         self.resolver = self.resolver_cls()
         self.log = None
 
     def _populate_logger(self, options):
         if self.log:
             return
@@ -231,29 +246,16 @@ class RefTest(object):
                                                      "benefit of legacy log parsers and"
                                                      "tools such as the reftest analyzer")
         fmt_options = {}
         if not options.log_tbpl_level and os.environ.get('MOZ_REFTEST_VERBOSE'):
             options.log_tbpl_level = fmt_options['level'] = 'debug'
         self.log = mozlog.commandline.setup_logging(
             "reftest harness", options, {"tbpl": sys.stdout}, fmt_options)
 
-    def update_mozinfo(self):
-        """walk up directories to find mozinfo.json update the info"""
-        # TODO: This should go in a more generic place, e.g. mozinfo
-
-        path = SCRIPT_DIRECTORY
-        dirs = set()
-        while path != os.path.expanduser('~'):
-            if path in dirs:
-                break
-            dirs.add(path)
-            path = os.path.split(path)[0]
-        mozinfo.find_and_update_from_json(*dirs)
-
     def getFullPath(self, path):
         "Get an absolute path relative to self.oldcwd."
         return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
 
     def createReftestProfile(self, options, manifests, server='localhost', port=0,
                              profile_to_clone=None, startAfter=None):
         """Sets up a profile for reftest.
 
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/conftest.py
@@ -0,0 +1,95 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import pytest
+from argparse import Namespace
+from cStringIO import StringIO
+
+import mozinfo
+from manifestparser import expression
+from moztest.selftest.fixtures import binary, setup_test_harness  # noqa
+
+here = os.path.abspath(os.path.dirname(__file__))
+setup_args = [False, 'reftest', 'reftest']
+
+
+@pytest.fixture  # noqa: F811
+def parser(setup_test_harness):
+    setup_test_harness(*setup_args)
+    cmdline = pytest.importorskip('reftestcommandline')
+    return cmdline.DesktopArgumentsParser()
+
+
+@pytest.fixture  # noqa: F811
+def runtests(setup_test_harness, binary, parser):
+    setup_test_harness(*setup_args)
+    runreftest = pytest.importorskip('runreftest')
+    harness_root = runreftest.SCRIPT_DIRECTORY
+
+    buf = StringIO()
+    build = parser.build_obj
+    options = vars(parser.parse_args([]))
+    options.update({
+        'app': binary,
+        'focusFilterMode': 'non-needs-focus',
+        'log_raw': [buf],
+        'suite': 'reftest',
+    })
+
+    if not os.path.isdir(build.bindir):
+        package_root = os.path.dirname(harness_root)
+        options.update({
+            'extraProfileFiles': [os.path.join(package_root, 'bin', 'plugins')],
+            'objPath': os.environ['PYTHON_TEST_TMP'],
+            'reftestExtensionPath': os.path.join(harness_root, 'reftest'),
+            'utilityPath': os.path.join(package_root, 'bin'),
+            'workPath': here,
+        })
+    else:
+        options.update({
+            'extraProfileFiles': [os.path.join(build.topobjdir, 'dist', 'plugins')],
+            'objPath': build.topobjdir,
+            'workPath': build.topsrcdir,
+        })
+
+    def normalize(test):
+        if os.path.isabs(test):
+            return test
+        return os.path.join(here, 'files', test)
+
+    def inner(*tests, **opts):
+        assert len(tests) > 0
+        tests = map(normalize, tests)
+
+        options['tests'] = tests
+        options.update(opts)
+
+        result = runreftest.run_test_harness(parser, Namespace(**options))
+        out = json.loads('[' + ','.join(buf.getvalue().splitlines()) + ']')
+        buf.close()
+        return result, out
+    return inner
+
+
+@pytest.fixture(autouse=True)  # noqa: F811
+def skip_using_mozinfo(request, setup_test_harness):
+    """Gives tests the ability to skip based on values from mozinfo.
+
+    Example:
+        @pytest.mark.skip_mozinfo("!e10s || os == 'linux'")
+        def test_foo():
+            pass
+    """
+
+    setup_test_harness(*setup_args)
+    runreftest = pytest.importorskip('runreftest')
+    runreftest.update_mozinfo()
+
+    skip_mozinfo = request.node.get_marker('skip_mozinfo')
+    if skip_mozinfo:
+        value = skip_mozinfo.args[0]
+        if expression.parse(value, **mozinfo.info):
+            pytest.skip("skipped due to mozinfo match: \n{}".format(value))
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/green.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div style="color: green">Text</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/red.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div style="color: red">Text</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-fail.list
@@ -0,0 +1,3 @@
+== green.html red.html
+!= green.html green.html
+!= red.html red.html
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-pass.list
@@ -0,0 +1,3 @@
+== green.html green.html
+== red.html red.html
+!= green.html red.html
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/python.ini
@@ -0,0 +1,5 @@
+[DEFAULT]
+subsuite=reftest
+sequential=true
+
+[test_reftest_output.py]
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/test_reftest_output.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from functools import partial
+
+import mozunit
+from moztest.selftest.output import get_mozharness_status, filter_action
+
+from mozharness.base.log import INFO, WARNING
+from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING
+
+get_mozharness_status = partial(get_mozharness_status, 'reftest')
+
+
+def test_output_pass(runtests):
+    status, lines = runtests('reftest-pass.list')
+    assert status == 0
+
+    tbpl_status, log_level = get_mozharness_status(lines, status)
+    assert tbpl_status == TBPL_SUCCESS
+    assert log_level in (INFO, WARNING)
+
+    test_end = filter_action('test_end', lines)
+    assert len(test_end) == 3
+    assert all(t['status'] == 'PASS' for t in test_end)
+
+
+def test_output_fail(runtests):
+    status, lines = runtests('reftest-fail.list')
+    assert status == 0
+
+    tbpl_status, log_level = get_mozharness_status(lines, status)
+    assert tbpl_status == TBPL_WARNING
+    assert log_level == WARNING
+
+    test_end = filter_action('test_end', lines)
+    assert len(test_end) == 3
+    assert all(t['status'] == 'FAIL' for t in test_end)
+
+
+if __name__ == '__main__':
+    mozunit.main()
--- a/moz.build
+++ b/moz.build
@@ -64,16 +64,17 @@ DIRS += [
     'taskcluster',
     'testing/mozbase',
     'third_party/python',
 ]
 
 if not CONFIG['JS_STANDALONE']:
     # These python manifests are included here so they get picked up without an objdir
     PYTHON_UNITTEST_MANIFESTS += [
+        'layout/tools/reftest/selftest/python.ini',
         'testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini',
         'testing/mochitest/tests/python/python.ini',
     ]
 
     CONFIGURE_SUBST_FILES += [
         'tools/update-packaging/Makefile',
     ]
     CONFIGURE_DEFINE_FILES += [
--- a/taskcluster/ci/source-test/python.yml
+++ b/taskcluster/ci/source-test/python.yml
@@ -144,8 +144,42 @@ mozlint:
                 max-run-time: 3600
     run:
         using: mach
         mach: python-test --subsuite mozlint
     when:
         files-changed:
             - 'python/mozlint/**'
             - 'python/mach_commands.py'
+
+reftest-harness:
+    description: layout/tools/reftest unittests
+    platform:
+        - linux64/opt
+    require-build: true
+    treeherder:
+        symbol: py(ref)
+        kind: test
+        tier: 2
+    worker-type:
+        by-platform:
+            linux64.*: aws-provisioner-v1/gecko-t-linux-xlarge
+    worker:
+        by-platform:
+            linux64.*:
+                docker-image: {in-tree: "desktop1604-test"}
+                max-run-time: 3600
+    run:
+        using: run-task
+        command: >
+            source /builds/worker/scripts/xvfb.sh &&
+            start_xvfb '1600x1200x24' 0 &&
+            cd /builds/worker/checkouts/gecko &&
+            ./mach python-test --subsuite reftest
+    when:
+        files-changed:
+            - 'config/mozunit.py'
+            - 'layout/tools/reftest/**'
+            - 'python/mach_commands.py'
+            - 'testing/mozbase/moztest/moztest/selftest/**'
+            - 'testing/mozharness/mozharness/base/log.py'
+            - 'testing/mozharness/mozharness/mozilla/structuredlog.py'
+            - 'testing/mozharness/mozharness/mozilla/testing/errors.py'
--- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
@@ -215,17 +215,17 @@ class TbplFormatter(BaseFormatter):
             if "reftest_screenshots" in extra:
                 screenshots = extra["reftest_screenshots"]
                 if len(screenshots) == 3:
                     message += ("\nREFTEST   IMAGE 1 (TEST): data:image/png;base64,%s\n"
                                 "REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,%s") % (
                                     screenshots[0]["screenshot"],
                                     screenshots[2]["screenshot"])
                 elif len(screenshots) == 1:
-                    message += "\nREFTEST   IMAGE: data:image/png;base64,%(image1)s" \
+                    message += "\nREFTEST   IMAGE: data:image/png;base64,%s" \
                                % screenshots[0]["screenshot"]
 
             failure_line = "TEST-UNEXPECTED-%s | %s | %s\n" % (
                 data["status"], test_id, message)
 
             if data["expected"] not in ("PASS", "OK"):
                 expected_msg = "expected %s | " % data["expected"]
             else: