Bug 1392390 - Create a reftest selftest harness, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Mon, 11 Sep 2017 16:08:01 -0400
changeset 432551 fcf7e964aba53f8a967f055a474e49d05107e24f
parent 432550 832cd947704341347a00c8e32ec1b7ff46bc942d
child 432552 e5b9fcb0f622b00e71dd057b864603b5653e969d
push id1567
push userjlorenzo@mozilla.com
push dateThu, 02 Nov 2017 12:36:05 +0000
treeherdermozilla-release@e512c14a0406 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1392390
milestone57.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1392390 - Create a reftest selftest harness, r=jmaher This just adds two basic tests, one for a passing test and another for a failing one. In mochitest, we use privileged APIs to also tests crashes, assertions, asan and leaks. But these APIs aren't available to reftests so I'm not sure how we can test these things. I figure it's not worth holding the framework up on this though, I'll file a follow-up to figure out something to do for that. MozReview-Commit-ID: 59TSbsugT5T
layout/tools/reftest/runreftest.py
layout/tools/reftest/selftest/conftest.py
layout/tools/reftest/selftest/files/green.html
layout/tools/reftest/selftest/files/red.html
layout/tools/reftest/selftest/files/reftest-fail.list
layout/tools/reftest/selftest/files/reftest-pass.list
layout/tools/reftest/selftest/python.ini
layout/tools/reftest/selftest/test_reftest_output.py
moz.build
taskcluster/ci/source-test/python.yml
testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
--- a/layout/tools/reftest/runreftest.py
+++ b/layout/tools/reftest/runreftest.py
@@ -70,16 +70,31 @@ summaryLines = [('Successful', [('pass',
                                 ('failedLoad', 'failed load'),
                                 ('exception', 'exception')]),
                 ('Known problems', [('knownFail', 'known fail'),
                                     ('knownAsserts', 'known asserts'),
                                     ('random', 'random'),
                                     ('skipped', 'skipped'),
                                     ('slow', 'slow')])]
 
+
+def update_mozinfo():
+    """walk up directories to find mozinfo.json update the info"""
+    # TODO: This should go in a more generic place, e.g. mozinfo
+
+    path = SCRIPT_DIRECTORY
+    dirs = set()
+    while path != os.path.expanduser('~'):
+        if path in dirs:
+            break
+        dirs.add(path)
+        path = os.path.split(path)[0]
+    mozinfo.find_and_update_from_json(*dirs)
+
+
 # Python's print is not threadsafe.
 printLock = threading.Lock()
 
 
 class ReftestThread(threading.Thread):
     def __init__(self, cmdargs):
         threading.Thread.__init__(self)
         self.cmdargs = cmdargs
@@ -211,17 +226,17 @@ class ReftestResolver(object):
 class RefTest(object):
     TEST_SEEN_INITIAL = 'reftest'
     TEST_SEEN_FINAL = 'Main app process exited normally'
     use_marionette = True
     oldcwd = os.getcwd()
     resolver_cls = ReftestResolver
 
     def __init__(self):
-        self.update_mozinfo()
+        update_mozinfo()
         self.lastTestSeen = self.TEST_SEEN_INITIAL
         self.haveDumpedScreen = False
         self.resolver = self.resolver_cls()
         self.log = None
 
     def _populate_logger(self, options):
         if self.log:
             return
@@ -231,29 +246,16 @@ class RefTest(object):
                                                      "benefit of legacy log parsers and"
                                                      "tools such as the reftest analyzer")
         fmt_options = {}
         if not options.log_tbpl_level and os.environ.get('MOZ_REFTEST_VERBOSE'):
             options.log_tbpl_level = fmt_options['level'] = 'debug'
         self.log = mozlog.commandline.setup_logging(
             "reftest harness", options, {"tbpl": sys.stdout}, fmt_options)
 
-    def update_mozinfo(self):
-        """walk up directories to find mozinfo.json update the info"""
-        # TODO: This should go in a more generic place, e.g. mozinfo
-
-        path = SCRIPT_DIRECTORY
-        dirs = set()
-        while path != os.path.expanduser('~'):
-            if path in dirs:
-                break
-            dirs.add(path)
-            path = os.path.split(path)[0]
-        mozinfo.find_and_update_from_json(*dirs)
-
     def getFullPath(self, path):
         "Get an absolute path relative to self.oldcwd."
         return os.path.normpath(os.path.join(self.oldcwd, os.path.expanduser(path)))
 
     def createReftestProfile(self, options, manifests, server='localhost', port=0,
                              profile_to_clone=None, startAfter=None):
         """Sets up a profile for reftest.
 
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/conftest.py
@@ -0,0 +1,95 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+import os
+import pytest
+from argparse import Namespace
+from cStringIO import StringIO
+
+import mozinfo
+from manifestparser import expression
+from moztest.selftest.fixtures import binary, setup_test_harness  # noqa
+
+here = os.path.abspath(os.path.dirname(__file__))
+setup_args = [False, 'reftest', 'reftest']
+
+
+@pytest.fixture  # noqa: F811
+def parser(setup_test_harness):
+    setup_test_harness(*setup_args)
+    cmdline = pytest.importorskip('reftestcommandline')
+    return cmdline.DesktopArgumentsParser()
+
+
+@pytest.fixture  # noqa: F811
+def runtests(setup_test_harness, binary, parser):
+    setup_test_harness(*setup_args)
+    runreftest = pytest.importorskip('runreftest')
+    harness_root = runreftest.SCRIPT_DIRECTORY
+
+    buf = StringIO()
+    build = parser.build_obj
+    options = vars(parser.parse_args([]))
+    options.update({
+        'app': binary,
+        'focusFilterMode': 'non-needs-focus',
+        'log_raw': [buf],
+        'suite': 'reftest',
+    })
+
+    if not os.path.isdir(build.bindir):
+        package_root = os.path.dirname(harness_root)
+        options.update({
+            'extraProfileFiles': [os.path.join(package_root, 'bin', 'plugins')],
+            'objPath': os.environ['PYTHON_TEST_TMP'],
+            'reftestExtensionPath': os.path.join(harness_root, 'reftest'),
+            'utilityPath': os.path.join(package_root, 'bin'),
+            'workPath': here,
+        })
+    else:
+        options.update({
+            'extraProfileFiles': [os.path.join(build.topobjdir, 'dist', 'plugins')],
+            'objPath': build.topobjdir,
+            'workPath': build.topsrcdir,
+        })
+
+    def normalize(test):
+        if os.path.isabs(test):
+            return test
+        return os.path.join(here, 'files', test)
+
+    def inner(*tests, **opts):
+        assert len(tests) > 0
+        tests = map(normalize, tests)
+
+        options['tests'] = tests
+        options.update(opts)
+
+        result = runreftest.run_test_harness(parser, Namespace(**options))
+        out = json.loads('[' + ','.join(buf.getvalue().splitlines()) + ']')
+        buf.close()
+        return result, out
+    return inner
+
+
+@pytest.fixture(autouse=True)  # noqa: F811
+def skip_using_mozinfo(request, setup_test_harness):
+    """Gives tests the ability to skip based on values from mozinfo.
+
+    Example:
+        @pytest.mark.skip_mozinfo("!e10s || os == 'linux'")
+        def test_foo():
+            pass
+    """
+
+    setup_test_harness(*setup_args)
+    runreftest = pytest.importorskip('runreftest')
+    runreftest.update_mozinfo()
+
+    skip_mozinfo = request.node.get_marker('skip_mozinfo')
+    if skip_mozinfo:
+        value = skip_mozinfo.args[0]
+        if expression.parse(value, **mozinfo.info):
+            pytest.skip("skipped due to mozinfo match: \n{}".format(value))
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/green.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div style="color: green">Text</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/red.html
@@ -0,0 +1,6 @@
+<!DOCTYPE html>
+<html>
+<body>
+<div style="color: red">Text</div>
+</body>
+</html>
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-fail.list
@@ -0,0 +1,3 @@
+== green.html red.html
+!= green.html green.html
+!= red.html red.html
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/files/reftest-pass.list
@@ -0,0 +1,3 @@
+== green.html green.html
+== red.html red.html
+!= green.html red.html
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/python.ini
@@ -0,0 +1,5 @@
+[DEFAULT]
+subsuite=reftest
+sequential=true
+
+[test_reftest_output.py]
new file mode 100644
--- /dev/null
+++ b/layout/tools/reftest/selftest/test_reftest_output.py
@@ -0,0 +1,43 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from functools import partial
+
+import mozunit
+from moztest.selftest.output import get_mozharness_status, filter_action
+
+from mozharness.base.log import INFO, WARNING
+from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING
+
+get_mozharness_status = partial(get_mozharness_status, 'reftest')
+
+
+def test_output_pass(runtests):
+    status, lines = runtests('reftest-pass.list')
+    assert status == 0
+
+    tbpl_status, log_level = get_mozharness_status(lines, status)
+    assert tbpl_status == TBPL_SUCCESS
+    assert log_level in (INFO, WARNING)
+
+    test_end = filter_action('test_end', lines)
+    assert len(test_end) == 3
+    assert all(t['status'] == 'PASS' for t in test_end)
+
+
+def test_output_fail(runtests):
+    status, lines = runtests('reftest-fail.list')
+    assert status == 0
+
+    tbpl_status, log_level = get_mozharness_status(lines, status)
+    assert tbpl_status == TBPL_WARNING
+    assert log_level == WARNING
+
+    test_end = filter_action('test_end', lines)
+    assert len(test_end) == 3
+    assert all(t['status'] == 'FAIL' for t in test_end)
+
+
+if __name__ == '__main__':
+    mozunit.main()
--- a/moz.build
+++ b/moz.build
@@ -64,16 +64,17 @@ DIRS += [
     'taskcluster',
     'testing/mozbase',
     'third_party/python',
 ]
 
 if not CONFIG['JS_STANDALONE']:
     # These python manifests are included here so they get picked up without an objdir
     PYTHON_UNITTEST_MANIFESTS += [
+        'layout/tools/reftest/selftest/python.ini',
         'testing/marionette/harness/marionette_harness/tests/harness_unit/python.ini',
         'testing/mochitest/tests/python/python.ini',
     ]
 
     CONFIGURE_SUBST_FILES += [
         'tools/update-packaging/Makefile',
     ]
     CONFIGURE_DEFINE_FILES += [
--- a/taskcluster/ci/source-test/python.yml
+++ b/taskcluster/ci/source-test/python.yml
@@ -144,8 +144,42 @@ mozlint:
                 max-run-time: 3600
     run:
         using: mach
         mach: python-test --subsuite mozlint
     when:
         files-changed:
             - 'python/mozlint/**'
             - 'python/mach_commands.py'
+
+reftest-harness:
+    description: layout/tools/reftest unittests
+    platform:
+        - linux64/opt
+    require-build: true
+    treeherder:
+        symbol: py(ref)
+        kind: test
+        tier: 2
+    worker-type:
+        by-platform:
+            linux64.*: aws-provisioner-v1/gecko-t-linux-xlarge
+    worker:
+        by-platform:
+            linux64.*:
+                docker-image: {in-tree: "desktop1604-test"}
+                max-run-time: 3600
+    run:
+        using: run-task
+        command: >
+            source /builds/worker/scripts/xvfb.sh &&
+            start_xvfb '1600x1200x24' 0 &&
+            cd /builds/worker/checkouts/gecko &&
+            ./mach python-test --subsuite reftest
+    when:
+        files-changed:
+            - 'config/mozunit.py'
+            - 'layout/tools/reftest/**'
+            - 'python/mach_commands.py'
+            - 'testing/mozbase/moztest/moztest/selftest/**'
+            - 'testing/mozharness/mozharness/base/log.py'
+            - 'testing/mozharness/mozharness/mozilla/structuredlog.py'
+            - 'testing/mozharness/mozharness/mozilla/testing/errors.py'
--- a/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
+++ b/testing/mozbase/mozlog/mozlog/formatters/tbplformatter.py
@@ -215,17 +215,17 @@ class TbplFormatter(BaseFormatter):
             if "reftest_screenshots" in extra:
                 screenshots = extra["reftest_screenshots"]
                 if len(screenshots) == 3:
                     message += ("\nREFTEST   IMAGE 1 (TEST): data:image/png;base64,%s\n"
                                 "REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,%s") % (
                                     screenshots[0]["screenshot"],
                                     screenshots[2]["screenshot"])
                 elif len(screenshots) == 1:
-                    message += "\nREFTEST   IMAGE: data:image/png;base64,%(image1)s" \
+                    message += "\nREFTEST   IMAGE: data:image/png;base64,%s" \
                                % screenshots[0]["screenshot"]
 
             failure_line = "TEST-UNEXPECTED-%s | %s | %s\n" % (
                 data["status"], test_id, message)
 
             if data["expected"] not in ("PASS", "OK"):
                 expected_msg = "expected %s | " % data["expected"]
             else: