Bug 1403983 - Fix reftest regression where screenshots aren't being output and add a test, r=jmaher
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 28 Sep 2017 12:26:51 -0400
changeset 383532 50958575534033662c87ef2423922a1ad2cc96c4
parent 383531 8855ba3f65ba482513c0bfe322b351d4555d8619
child 383534 16f3c04bb0ead41a9646c7bc806f182c3d31c0e0
push id52281
push userahalberstadt@mozilla.com
push dateThu, 28 Sep 2017 17:05:57 +0000
treeherderautoland@509585755340 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1403983
milestone58.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1403983 - Fix reftest regression where screenshots aren't being output and add a test, r=jmaher MozReview-Commit-ID: 8ZqmukDtWpO
layout/tools/reftest/output.py
layout/tools/reftest/selftest/test_reftest_output.py
testing/mozbase/moztest/moztest/selftest/output.py
--- a/layout/tools/reftest/output.py
+++ b/layout/tools/reftest/output.py
@@ -47,46 +47,47 @@ class ReftestFormatter(TbplFormatter):
             if status not in ("PASS", "SKIP"):
                 status_msg += "KNOWN-"
             status_msg += status
             if extra.get('status_msg') == 'Random':
                 status_msg += "(EXPECTED RANDOM)"
         return status_msg
 
     def test_status(self, data):
+        extra = data.get('extra', {})
         test = data['test']
 
         status_msg = self._format_status(data)
         output_text = "%s | %s | %s" % (status_msg, test, data.get("subtest", "unknown test"))
         if data.get('message'):
             output_text += " | %s" % data['message']
-        return output_text + "\n"
-
-    def test_end(self, data):
-        extra = data.get('extra', {})
-        status = data['status']
-        test = data['test']
-
-        output_text = ""
-        if status != "OK":
-            status_msg = self._format_status(data)
-            output_text = "%s | %s | %s" % (status_msg, test, data.get("message", ""))
 
         if "reftest_screenshots" in extra:
             screenshots = extra["reftest_screenshots"]
             image_1 = screenshots[0]["screenshot"]
 
             if len(screenshots) == 3:
                 image_2 = screenshots[2]["screenshot"]
                 output_text += ("\nREFTEST   IMAGE 1 (TEST): data:image/png;base64,%s\n"
                                 "REFTEST   IMAGE 2 (REFERENCE): data:image/png;base64,%s") % (
                                 image_1, image_2)
             elif len(screenshots) == 1:
                 output_text += "\nREFTEST   IMAGE: data:image/png;base64,%s" % image_1
 
+        return output_text + "\n"
+
+    def test_end(self, data):
+        status = data['status']
+        test = data['test']
+
+        output_text = ""
+        if status != "OK":
+            status_msg = self._format_status(data)
+            output_text = "%s | %s | %s" % (status_msg, test, data.get("message", ""))
+
         if output_text:
             output_text += "\nREFTEST "
         output_text += "TEST-END | %s" % test
         return "%s\n" % output_text
 
     def process_output(self, data):
         return "%s\n" % data["data"]
 
--- a/layout/tools/reftest/selftest/test_reftest_output.py
+++ b/layout/tools/reftest/selftest/test_reftest_output.py
@@ -1,13 +1,14 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os
+from cStringIO import StringIO
 from functools import partial
 
 import mozunit
 import pytest
 from moztest.selftest.output import get_mozharness_status, filter_action
 
 from mozharness.base.log import INFO, WARNING, ERROR
 from mozharness.mozilla.buildbot import TBPL_SUCCESS, TBPL_WARNING, TBPL_FAILURE
@@ -29,31 +30,42 @@ def test_output_pass(runtests):
     assert all(t['status'] == 'PASS' for t in test_status)
 
     test_end = filter_action('test_end', lines)
     assert len(test_end) == 3
     assert all(t['status'] == 'OK' for t in test_end)
 
 
 def test_output_fail(runtests):
+    formatter = pytest.importorskip('output').ReftestFormatter()
+
     status, lines = runtests('reftest-fail.list')
     assert status == 0
 
-    tbpl_status, log_level = get_mozharness_status(lines, status)
+    buf = StringIO()
+    tbpl_status, log_level = get_mozharness_status(
+        lines, status, formatter=formatter, buf=buf)
+
     assert tbpl_status == TBPL_WARNING
     assert log_level == WARNING
 
     test_status = filter_action('test_status', lines)
     assert len(test_status) == 3
     assert all(t['status'] == 'FAIL' for t in test_status)
+    assert all('reftest_screenshots' in t['extra'] for t in test_status)
 
     test_end = filter_action('test_end', lines)
     assert len(test_end) == 3
     assert all(t['status'] == 'OK' for t in test_end)
 
+    # ensure screenshots were printed
+    formatted = buf.getvalue()
+    assert 'REFTEST   IMAGE 1' in formatted
+    assert 'REFTEST   IMAGE 2' in formatted
+
 
 @pytest.mark.skip_mozinfo("!crashreporter")
 def test_output_crash(runtests):
     status, lines = runtests('reftest-crash.list', environment=["MOZ_CRASHREPORTER_SHUTDOWN=1"])
     assert status == 1
 
     tbpl_status, log_level = get_mozharness_status(lines, status)
     assert tbpl_status == TBPL_FAILURE
--- a/testing/mozbase/moztest/moztest/selftest/output.py
+++ b/testing/mozbase/moztest/moztest/selftest/output.py
@@ -15,33 +15,36 @@ build = MozbuildObject.from_environment(
 
 sys.path.insert(0, os.path.join(build.topsrcdir, 'testing', 'mozharness'))
 from mozharness.base.log import INFO
 from mozharness.base.errors import BaseErrorList
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.mozilla.testing.errors import HarnessErrorList
 
 
-def get_mozharness_status(suite, lines, status):
+def get_mozharness_status(suite, lines, status, formatter=None, buf=None):
     """Given list of log lines, determine what the mozharness status would be."""
     parser = StructuredOutputParser(
         config={'log_level': INFO},
         error_list=BaseErrorList+HarnessErrorList,
         strict=False,
         suite_category=suite,
     )
 
+    if formatter:
+        parser.formatter = formatter
+
     # Processing the log with mozharness will re-print all the output to stdout
     # Since this exact same output has already been printed by the actual test
     # run, temporarily redirect stdout to devnull.
-    with open(os.devnull, 'w') as fh:
-        orig = sys.stdout
-        sys.stdout = fh
-        for line in lines:
-            parser.parse_single_line(json.dumps(line))
-        sys.stdout = orig
+    buf = buf or open(os.devnull, 'w')
+    orig = sys.stdout
+    sys.stdout = buf
+    for line in lines:
+        parser.parse_single_line(json.dumps(line))
+    sys.stdout = orig
     return parser.evaluate_parser(status)
 
 
 def filter_action(actions, lines):
     if isinstance(actions, basestring):
         actions = (actions,)
     return filter(lambda x: x['action'] in actions, lines)