Bug 1521072 - Log wpt reftest screenshots on fail when running mach, r=ato
authorJames Graham <james@hoppipolla.co.uk>
Fri, 18 Jan 2019 14:21:56 +0000
changeset 511559 b0377ec57ddd05e44c8336609f794986c719ab50
parent 511558 ef8493b00486cefceab561b75c6a993cd7b8d19f
child 511560 06c3ceef79456689c1b53d4bd07f75cfb71be142
push id10547
push userffxbld-merge
push dateMon, 21 Jan 2019 13:03:58 +0000
treeherdermozilla-beta@24ec1916bffe [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersato
bugs1521072
milestone66.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1521072 - Log wpt reftest screenshots on fail when running mach, r=ato On CI we only want to log screenshots when something unexpected happens since anything else is rather wasteful of resources. But locally getting screenshots for expected failures seems helpful for debugging, so worth making the default. Hopefully this isn't too confusing for people just checking if their patch regresses anything rather than actively working on fixing failures. Depends on D16973 Differential Revision: https://phabricator.services.mozilla.com/D16974
testing/web-platform/mach_commands.py
testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
--- a/testing/web-platform/mach_commands.py
+++ b/testing/web-platform/mach_commands.py
@@ -75,16 +75,19 @@ class WebPlatformTestsRunnerSetup(Mozbui
                 kwargs["host_key_path"] = os.path.join(cert_root, "web-platform.test.key")
 
             if kwargs["host_cert_path"] is None:
                 kwargs["host_cert_path"] = os.path.join(cert_root, "web-platform.test.pem")
 
         if kwargs["lsan_dir"] is None:
             kwargs["lsan_dir"] = os.path.join(self.topsrcdir, "build", "sanitizers")
 
+        if kwargs["reftest_screenshot"] is None:
+            kwargs["reftest_screenshot"] = "fail"
+
         kwargs["capture_stdio"] = True
 
         return kwargs
 
     def kwargs_firefox(self, kwargs):
         import mozinfo
         from wptrunner import wptcommandline
         kwargs = self.kwargs_common(kwargs)
--- a/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
+++ b/testing/web-platform/tests/tools/wptrunner/wptrunner/wptcommandline.py
@@ -263,17 +263,17 @@ scheme host and port.""")
                              help="Disable leak checking")
     gecko_group.add_argument("--stylo-threads", action="store", type=int, default=1,
                              help="Number of parallel threads to use for stylo")
     gecko_group.add_argument("--reftest-internal", dest="reftest_internal", action="store_true",
                              default=None, help="Enable reftest runner implemented inside Marionette")
     gecko_group.add_argument("--reftest-external", dest="reftest_internal", action="store_false",
                              help="Disable reftest runner implemented inside Marionette")
     gecko_group.add_argument("--reftest-screenshot", dest="reftest_screenshot", action="store",
-                             choices=["always", "fail", "unexpected"], default="unexpected",
+                             choices=["always", "fail", "unexpected"], default=None,
                              help="With --reftest-internal, when to take a screenshot")
     gecko_group.add_argument("--chaos", dest="chaos_mode_flags", action="store",
                              nargs="?", const=0xFFFFFFFF, type=int,
                              help="Enable chaos mode with the specified feature flag "
                              "(see http://searchfox.org/mozilla-central/source/mfbt/ChaosMode.h for "
                              "details). If no value is supplied, all features are activated")
 
     servo_group = parser.add_argument_group("Servo-specific")
@@ -525,16 +525,19 @@ def check_args(kwargs):
                                  kwargs['extra_prefs']]
 
     if kwargs["reftest_internal"] is None:
         kwargs["reftest_internal"] = True
 
     if kwargs["lsan_dir"] is None:
         kwargs["lsan_dir"] = kwargs["prefs_root"]
 
+    if kwargs["reftest_screenshot"] is None:
+        kwargs["reftest_screenshot"] = "unexpected"
+
     return kwargs
 
 
 def check_args_update(kwargs):
     set_from_config(kwargs)
 
     if kwargs["product"] is None:
         kwargs["product"] = "firefox"