Bug 991348 - Give the ability to take screnshots on all test failures. r=jmaher
authorManish Goregaokar <manishearth@gmail.com>
Sat, 12 Apr 2014 04:23:00 +0200
changeset 197040 6394c88d9f90ed0b28b33c2e5cab1af0836c738a
parent 197039 bb7feacef303345002684a8da97c7a8a95a826be
child 197041 4673c1a42a9c6f1250c9987fd16b9b6482efa806
push id3624
push userasasaki@mozilla.com
push dateMon, 09 Jun 2014 21:49:01 +0000
treeherdermozilla-beta@b1a5da15899a [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs991348
milestone31.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 991348 - Give the ability to take screnshots on all test failures. r=jmaher
build/automation.py.in
testing/mochitest/mach_commands.py
testing/mochitest/mochitest_options.py
testing/mochitest/runtests.py
--- a/build/automation.py.in
+++ b/build/automation.py.in
@@ -809,17 +809,17 @@ class Automation(object):
   def checkForCrashes(self, minidumpDir, symbolsPath):
     return mozcrash.check_for_crashes(minidumpDir, symbolsPath, test_name=self.lastTestSeen)
 
   def runApp(self, testURL, env, app, profileDir, extraArgs,
              runSSLTunnel = False, utilityPath = None,
              xrePath = None, certPath = None,
              debuggerInfo = None, symbolsPath = None,
              timeout = -1, maxTime = None, onLaunch = None,
-             webapprtChrome = False, hide_subtests=None):
+             webapprtChrome = False, hide_subtests=None, screenshotOnFail=False):
     """
     Run the app, log the duration it took to execute, return the status code.
     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
     """
 
     if utilityPath == None:
       utilityPath = self.DIST_BIN
     if xrePath == None:
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -179,17 +179,17 @@ class MochitestRunner(MozbuildObject):
 
         options.b2gPath = b2g_home
         options.logcat_dir = self.mochitest_dir
         options.httpdPath = self.mochitest_dir
         options.xrePath = xre_path
         return mochitest.run_remote_mochitests(parser, options)
 
     def run_desktop_test(self, context, suite=None, test_paths=None, debugger=None,
-        debugger_args=None, slowscript=False, shuffle=False, keep_open=False,
+        debugger_args=None, slowscript=False, screenshot_on_fail = False, shuffle=False, keep_open=False,
         rerun_failures=False, no_autorun=False, repeat=0, run_until_failure=False,
         slow=False, chunk_by_dir=0, total_chunks=None, this_chunk=None,
         jsdebugger=False, debug_on_failure=False, start_at=None, end_at=None,
         e10s=False, dmd=False, dump_output_directory=None,
         dump_about_memory_after_test=False, dump_dmd_after_test=False,
         install_extension=None, quiet=False, **kwargs):
         """Runs a mochitest.
 
@@ -288,16 +288,17 @@ class MochitestRunner(MozbuildObject):
             raise Exception('None or unrecognized mochitest suite type.')
 
         if dmd:
             options.dmdPath = self.bin_dir
 
         options.autorun = not no_autorun
         options.closeWhenDone = not keep_open
         options.slowscript = slowscript
+        options.screenshotOnFail = screenshot_on_fail
         options.shuffle = shuffle
         options.consoleLevel = 'INFO'
         options.repeat = repeat
         options.runUntilFailure = run_until_failure
         options.runSlower = slow
         options.testingModulesDir = os.path.join(self.tests_dir, 'modules')
         options.extraProfileFiles.append(os.path.join(self.distdir, 'plugins'))
         options.symbolsPath = os.path.join(self.distdir, 'crashreporter-symbols')
@@ -406,16 +407,20 @@ def MochitestCommand(func):
     # segfaults induced by the slow-script-detecting logic for Ion/Odin JITted
     # code. If we don't pass this, the user will need to periodically type
     # "continue" to (safely) resume execution. There are ways to implement
     # automatic resuming; see the bug.
     slowscript = CommandArgument('--slowscript', action='store_true',
         help='Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; when not set, recoverable but misleading SIGSEGV instances may occur in Ion/Odin JIT code')
     func = slowscript(func)
 
+    screenshot_on_fail = CommandArgument('--screenshot-on-fail', action='store_true',
+        help='Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory for storing the screenshots.')
+    func = screenshot_on_fail(func)
+
     shuffle = CommandArgument('--shuffle', action='store_true',
         help='Shuffle execution order.')
     func = shuffle(func)
 
     keep_open = CommandArgument('--keep-open', action='store_true',
         help='Keep the browser open after tests complete.')
     func = keep_open(func)
 
--- a/testing/mochitest/mochitest_options.py
+++ b/testing/mochitest/mochitest_options.py
@@ -392,16 +392,22 @@ class MochitestOptions(optparse.OptionPa
         [["--slowscript"],
          { "action": "store_true",
            "default": False,
            "dest": "slowscript",
            "help": "Do not set the JS_DISABLE_SLOW_SCRIPT_SIGNALS env variable; "
                    "when not set, recoverable but misleading SIGSEGV instances "
                    "may occur in Ion/Odin JIT code."
         }],
+        [["--screenshot-on-fail"],
+         { "action": "store_true",
+           "default": False,
+           "dest": "screenshotOnFail",
+           "help": "Take screenshots on all test failures. Set $MOZ_UPLOAD_DIR to a directory for storing the screenshots."
+        }],
         [["--quiet"],
          { "action": "store_true",
            "default": False,
            "dest": "quiet",
            "help": "Do not print test log lines unless a failure occurs."
          }],
     ]
 
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -885,17 +885,18 @@ class Mochitest(MochitestUtilsMixin):
              utilityPath,
              xrePath,
              certPath,
              debuggerInfo=None,
              symbolsPath=None,
              timeout=-1,
              onLaunch=None,
              webapprtChrome=False,
-             hide_subtests=False):
+             hide_subtests=False,
+             screenshotOnFail=False):
     """
     Run the app, log the duration it took to execute, return the status code.
     Kills the app if it runs for longer than |maxTime| seconds, or outputs nothing for |timeout| seconds.
     """
 
     # debugger information
     interactive = False
     debug_args = None
@@ -975,16 +976,17 @@ class Mochitest(MochitestUtilsMixin):
       else:
         shutdownLeaks = None
 
       # create an instance to process the output
       outputHandler = self.OutputHandler(harness=self,
                                          utilityPath=utilityPath,
                                          symbolsPath=symbolsPath,
                                          dump_screen_on_timeout=not debuggerInfo,
+                                         dump_screen_on_fail=screenshotOnFail,
                                          hide_subtests=hide_subtests,
                                          shutdownLeaks=shutdownLeaks,
         )
 
       def timeoutHandler():
         outputHandler.log_output_buffer()
         browserProcessId = outputHandler.browserProcessId
         self.handleTimeout(timeout, proc, utilityPath, debuggerInfo, browserProcessId)
@@ -1154,17 +1156,18 @@ class Mochitest(MochitestUtilsMixin):
                                utilityPath=options.utilityPath,
                                xrePath=options.xrePath,
                                certPath=options.certPath,
                                debuggerInfo=debuggerInfo,
                                symbolsPath=options.symbolsPath,
                                timeout=timeout,
                                onLaunch=onLaunch,
                                webapprtChrome=options.webapprtChrome,
-                               hide_subtests=options.hide_subtests
+                               hide_subtests=options.hide_subtests,
+                               screenshotOnFail=options.screenshotOnFail
                                )
         except KeyboardInterrupt:
           log.info("runtests.py | Received keyboard interrupt.\n");
           status = -1
         except:
           traceback.print_exc()
           log.error("Automation Error: Received unexpected exception while running application\n")
           status = 1
@@ -1207,28 +1210,29 @@ class Mochitest(MochitestUtilsMixin):
     log.info("TEST-UNEXPECTED-FAIL | %s | application timed out after %d seconds with no output", self.lastTestSeen, int(timeout))
     browserProcessId = browserProcessId or proc.pid
     self.killAndGetStack(browserProcessId, utilityPath, debuggerInfo, dump_screen=not debuggerInfo)
 
   ### output processing
 
   class OutputHandler(object):
     """line output handler for mozrunner"""
-    def __init__(self, harness, utilityPath, symbolsPath=None, dump_screen_on_timeout=True,
+    def __init__(self, harness, utilityPath, symbolsPath=None, dump_screen_on_timeout=True, dump_screen_on_fail=False,
                  hide_subtests=False, shutdownLeaks=None):
       """
       harness -- harness instance
       dump_screen_on_timeout -- whether to dump the screen on timeout
       """
       self.harness = harness
       self.output_buffer = []
       self.running_test = False
       self.utilityPath = utilityPath
       self.symbolsPath = symbolsPath
       self.dump_screen_on_timeout = dump_screen_on_timeout
+      self.dump_screen_on_fail = dump_screen_on_fail
       self.hide_subtests = hide_subtests
       self.shutdownLeaks = shutdownLeaks
 
       # perl binary to use
       self.perl = which('perl')
 
       # With metro browser runs this script launches the metro test harness which launches the browser.
       # The metro test harness hands back the real browser process id via log output which we need to
@@ -1244,16 +1248,17 @@ class Mochitest(MochitestUtilsMixin):
         line = handler(line)
     __call__ = processOutputLine
 
     def outputHandlers(self):
       """returns ordered list of output handlers"""
       return [self.fix_stack,
               self.format,
               self.dumpScreenOnTimeout,
+              self.dumpScreenOnFail,
               self.metro_subprocess_id,
               self.trackShutdownLeaks,
               self.check_test_failure,
               self.log,
               self.record_last_test,
               ]
 
     def stackFixer(self):
@@ -1316,17 +1321,23 @@ class Mochitest(MochitestUtilsMixin):
         return self.stackFixerFunction(line)
       return line
 
     def format(self, line):
       """format the line"""
       return line.rstrip().decode("UTF-8", "ignore")
 
     def dumpScreenOnTimeout(self, line):
-      if self.dump_screen_on_timeout and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
+      if not self.dump_screen_on_fail and self.dump_screen_on_timeout and "TEST-UNEXPECTED-FAIL" in line and "Test timed out" in line:
+        self.log_output_buffer()
+        self.harness.dumpScreen(self.utilityPath)
+      return line
+
+    def dumpScreenOnFail(self, line):
+      if self.dump_screen_on_fail and "TEST-UNEXPECTED-FAIL" in line:
         self.log_output_buffer()
         self.harness.dumpScreen(self.utilityPath)
       return line
 
     def metro_subprocess_id(self, line):
       """look for metro browser subprocess id"""
       if "METRO_BROWSER_PROCESS" in line:
         index = line.find("=")