Bug 930025 - b2g unittests need to check for crashes in more places, r=jgriffin
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Wed, 23 Oct 2013 14:45:48 -0400
changeset 151883 dfb15cca5df4504be76e9a8965c7f57b2aeab600
parent 151882 376956b7585a8b0e98b82e0fa8f10e5180fd744f
child 151884 9d7ed37acfe6f2e9797869b3e690cd8f38a01141
push id25512
push usercbook@mozilla.com
push dateThu, 24 Oct 2013 05:06:01 +0000
treeherderautoland@19fd3388c372 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjgriffin
bugs930025
milestone27.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 930025 - b2g unittests need to check for crashes in more places, r=jgriffin
testing/marionette/client/marionette/b2ginstance.py
testing/marionette/client/marionette/emulator.py
testing/marionette/client/marionette/marionette.py
testing/mochitest/runtestsb2g.py
testing/mozbase/mozrunner/mozrunner/remote.py
testing/mozbase/mozrunner/mozrunner/runner.py
--- a/testing/marionette/client/marionette/b2ginstance.py
+++ b/testing/marionette/client/marionette/b2ginstance.py
@@ -10,21 +10,22 @@ import traceback
 
 from b2gbuild import B2GBuild
 from mozdevice import DeviceManagerADB
 import mozcrash
 
 
 class B2GInstance(B2GBuild):
 
-    def __init__(self, devicemanager=None, **kwargs):
+    def __init__(self, devicemanager=None, symbols_path=None, **kwargs):
         B2GBuild.__init__(self, **kwargs)
 
         self._dm = devicemanager
         self._remote_profiles = None
+        self.symbols_path = symbols_path
 
     @property
     def dm(self):
         if not self._dm:
             self._dm = DeviceManagerADB(adbPath=self.adb_path)
         return self._dm
 
     @property
@@ -47,23 +48,23 @@ class B2GInstance(B2GBuild):
             if cfg.has_option(section, 'Path'):
                 if cfg.has_option(section, 'IsRelative') and cfg.getint(section, 'IsRelative'):
                     remote_profiles.append(posixpath.join(posixpath.dirname(remote_profiles_ini), cfg.get(section, 'Path')))
                 else:
                     remote_profiles.append(cfg.get(section, 'Path'))
         self._remote_profiles = remote_profiles
         return remote_profiles
 
-    def check_for_crashes(self, symbols_path):
+    def check_for_crashes(self):
         remote_dump_dirs = [posixpath.join(p, 'minidumps') for p in self.remote_profiles]
         crashed = False
         for remote_dump_dir in remote_dump_dirs:
             local_dump_dir = tempfile.mkdtemp()
             self.dm.getDirectory(remote_dump_dir, local_dump_dir)
             try:
-                if mozcrash.check_for_crashes(local_dump_dir, symbols_path):
+                if mozcrash.check_for_crashes(local_dump_dir, self.symbols_path):
                     crashed = True
             except:
                 traceback.print_exc()
             finally:
                 shutil.rmtree(local_dump_dir)
                 self.dm.removeDir(remote_dump_dir)
         return crashed
--- a/testing/marionette/client/marionette/emulator.py
+++ b/testing/marionette/client/marionette/emulator.py
@@ -43,17 +43,17 @@ class Emulator(object):
     deviceRe = re.compile(r"^emulator-(\d+)(\s*)(.*)$")
     _default_res = '320x480'
     prefs = {'app.update.enabled': False,
              'app.update.staging.enabled': False,
              'app.update.service.enabled': False}
 
     def __init__(self, homedir=None, noWindow=False, logcat_dir=None,
                  arch="x86", emulatorBinary=None, res=None, sdcard=None,
-                 userdata=None):
+                 symbols_path=None, userdata=None):
         self.port = None
         self.dm = None
         self._emulator_launched = False
         self.proc = None
         self.marionette_port = None
         self.telnet = None
         self._tmp_sdcard = None
         self._tmp_userdata = None
@@ -64,24 +64,26 @@ class Emulator(object):
         self.arch = arch
         self.binary = emulatorBinary
         self.res = res or self._default_res
         self.battery = EmulatorBattery(self)
         self.geo = EmulatorGeo(self)
         self.screen = EmulatorScreen(self)
         self.homedir = homedir
         self.sdcard = sdcard
+        self.symbols_path = symbols_path
         self.noWindow = noWindow
         if self.homedir is not None:
             self.homedir = os.path.expanduser(homedir)
         self.dataImg = userdata
         self.copy_userdata = self.dataImg is None
 
     def _check_for_b2g(self):
-        self.b2g = B2GInstance(homedir=self.homedir, emulator=True)
+        self.b2g = B2GInstance(homedir=self.homedir, emulator=True,
+                               symbols_path=self.symbols_path)
         self.adb = self.b2g.adb_path
         self.homedir = self.b2g.homedir
 
         if self.arch not in ("x86", "arm"):
             raise Exception("Emulator architecture must be one of x86, arm, got: %s" %
                             self.arch)
 
         host_dir = "linux-x86"
@@ -154,23 +156,21 @@ class Emulator(object):
         """
         Checks if the emulator has crashed or not.  Always returns False if
         we've connected to an already-running emulator, since we can't track
         the emulator's pid in that case.  Otherwise, returns True iff
         self.proc is not None (meaning the emulator hasn't been explicitly
         closed), and self.proc.poll() is also not None (meaning the emulator
         process has terminated).
         """
-        if (self._emulator_launched and self.proc is not None
-                                    and self.proc.poll() is not None):
-            return True
-        return False
+        return self._emulator_launched and self.proc is not None \
+                                       and self.proc.poll() is not None
 
-    def check_for_minidumps(self, symbols_path):
-        return self.b2g.check_for_crashes(symbols_path)
+    def check_for_minidumps(self):
+        return self.b2g.check_for_crashes()
 
     def create_sdcard(self, sdcard):
         self._tmp_sdcard = tempfile.mktemp(prefix='sdcard')
         sdargs = [self.mksdcard, "-l", "mySdCard", sdcard, self._tmp_sdcard]
         sd = subprocess.Popen(sdargs, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         retcode = sd.wait()
         if retcode:
             raise Exception('unable to create sdcard : exit code %d: %s'
@@ -269,16 +269,19 @@ waitFor(
             """)
         except ScriptTimeoutException:
             print 'timed out'
             # We silently ignore the timeout if it occurs, since
             # isSystemMessageListenerReady() isn't available on
             # older emulators.  45s *should* be enough of a delay
             # to allow telephony API's to work.
             pass
+        except InvalidResponseException:
+            self.check_for_minidumps()
+            raise
         print 'done'
         marionette.set_context(marionette.CONTEXT_CONTENT)
         marionette.delete_session()
 
     def connect(self):
         self.adb = B2GInstance.check_adb(self.homedir, emulator=True)
         self.start_adb()
 
--- a/testing/marionette/client/marionette/marionette.py
+++ b/testing/marionette/client/marionette/marionette.py
@@ -437,17 +437,16 @@ class Marionette(object):
         self.window = None
         self.emulator = None
         self.extra_emulators = []
         self.homedir = homedir
         self.baseurl = baseurl
         self.noWindow = noWindow
         self.logcat_dir = logcat_dir
         self._test_name = None
-        self.symbols_path = symbols_path
         self.timeout = timeout
         self.device_serial = device_serial
 
         if bin:
             port = int(self.port)
             if not Marionette.is_port_available(port, host=self.host):
                 ex_msg = "%s:%d is unavailable." % (self.host, port)
                 raise MarionetteException(message=ex_msg)
@@ -466,16 +465,17 @@ class Marionette(object):
             assert(self.wait_for_port()), "Timed out waiting for port!"
 
         if emulator:
             self.emulator = Emulator(homedir=homedir,
                                      noWindow=self.noWindow,
                                      logcat_dir=self.logcat_dir,
                                      arch=emulator,
                                      sdcard=sdcard,
+                                     symbols_path=symbols_path,
                                      emulatorBinary=emulatorBinary,
                                      userdata=emulatorImg,
                                      res=emulator_res)
             self.emulator.start()
             self.port = self.emulator.setup_port_forwarding(self.port)
             assert(self.emulator.wait_for_port()), "Timed out waiting for port!"
 
         if connectToRunningEmulator:
@@ -643,17 +643,17 @@ class Marionette(object):
         name = None
         crashed = False
         if self.emulator:
             if self.emulator.check_for_crash():
                 returncode = self.emulator.proc.returncode
                 name = 'emulator'
                 crashed = True
 
-            if self.symbols_path and self.emulator.check_for_minidumps(self.symbols_path):
+            if self.emulator.check_for_minidumps():
                 crashed = True
         elif self.instance:
             # In the future, a check for crashed Firefox processes
             # should be here.
             pass
         if returncode is not None:
             print ('PROCESS-CRASH | %s | abnormal termination with exit code %d' %
                 (name, returncode))
--- a/testing/mochitest/runtestsb2g.py
+++ b/testing/mochitest/runtestsb2g.py
@@ -125,30 +125,32 @@ class B2GMochitest(MochitestUtilsMixin):
 
         log.info("runtestsb2g.py | Running tests: start.")
         status = 0
         try:
             runner_args = { 'profile': self.profile,
                             'devicemanager': self._dm,
                             'marionette': self.marionette,
                             'remote_test_root': self.remote_test_root,
+                            'symbols_path': options.symbolsPath,
                             'test_script': self.test_script,
                             'test_script_args': self.test_script_args }
             self.runner = B2GRunner(**runner_args)
             self.runner.start(outputTimeout=timeout)
             status = self.runner.wait()
             if status is None:
                 # the runner has timed out
                 status = 124
         except KeyboardInterrupt:
             log.info("runtests.py | Received keyboard interrupt.\n");
             status = -1
         except:
             traceback.print_exc()
             log.error("Automation Error: Received unexpected exception while running application\n")
+            self.runner.check_for_crashes()
             status = 1
 
         self.stopWebServer(options)
         self.stopWebSocketServer(options)
 
         log.info("runtestsb2g.py | Running tests: end.")
 
         if manifest is not None:
--- a/testing/mozbase/mozrunner/mozrunner/remote.py
+++ b/testing/mozbase/mozrunner/mozrunner/remote.py
@@ -1,68 +1,69 @@
 import ConfigParser
 import os
 import posixpath
 import re
 import shutil
 import subprocess
 import tempfile
 import time
-import traceback
 
 from runner import Runner
 from mozdevice import DMError
-import mozcrash
 import mozlog
 
 __all__ = ['RemoteRunner', 'B2GRunner', 'remote_runners']
 
 class RemoteRunner(Runner):
 
     def __init__(self, profile,
                        devicemanager,
                        clean_profile=None,
                        process_class=None,
                        env=None,
                        remote_test_root=None,
-                       restore=True):
+                       restore=True,
+                       **kwargs):
 
         super(RemoteRunner, self).__init__(profile, clean_profile=clean_profile,
-                                                process_class=process_class, env=env)
+                                           process_class=process_class, env=env, **kwargs)
         self.log = mozlog.getLogger('RemoteRunner')
 
         self.dm = devicemanager
+        self.last_test = None
         self.remote_test_root = remote_test_root or self.dm.getDeviceRoot()
         self.remote_profile = posixpath.join(self.remote_test_root, 'profile')
         self.restore = restore
         self.backup_files = set([])
 
     def backup_file(self, remote_path):
         if not self.restore:
             return
 
         if self.dm.fileExists(remote_path):
             self.dm.shellCheckOutput(['dd', 'if=%s' % remote_path, 'of=%s.orig' % remote_path])
             self.backup_files.add(remote_path)
 
 
-    def check_for_crashes(self, symbols_path, last_test=None):
+    def check_for_crashes(self, last_test=None):
+        last_test = last_test or self.last_test
+        remote_dump_dir = posixpath.join(self.remote_profile, 'minidumps')
         crashed = False
-        remote_dump_dir = posixpath.join(self.remote_profile, 'minidumps')
+
         self.log.info("checking for crashes in '%s'" % remote_dump_dir)
         if self.dm.dirExists(remote_dump_dir):
             local_dump_dir = tempfile.mkdtemp()
             self.dm.getDirectory(remote_dump_dir, local_dump_dir)
-            try:
-                crashed = mozcrash.check_for_crashes(local_dump_dir, symbols_path, test_name=last_test)
-            except:
-                traceback.print_exc()
-            finally:
-                shutil.rmtree(local_dump_dir)
-                self.dm.removeDir(remote_dump_dir)
+
+            crashed = super(RemoteRunner, self).check_for_crashes(local_dump_dir, \
+                                                                  test_name=last_test)
+            shutil.rmtree(local_dump_dir)
+            self.dm.removeDir(remote_dump_dir)
+
         return crashed
 
     def cleanup(self):
         if not self.restore:
             return
 
         super(RemoteRunner, self).cleanup()
 
@@ -192,16 +193,17 @@ class B2GRunner(RemoteRunner):
 
         if self.timeout:
             timeout = self.timeout
         else:
             timeout = self.outputTimeout
             msg = "%s with no output" % msg
 
         self.log.testFail(msg % (self.last_test, timeout))
+        self.check_for_crashes()
 
     def _reboot_device(self):
         serial, status = self._get_device_status()
         self.dm.shellCheckOutput(['/system/bin/reboot'])
 
         # The reboot command can return while adb still thinks the device is
         # connected, so wait a little bit for it to disconnect from adb.
         time.sleep(10)
--- a/testing/mozbase/mozrunner/mozrunner/runner.py
+++ b/testing/mozbase/mozrunner/mozrunner/runner.py
@@ -1,39 +1,43 @@
 #!/usr/bin/env python
 
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 import subprocess
+import traceback
 
 from mozprocess.processhandler import ProcessHandler
+import mozcrash
 import mozlog
 
 # we can replace this method with 'abc'
 # (http://docs.python.org/library/abc.html) when we require Python 2.6+
 def abstractmethod(method):
   line = method.func_code.co_firstlineno
   filename = method.func_code.co_filename
   def not_implemented(*args, **kwargs):
     raise NotImplementedError('Abstract method %s at File "%s", line %s '
                               'should be implemented by a concrete class' %
                               (repr(method), filename, line))
   return not_implemented
 
 class Runner(object):
 
-    def __init__(self, profile, clean_profile=True, process_class=None, kp_kwargs=None, env=None):
+    def __init__(self, profile, clean_profile=True, process_class=None,
+                 kp_kwargs=None, env=None, symbols_path=None):
         self.clean_profile = clean_profile
         self.env = env or {}
         self.kp_kwargs = kp_kwargs or {}
         self.process_class = process_class or ProcessHandler
         self.process_handler = None
         self.profile = profile
         self.log = mozlog.getLogger('MozRunner')
+        self.symbols_path = symbols_path
 
     @abstractmethod
     def start(self, *args, **kwargs):
         """
         Run the process
         """
 
         # ensure you are stopped
@@ -98,16 +102,26 @@ class Runner(object):
 
     def reset(self):
         """
         Reset the runner to its default state
         """
         if getattr(self, 'profile', False):
             self.profile.reset()
 
+    def check_for_crashes(self, dump_directory, test_name=None):
+        crashed = False
+        try:
+            crashed = mozcrash.check_for_crashes(dump_directory,
+                                                 self.symbols_path,
+                                                 test_name=test_name)
+        except:
+            traceback.print_exc()
+        return crashed
+
     def cleanup(self):
         """
         Cleanup all runner state
         """
         if self.is_running():
             self.stop()
         if getattr(self, 'profile', False) and self.clean_profile:
             self.profile.cleanup()