Bug 1477807 - Distinguish ADBTimeoutErrors from other exceptions in Android remote tests, r=gbrown.
authorBob Clary <bclary@bclary.com>
Fri, 27 Jul 2018 08:27:16 -0700
changeset 483886 69d756618c48147cc4fe860501ec45e9a4bead39
parent 483885 036eaf559f72ab3054b2fa1699feb748cc94431d
child 483887 5236d5fadfa0b8ed7e446cb679a0ef4c998a8614
push id9719
push userffxbld-merge
push dateFri, 24 Aug 2018 17:49:46 +0000
treeherdermozilla-beta@719ec98fba77 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1477807
milestone63.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1477807 - Distinguish ADBTimeoutErrors from other exceptions in Android remote tests, r=gbrown.
build/mobile/remoteautomation.py
js/src/tests/lib/jittests.py
layout/tools/reftest/remotereftest.py
testing/mochitest/runrobocop.py
testing/mochitest/runtests.py
testing/mochitest/runtestsremote.py
testing/remotecppunittests.py
testing/xpcshell/remotexpcshelltests.py
--- a/build/mobile/remoteautomation.py
+++ b/build/mobile/remoteautomation.py
@@ -8,16 +8,17 @@ import time
 import re
 import os
 import posixpath
 import tempfile
 import shutil
 import sys
 
 from automation import Automation
+from mozdevice import ADBTimeoutError
 from mozlog import get_default_logger
 from mozscreenshot import dump_screen
 import mozcrash
 
 # signatures for logcat messages that we don't care about much
 fennecLogcatFilters = ["The character encoding of the HTML document was not declared",
                        "Use of Mutation Events is deprecated. Use MutationObserver instead.",
                        "Unexpected value from nativeGetEnabledTags: 0"]
@@ -289,16 +290,18 @@ class RemoteAutomation(Automation):
             Fetch the full remote log file, log any new content and return True if new
             content processed.
             """
             if not self.device.is_file(self.proc):
                 return False
             try:
                 newLogContent = self.device.get_file(
                     self.proc, offset=self.stdoutlen)
+            except ADBTimeoutError:
+                raise
             except Exception:
                 return False
             if not newLogContent:
                 return False
 
             self.stdoutlen += len(newLogContent)
 
             if self.messageLogger is None:
@@ -339,16 +342,18 @@ class RemoteAutomation(Automation):
                                 try:
                                     val = int(m.group(1))
                                     if "Passed:" in line:
                                         self.counts['pass'] += val
                                     elif "Failed:" in line:
                                         self.counts['fail'] += val
                                     elif "Todo:" in line:
                                         self.counts['todo'] += val
+                                except ADBTimeoutError:
+                                    raise
                                 except Exception:
                                     pass
 
             return True
 
         @property
         def getLastTestSeen(self):
             return self.lastTestSeen
@@ -407,33 +412,39 @@ class RemoteAutomation(Automation):
                 # options but they rarely work well with Firefox on the
                 # Android emulator. dump_screen provides an effective
                 # screenshot of the emulator and its host desktop.
                 dump_screen(self.utilityPath, get_default_logger())
             if stagedShutdown:
                 # Trigger an ANR report with "kill -3" (SIGQUIT)
                 try:
                     self.device.pkill(self.procName, sig=3, attempts=1)
+                except ADBTimeoutError:
+                    raise
                 except:  # NOQA: E722
                     pass
                 time.sleep(3)
                 # Trigger a breakpad dump with "kill -6" (SIGABRT)
                 try:
                     self.device.pkill(self.procName, sig=6, attempts=1)
+                except ADBTimeoutError:
+                    raise
                 except:  # NOQA: E722
                     pass
                 # Wait for process to end
                 retries = 0
                 while retries < 3:
                     if self.device.process_exist(self.procName):
                         print("%s still alive after SIGABRT: waiting..." % self.procName)
                         time.sleep(5)
                     else:
                         return
                     retries += 1
                 try:
                     self.device.pkill(self.procName, sig=9, attempts=1)
+                except ADBTimeoutError:
+                    raise
                 except:  # NOQA: E722
                     print("%s still alive after SIGKILL!" % self.procName)
                 if self.device.process_exist(self.procName):
                     self.device.stop_application(self.procName)
             else:
                 self.device.stop_application(self.procName)
--- a/js/src/tests/lib/jittests.py
+++ b/js/src/tests/lib/jittests.py
@@ -376,17 +376,17 @@ def find_tests(substring=None):
             test = os.path.join(dirpath, filename)
             if substring is None \
                or substring in os.path.relpath(test, TEST_DIR):
                 ans.append(test)
     return ans
 
 
 def run_test_remote(test, device, prefix, options):
-    from mozdevice import ADBDevice, ADBProcessError
+    from mozdevice import ADBDevice, ADBProcessError, ADBTimeoutError
 
     if options.test_reflect_stringify:
         raise ValueError("can't run Reflect.stringify tests remotely")
     cmd = test.command(prefix,
                        posixpath.join(options.remote_test_root, 'lib/'),
                        posixpath.join(options.remote_test_root, 'modules/'),
                        posixpath.join(options.remote_test_root, 'tests'))
     if options.show_cmd:
@@ -400,16 +400,18 @@ def run_test_remote(test, device, prefix
 
     cmd = ADBDevice._escape_command_line(cmd)
     start = datetime.now()
     try:
         out = device.shell_output(cmd, env=env,
                                   cwd=options.remote_test_root,
                                   timeout=int(options.timeout))
         returncode = 0
+    except ADBTimeoutError:
+        raise
     except ADBProcessError as e:
         out = e.adb_process.stdout
         print("exception output: %s" % str(out))
         returncode = e.adb_process.exitcode
 
     elapsed = (datetime.now() - start).total_seconds()
 
     # We can't distinguish between stdout and stderr so we pass
--- a/layout/tools/reftest/remotereftest.py
+++ b/layout/tools/reftest/remotereftest.py
@@ -8,17 +8,17 @@ import psutil
 import signal
 import sys
 import tempfile
 import time
 import traceback
 import urllib2
 from contextlib import closing
 
-from mozdevice import ADBAndroid
+from mozdevice import ADBAndroid, ADBTimeoutError
 import mozinfo
 from automation import Automation
 from remoteautomation import RemoteAutomation, fennecLogcatFilters
 
 from output import OutputHandler
 from runreftest import RefTest, ReftestResolver
 import reftestcommandline
 
@@ -344,16 +344,18 @@ class RemoteReftest(RefTest):
             for category in devinfo:
                 if type(devinfo[category]) is list:
                     print "  %s:" % category
                     for item in devinfo[category]:
                         print "     %s" % item
                 else:
                     print "  %s: %s" % (category, devinfo[category])
             print "Test root: %s" % self.device.test_root
+        except ADBTimeoutError:
+            raise
         except Exception as e:
             print "WARNING: Error getting device information: %s" % str(e)
 
     def environment(self, **kwargs):
         return self.automation.environment(**kwargs)
 
     def buildBrowserEnv(self, options, profileDir):
         browserEnv = RefTest.buildBrowserEnv(self, options, profileDir)
--- a/testing/mochitest/runrobocop.py
+++ b/testing/mochitest/runrobocop.py
@@ -17,17 +17,17 @@ sys.path.insert(
 
 from automation import Automation
 from remoteautomation import RemoteAutomation, fennecLogcatFilters
 from runtests import KeyValueParseError, MochitestDesktop, MessageLogger, parseKeyValue
 from mochitest_options import MochitestArgumentParser
 
 from manifestparser import TestManifest
 from manifestparser.filters import chunk_by_slice
-from mozdevice import ADBAndroid
+from mozdevice import ADBAndroid, ADBTimeoutError
 import mozfile
 import mozinfo
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 
 
 class RobocopTestRunner(MochitestDesktop):
     """
@@ -336,16 +336,18 @@ class RobocopTestRunner(MochitestDesktop
             for category in devinfo:
                 if type(devinfo[category]) is list:
                     self.log.info("  %s:" % category)
                     for item in devinfo[category]:
                         self.log.info("     %s" % item)
                 else:
                     self.log.info("  %s: %s" % (category, devinfo[category]))
             self.log.info("Test root: %s" % self.device.test_root)
+        except ADBTimeoutError:
+            raise
         except Exception as e:
             self.log.warning("Error getting device information: %s" % str(e))
 
     def setupRobotiumConfig(self, browserEnv):
         """
            Create robotium.config and push it to the device.
         """
         fHandle = tempfile.NamedTemporaryFile(suffix='.config',
--- a/testing/mochitest/runtests.py
+++ b/testing/mochitest/runtests.py
@@ -2808,20 +2808,23 @@ toolbar#nav-bar {
                     screenshotOnFail=options.screenshotOnFail,
                     bisectChunk=options.bisectChunk,
                     marionette_args=marionette_args,
                 )
                 status = ret or status
         except KeyboardInterrupt:
             self.log.info("runtests.py | Received keyboard interrupt.\n")
             status = -1
-        except Exception:
+        except Exception as e:
             traceback.print_exc()
             self.log.error(
                 "Automation Error: Received unexpected exception while running application\n")
+            if 'ADBTimeoutError' in repr(e):
+                self.log.info("runtests.py | Device disconnected. Aborting test.\n")
+                raise
             status = 1
         finally:
             self.stopServers()
 
         ignoreMissingLeaks = options.ignoreMissingLeaks
         leakThresholds = options.leakThresholds
 
         # Stop leak detection if m-bc code coverage is enabled
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -12,17 +12,17 @@ sys.path.insert(
         os.path.realpath(
             os.path.dirname(__file__))))
 
 from automation import Automation
 from remoteautomation import RemoteAutomation, fennecLogcatFilters
 from runtests import MochitestDesktop, MessageLogger
 from mochitest_options import MochitestArgumentParser
 
-from mozdevice import ADBAndroid
+from mozdevice import ADBAndroid, ADBTimeoutError
 import mozinfo
 
 SCRIPT_DIR = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
 
 
 class MochiRemote(MochitestDesktop):
     localProfile = None
     logMessages = []
@@ -278,16 +278,18 @@ class MochiRemote(MochitestDesktop):
             for category in devinfo:
                 if type(devinfo[category]) is list:
                     self.log.info("  %s:" % category)
                     for item in devinfo[category]:
                         self.log.info("     %s" % item)
                 else:
                     self.log.info("  %s: %s" % (category, devinfo[category]))
             self.log.info("Test root: %s" % self.device.test_root)
+        except ADBTimeoutError:
+            raise
         except Exception as e:
             self.log.warning("Error getting device information: %s" % str(e))
 
     def getGMPPluginPath(self, options):
         # TODO: bug 1149374
         return None
 
     def buildBrowserEnv(self, options, debugger=False):
@@ -339,31 +341,36 @@ def run_test_harness(parser, options):
         options.extensionsToExclude.append('roboextender@mozilla.org')
 
     mochitest = MochiRemote(options)
 
     if options.log_mach is None and not options.verify:
         mochitest.printDeviceInfo()
 
     try:
+        device_exception = False
         if options.verify:
             retVal = mochitest.verifyTests(options)
         else:
             retVal = mochitest.runTests(options)
-    except Exception:
+    except Exception as e:
         mochitest.log.error("Automation Error: Exception caught while running tests")
         traceback.print_exc()
-        try:
-            mochitest.cleanup(options)
-        except Exception:
-            # device error cleaning up... oh well!
-            traceback.print_exc()
+        if isinstance(e, ADBTimeoutError):
+            mochitest.log.info("Device disconnected. Will not run mochitest.cleanup().")
+            device_exception = True
+        else:
+            try:
+                mochitest.cleanup(options)
+            except Exception:
+                # device error cleaning up... oh well!
+                traceback.print_exc()
         retVal = 1
 
-    if options.log_mach is None and not options.verify:
+    if not device_exception and options.log_mach is None and not options.verify:
         mochitest.printDeviceInfo(printLogcat=True)
 
     mochitest.message_logger.finish()
 
     return retVal
 
 
 def main(args=sys.argv[1:]):
--- a/testing/remotecppunittests.py
+++ b/testing/remotecppunittests.py
@@ -9,17 +9,17 @@ import sys
 import subprocess
 from zipfile import ZipFile
 import runcppunittests as cppunittests
 import mozcrash
 import mozfile
 import mozinfo
 import mozlog
 import posixpath
-from mozdevice import ADBAndroid, ADBProcessError
+from mozdevice import ADBAndroid, ADBProcessError, ADBTimeoutError
 
 try:
     from mozbuild.base import MozbuildObject
     build_obj = MozbuildObject.from_environment()
 except ImportError:
     build_obj = None
 
 
@@ -134,16 +134,18 @@ class RemoteCPPUnitTests(cppunittests.CP
         test_timeout = cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT * \
             timeout_factor
 
         try:
             output = self.device.shell_output(remote_bin, env=env,
                                               cwd=self.remote_home_dir,
                                               timeout=test_timeout)
             returncode = 0
+        except ADBTimeoutError:
+            raise
         except ADBProcessError as e:
             output = e.adb_process.stdout
             returncode = e.adb_process.exitcode
 
         self.log.process_output(basename, "\n%s" % output,
                                 command=[remote_bin])
         with mozfile.TemporaryDirectory() as tempdir:
             self.device.pull(self.remote_home_dir, tempdir)
--- a/testing/xpcshell/remotexpcshelltests.py
+++ b/testing/xpcshell/remotexpcshelltests.py
@@ -11,17 +11,17 @@ import os
 import posixpath
 import shutil
 import subprocess
 import sys
 import runxpcshelltests as xpcshell
 import tempfile
 from zipfile import ZipFile
 
-from mozdevice import ADBAndroid, ADBDevice
+from mozdevice import ADBAndroid, ADBDevice, ADBTimeoutError
 import mozfile
 import mozinfo
 from mozlog import commandline
 
 from xpcshellcommandline import parser_remote
 
 here = os.path.dirname(os.path.abspath(__file__))
 
@@ -151,16 +151,18 @@ class RemoteXPCShellTestThread(xpcshell.
         cmd.insert(1, self.remoteHere)
         cmd = ADBDevice._escape_command_line(cmd)
         try:
             # env is ignored here since the environment has already been
             # set for the command via the pushWrapper method.
             adb_process = self.device.shell(cmd, timeout=timeout+10, root=True)
             output_file = adb_process.stdout_file
             self.shellReturnCode = adb_process.exitcode
+        except ADBTimeoutError:
+            raise
         except Exception as e:
             if self.timedout:
                 # If the test timed out, there is a good chance the shell
                 # call also timed out and raised this Exception.
                 # Ignore this exception to simplify the error report.
                 self.shellReturnCode = None
             else:
                 raise e
@@ -209,16 +211,18 @@ class RemoteXPCShellTestThread(xpcshell.
         if self.shellReturnCode is not None:
             return self.shellReturnCode
         else:
             return -1
 
     def removeDir(self, dirname):
         try:
             self.device.rm(dirname, recursive=True, root=True)
+        except ADBTimeoutError:
+            raise
         except Exception as e:
             self.log.warning(str(e))
 
     # TODO: consider creating a separate log dir.  We don't have the test file structure,
     # so we use filename.log.  Would rather see ./logs/filename.log
     def createLogFile(self, test, stdout):
         try:
             f = None