Bug 1175540 - Reduce timeouts for many adb devicemanager calls; r=mcote
authorGeoff Brown <gbrown@mozilla.com>
Sun, 21 Jun 2015 13:58:34 -0600
changeset 280721 2505945b9d4335204c2901f5f6fc04f0e828494b
parent 280720 cc5d4eaf1a5eb760a369ec619a559cd88f94bc73
child 280722 e7d39a6671ea6768cb88a285f833fa07d2322c55
push id4932
push userjlund@mozilla.com
push dateMon, 10 Aug 2015 18:23:06 +0000
treeherdermozilla-beta@6dd5a4f5f745 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmcote
bugs1175540
milestone41.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1175540 - Reduce timeouts for many adb devicemanager calls; r=mcote
build/mobile/remoteautomation.py
testing/mozbase/mozdevice/mozdevice/devicemanager.py
testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
testing/mozbase/mozdevice/mozdevice/droid.py
--- a/build/mobile/remoteautomation.py
+++ b/build/mobile/remoteautomation.py
@@ -7,17 +7,17 @@ import time
 import re
 import os
 import tempfile
 import shutil
 import subprocess
 import sys
 
 from automation import Automation
-from devicemanager import DMError
+from devicemanager import DMError, DeviceManager
 from mozlog.structured import get_default_logger
 import mozcrash
 
 # signatures for logcat messages that we don't care about much
 fennecLogcatFilters = [ "The character encoding of the HTML document was not declared",
                         "Use of Mutation Events is deprecated. Use MutationObserver instead.",
                         "Unexpected value from nativeGetEnabledTags: 0" ]
 
@@ -119,18 +119,20 @@ class RemoteAutomation(Automation):
 
         return status
 
     def deleteANRs(self):
         # empty ANR traces.txt file; usually need root permissions
         # we make it empty and writable so we can test the ANR reporter later
         traces = "/data/anr/traces.txt"
         try:
-            self._devicemanager.shellCheckOutput(['echo', '', '>', traces], root=True)
-            self._devicemanager.shellCheckOutput(['chmod', '666', traces], root=True)
+            self._devicemanager.shellCheckOutput(['echo', '', '>', traces], root=True,
+                                                 timeout=DeviceManager.short_timeout)
+            self._devicemanager.shellCheckOutput(['chmod', '666', traces], root=True,
+                                                 timeout=DeviceManager.short_timeout)
         except DMError:
             print "Error deleting %s" % traces
             pass
 
     def checkForANRs(self):
         traces = "/data/anr/traces.txt"
         if self._devicemanager.fileExists(traces):
             try:
@@ -145,33 +147,36 @@ class RemoteAutomation(Automation):
                 print "Error pulling %s" % traces
         else:
             print "%s not found" % traces
 
     def deleteTombstones(self):
         # delete any existing tombstone files from device
         remoteDir = "/data/tombstones"
         try:
-            self._devicemanager.shellCheckOutput(['rm', '-r', remoteDir], root=True)
+            self._devicemanager.shellCheckOutput(['rm', '-r', remoteDir], root=True,
+                                                 timeout=DeviceManager.short_timeout)
         except DMError:
             # This may just indicate that the tombstone directory is missing
             pass
 
     def checkForTombstones(self):
         # pull any tombstones from device and move to MOZ_UPLOAD_DIR
         remoteDir = "/data/tombstones"
         blobberUploadDir = os.environ.get('MOZ_UPLOAD_DIR', None)
         if blobberUploadDir:
             if not os.path.exists(blobberUploadDir):
                 os.mkdir(blobberUploadDir)
             if self._devicemanager.dirExists(remoteDir):
                 # copy tombstone files from device to local blobber upload directory
                 try:
-                    self._devicemanager.shellCheckOutput(['chmod', '777', remoteDir], root=True)
-                    self._devicemanager.shellCheckOutput(['chmod', '666', os.path.join(remoteDir, '*')], root=True)
+                    self._devicemanager.shellCheckOutput(['chmod', '777', remoteDir], root=True,
+                                                 timeout=DeviceManager.short_timeout)
+                    self._devicemanager.shellCheckOutput(['chmod', '666', os.path.join(remoteDir, '*')], root=True,
+                                                 timeout=DeviceManager.short_timeout)
                     self._devicemanager.getDirectory(remoteDir, blobberUploadDir, False)
                 except DMError:
                     # This may just indicate that no tombstone files are present
                     pass
                 self.deleteTombstones()
                 # add a .txt file extension to each tombstone file name, so
                 # that blobber will upload it
                 for f in glob.glob(os.path.join(blobberUploadDir, "tombstone_??")):
@@ -187,22 +192,17 @@ class RemoteAutomation(Automation):
                 print "%s does not exist; tombstone check skipped" % remoteDir
         else:
             print "MOZ_UPLOAD_DIR not defined; tombstone check skipped"
 
     def checkForCrashes(self, directory, symbolsPath):
         self.checkForANRs()
         self.checkForTombstones()
 
-        try:
-            logcat = self._devicemanager.getLogcat(filterOutRegexps=fennecLogcatFilters)
-        except DMError:
-            print "getLogcat threw DMError; re-trying just once..."
-            time.sleep(1)
-            logcat = self._devicemanager.getLogcat(filterOutRegexps=fennecLogcatFilters)
+        logcat = self._devicemanager.getLogcat(filterOutRegexps=fennecLogcatFilters)
 
         javaException = mozcrash.check_for_java_exception(logcat)
         if javaException:
             return True
 
         # If crash reporting is disabled (MOZ_CRASHREPORTER!=1), we can't say
         # anything.
         if not self.CRASHREPORTER:
--- a/testing/mozbase/mozdevice/mozdevice/devicemanager.py
+++ b/testing/mozbase/mozdevice/mozdevice/devicemanager.py
@@ -40,16 +40,18 @@ class DeviceManager(object):
     the device, launch processes on the device, and install or remove
     applications from the device.
 
     Never instantiate this class directly! Instead, instantiate an
     implementation of it like DeviceManagerADB or DeviceManagerSUT.
     """
 
     _logcatNeedsRoot = True
+    default_timeout = 300
+    short_timeout = 30
 
     def __init__(self, logLevel=None, deviceRoot=None):
         try:
             self._logger = mozlog.structured.structuredlog.get_default_logger(component="mozdevice")
             if not self._logger: # no global structured logger, fall back to reg logging
                 self._logger = mozlog.getLogger("mozdevice")
                 if logLevel is not None:
                     self._logger.setLevel(logLevel)
@@ -128,41 +130,44 @@ class DeviceManager(object):
         """
 
     def getIP(self, interfaces=['eth0', 'wlan0']):
         """
         Returns the IP of the device, or None if no connection exists.
         """
         for interface in interfaces:
             match = re.match(r"%s: ip (\S+)" % interface,
-                             self.shellCheckOutput(['ifconfig', interface]))
+                             self.shellCheckOutput(['ifconfig', interface],
+                             timeout=self.short_timeout))
             if match:
                 return match.group(1)
 
     def recordLogcat(self):
         """
         Clears the logcat file making it easier to view specific events.
         """
         #TODO: spawn this off in a separate thread/process so we can collect all the logcat information
 
         # Right now this is just clearing the logcat so we can only see what happens after this call.
-        self.shellCheckOutput(['/system/bin/logcat', '-c'], root=self._logcatNeedsRoot)
+        self.shellCheckOutput(['/system/bin/logcat', '-c'], root=self._logcatNeedsRoot,
+                              timeout=self.short_timeout)
 
     def getLogcat(self, filterSpecs=["dalvikvm:I", "ConnectivityService:S",
                                       "WifiMonitor:S", "WifiStateTracker:S",
                                       "wpa_supplicant:S", "NetworkStateTracker:S"],
                   format="time",
                   filterOutRegexps=[]):
         """
         Returns the contents of the logcat file as a list of
         '\n' terminated strings
         """
         cmdline = ["/system/bin/logcat", "-v", format, "-d"] + filterSpecs
         output = self.shellCheckOutput(cmdline,
-                                      root=self._logcatNeedsRoot)
+                                      root=self._logcatNeedsRoot,
+                                      timeout=self.short_timeout)
         lines = output.replace('\r\n', '\n').splitlines(True)
 
         for regex in filterOutRegexps:
             lines = [line for line in lines if not re.search(regex, line)]
 
         return lines
 
     def saveScreenshot(self, filename):
--- a/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
+++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
@@ -26,17 +26,16 @@ class DeviceManagerADB(DeviceManager):
     _haveRootShell = False
     _haveSu = False
     _useZip = False
     _logcatNeedsRoot = False
     _pollingInterval = 0.01
     _packageName = None
     _tempDir = None
     connected = False
-    default_timeout = 300
 
     def __init__(self, host=None, port=5555, retryLimit=5, packageName='fennec',
                  adbPath='adb', deviceSerial=None, deviceRoot=None,
                  logLevel=mozlog.ERROR, autoconnect=True, runAdbAsRoot=False,
                  serverHost=None, serverPort=None, **kwargs):
         DeviceManager.__init__(self, logLevel=logLevel,
                                deviceRoot=deviceRoot)
         self.host = host
@@ -170,22 +169,22 @@ class DeviceManagerADB(DeviceManager):
         Forward specs are one of:
           tcp:<port>
           localabstract:<unix domain socket name>
           localreserved:<unix domain socket name>
           localfilesystem:<unix domain socket name>
           dev:<character device name>
           jdwp:<process pid> (remote only)
         """
-        if not self._checkCmd(['forward', local, remote]) == 0:
+        if not self._checkCmd(['forward', local, remote], timeout=self.short_timeout) == 0:
             raise DMError("Failed to forward socket connection.")
 
     def remount(self):
         "Remounts the /system partition on the device read-write."
-        return self._checkCmd(['remount'])
+        return self._checkCmd(['remount'], timeout=self.short_timeout)
 
     def devices(self):
         "Return a list of connected devices as (serial, status) tuples."
         proc = self._runCmd(['devices'])
         proc.output.pop(0) # ignore first line of output
         devices = []
         for line in proc.output:
             result = re.match('(.*?)\t(.*)', line)
@@ -212,17 +211,17 @@ class DeviceManagerADB(DeviceManager):
             raise DMError("File not found: %s" % localname)
 
         proc = self._runCmd(["push", os.path.realpath(localname), destname],
                 retryLimit=retryLimit)
         if proc.returncode != 0:
             raise DMError("Error pushing file %s -> %s; output: %s" % (localname, destname, proc.output))
 
     def mkDir(self, name):
-        result = self._runCmd(["shell", "mkdir", name]).output
+        result = self._runCmd(["shell", "mkdir", name], timeout=self.short_timeout).output
         if len(result) and 'read-only file system' in result[0].lower():
             raise DMError("Error creating directory: read only file system")
 
     def pushDir(self, localDir, remoteDir, retryLimit=None, timeout=None):
         # adb "push" accepts a directory as an argument, but if the directory
         # contains symbolic links, the links are pushed, rather than the linked
         # files; we either zip/unzip or re-copy the directory into a temporary
         # one to get around this limitation
@@ -237,17 +236,17 @@ class DeviceManagerADB(DeviceManager):
                               processOutputLine=self._log)
                 proc.run()
                 proc.wait()
                 self.pushFile(localZip, remoteZip, retryLimit=retryLimit, createDir=False)
                 mozfile.remove(localZip)
                 data = self._runCmd(["shell", "unzip", "-o", remoteZip,
                                      "-d", remoteDir]).output[0]
                 self._checkCmd(["shell", "rm", remoteZip],
-                               retryLimit=retryLimit, timeout=timeout)
+                               retryLimit=retryLimit, timeout=self.short_timeout)
                 if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
                     raise Exception("unzip failed, or permissions error")
             except:
                 self._logger.warning(traceback.format_exc())
                 self._logger.warning("zip/unzip failure: falling back to normal push")
                 self._useZip = False
                 self.pushDir(localDir, remoteDir, retryLimit=retryLimit, timeout=timeout)
         else:
@@ -255,66 +254,66 @@ class DeviceManagerADB(DeviceManager):
             # copytree's target dir must not already exist, so create a subdir
             tmpDirTarget = os.path.join(tmpDir, "tmp")
             shutil.copytree(localDir, tmpDirTarget)
             self._checkCmd(["push", tmpDirTarget, remoteDir],
                            retryLimit=retryLimit, timeout=timeout)
             mozfile.remove(tmpDir)
 
     def dirExists(self, remotePath):
-        data = self._runCmd(["shell", "ls", "-a", remotePath + '/']).output
+        data = self._runCmd(["shell", "ls", "-a", remotePath + '/'], timeout=self.short_timeout).output
 
         if len(data) == 1:
             res = data[0]
             if "Not a directory" in res or "No such file or directory" in res:
                 return False
         return True
 
     def fileExists(self, filepath):
-        data = self._runCmd(["shell", "ls", "-a", filepath]).output
+        data = self._runCmd(["shell", "ls", "-a", filepath], timeout=self.short_timeout).output
         if len(data) == 1:
             foundpath = data[0].decode('utf-8').rstrip()
             if foundpath == filepath:
                 return True
         return False
 
     def removeFile(self, filename):
         if self.fileExists(filename):
-            self._checkCmd(["shell", "rm", filename])
+            self._checkCmd(["shell", "rm", filename], timeout=self.short_timeout)
 
     def removeDir(self, remoteDir):
         if self.dirExists(remoteDir):
-            self._checkCmd(["shell", "rm", "-r", remoteDir])
+            self._checkCmd(["shell", "rm", "-r", remoteDir], timeout=self.short_timeout)
         else:
             self.removeFile(remoteDir.strip())
 
     def moveTree(self, source, destination):
-        self._checkCmd(["shell", "mv", source, destination])
+        self._checkCmd(["shell", "mv", source, destination], timeout=self.short_timeout)
 
     def copyTree(self, source, destination):
         self._checkCmd(["shell", "dd", "if=%s" % source, "of=%s" % destination])
 
     def listFiles(self, rootdir):
-        data = self._runCmd(["shell", "ls", "-a", rootdir]).output
+        data = self._runCmd(["shell", "ls", "-a", rootdir], timeout=self.short_timeout).output
         data[:] = [item.rstrip('\r\n') for item in data]
         if (len(data) == 1):
             if (data[0] == rootdir):
                 return []
             if (data[0].find("No such file or directory") != -1):
                 return []
             if (data[0].find("Not a directory") != -1):
                 return []
             if (data[0].find("Permission denied") != -1):
                 return []
             if (data[0].find("opendir failed") != -1):
                 return []
         return data
 
     def getProcessList(self):
-        p = self._runCmd(["shell", "ps"])
+        p = self._runCmd(["shell", "ps"], timeout=self.short_timeout)
         # first line is the headers
         p.output.pop(0)
         ret = []
         for proc in p.output:
             els = proc.split()
             # We need to figure out if this is "user pid name" or
             # "pid user vsz stat command"
             if els[1].isdigit():
@@ -389,17 +388,17 @@ class DeviceManagerADB(DeviceManager):
     def killProcess(self, appname, sig=None):
         procs = self.getProcessList()
         for (pid, name, user) in procs:
             if name == appname:
                 args = ["shell", "kill"]
                 if sig:
                     args.append("-%d" % sig)
                 args.append(str(pid))
-                p = self._runCmd(args)
+                p = self._runCmd(args, timeout=self.short_timeout)
                 if p.returncode != 0:
                     raise DMError("Error killing process "
                                   "'%s': %s" % (appname, p.output))
 
     def _runPull(self, remoteFile, localFile):
         """
         Pulls remoteFile from device to host
         """
@@ -481,56 +480,56 @@ class DeviceManagerADB(DeviceManager):
         return self._tempDir
 
     def reboot(self, wait = False, **kwargs):
         self._checkCmd(["reboot"])
         if wait:
             self._checkCmd(["wait-for-device"])
             if self._runAdbAsRoot:
                 self._adb_root()
-            self._checkCmd(["shell", "ls", "/sbin"])
+            self._checkCmd(["shell", "ls", "/sbin"], timeout=self.short_timeout)
 
     def updateApp(self, appBundlePath, **kwargs):
         return self._runCmd(["install", "-r", appBundlePath]).output
 
     def getCurrentTime(self):
-        timestr = str(self._runCmd(["shell", "date", "+%s"]).output[0])
+        timestr = str(self._runCmd(["shell", "date", "+%s"], timeout=self.short_timeout).output[0])
         if (not timestr or not timestr.isdigit()):
             raise DMError("Unable to get current time using date (got: '%s')" % timestr)
         return int(timestr)*1000
 
     def getInfo(self, directive=None):
         directive = directive or "all"
         ret = {}
         if directive == "id" or directive == "all":
-            ret["id"] = self._runCmd(["get-serialno"]).output[0]
+            ret["id"] = self._runCmd(["get-serialno"], timeout=self.short_timeout).output[0]
         if directive == "os" or directive == "all":
-            ret["os"] = self.shellCheckOutput(["getprop", "ro.build.display.id"])
+            ret["os"] = self.shellCheckOutput(["getprop", "ro.build.display.id"], timeout=self.short_timeout)
         if directive == "uptime" or directive == "all":
-            uptime = self.shellCheckOutput(["uptime"])
+            uptime = self.shellCheckOutput(["uptime"], timeout=self.short_timeout)
             if not uptime:
                 raise DMError("error getting uptime")
             m = re.match("up time: ((\d+) days, )*(\d{2}):(\d{2}):(\d{2})", uptime)
             if m:
                 uptime = "%d days %d hours %d minutes %d seconds" % tuple(
                     [int(g or 0) for g in m.groups()[1:]])
             ret["uptime"] = uptime
         if directive == "process" or directive == "all":
-            data = self.shellCheckOutput(["ps"])
+            data = self.shellCheckOutput(["ps"], timeout=self.short_timeout)
             ret["process"] = data.split('\n')
         if directive == "systime" or directive == "all":
-            ret["systime"] = self.shellCheckOutput(["date"])
+            ret["systime"] = self.shellCheckOutput(["date"], timeout=self.short_timeout)
         if directive == "memtotal" or directive == "all":
             meminfo = {}
             for line in self.pullFile("/proc/meminfo").splitlines():
                 key, value = line.split(":")
                 meminfo[key] = value.strip()
             ret["memtotal"] = meminfo["MemTotal"]
         if directive == "disk" or directive == "all":
-            data = self.shellCheckOutput(["df", "/data", "/system", "/sdcard"])
+            data = self.shellCheckOutput(["df", "/data", "/system", "/sdcard"], timeout=self.short_timeout)
             ret["disk"] = data.split('\n')
         self._logger.debug("getInfo: %s" % ret)
         return ret
 
     def uninstallApp(self, appName, installPath=None):
         status = self._runCmd(["uninstall", appName]).output[0].strip()
         if status != 'Success':
             raise DMError("uninstall failed for %s. Got: %s" % (appName, status))
@@ -618,34 +617,34 @@ class DeviceManagerADB(DeviceManager):
     def chmodDir(self, remoteDir, mask="777"):
         if (self.dirExists(remoteDir)):
             files = self.listFiles(remoteDir.strip())
             for f in files:
                 remoteEntry = remoteDir.strip() + "/" + f.strip()
                 if (self.dirExists(remoteEntry)):
                     self.chmodDir(remoteEntry)
                 else:
-                    self._checkCmd(["shell", "chmod", mask, remoteEntry])
+                    self._checkCmd(["shell", "chmod", mask, remoteEntry], timeout=self.short_timeout)
                     self._logger.info("chmod %s" % remoteEntry)
-            self._checkCmd(["shell", "chmod", mask, remoteDir])
+            self._checkCmd(["shell", "chmod", mask, remoteDir], timeout=self.short_timeout)
             self._logger.debug("chmod %s" % remoteDir)
         else:
-            self._checkCmd(["shell", "chmod", mask, remoteDir.strip()])
+            self._checkCmd(["shell", "chmod", mask, remoteDir.strip()], timeout=self.short_timeout)
             self._logger.debug("chmod %s" % remoteDir.strip())
 
     def _verifyADB(self):
         """
         Check to see if adb itself can be executed.
         """
         if self._adbPath != 'adb':
             if not os.access(self._adbPath, os.X_OK):
                 raise DMError("invalid adb path, or adb not executable: %s" % self._adbPath)
 
         try:
-            self._checkCmd(["version"])
+            self._checkCmd(["version"], timeout=self.short_timeout)
         except os.error, err:
             raise DMError("unable to execute ADB (%s): ensure Android SDK is installed and adb is in your $PATH" % err)
 
     def _verifyDevice(self):
         # If there is a device serial number, see if adb is connected to it
         if self._deviceSerial:
             deviceStatus = None
             for line in self._runCmd(["devices"]).output:
@@ -654,32 +653,32 @@ class DeviceManagerADB(DeviceManager):
                     if self._deviceSerial == m.group(1):
                         deviceStatus = m.group(2)
             if deviceStatus == None:
                 raise DMError("device not found: %s" % self._deviceSerial)
             elif deviceStatus != "device":
                 raise DMError("bad status for device %s: %s" % (self._deviceSerial, deviceStatus))
 
         # Check to see if we can connect to device and run a simple command
-        if not self._checkCmd(["shell", "echo"]) == 0:
+        if not self._checkCmd(["shell", "echo"], timeout=self.short_timeout) == 0:
             raise DMError("unable to connect to device")
 
     def _checkForRoot(self):
         # Check whether we _are_ root by default (some development boards work
         # this way, this is also the result of some relatively rare rooting
         # techniques)
-        proc = self._runCmd(["shell", "id"])
+        proc = self._runCmd(["shell", "id"], timeout=self.short_timeout)
         if proc.output and 'uid=0(root)' in proc.output[0]:
             self._haveRootShell = True
             # if this returns true, we don't care about su
             return
 
         # if root shell is not available, check if 'su' can be used to gain
         # root
-        proc = self._runCmd(["shell", "su", "-c", "id"])
+        proc = self._runCmd(["shell", "su", "-c", "id"], timeout=self.short_timeout)
 
         # wait for response for maximum of 15 seconds, in case su prompts for a
         # password or triggers the Android SuperUser prompt
         start_time = time.time()
         retcode = None
         while (time.time() - start_time) <= 15 and retcode is None:
             retcode = proc.poll()
         if retcode is None: # still not terminated, kill
--- a/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
+++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
@@ -25,17 +25,16 @@ class DeviceManagerSUT(DeviceManager):
     app must be present and listening for connections for this to work.
     """
 
     _base_prompt = '$>'
     _base_prompt_re = '\$\>'
     _prompt_sep = '\x00'
     _prompt_regex = '.*(' + _base_prompt_re + _prompt_sep + ')'
     _agentErrorRE = re.compile('^##AGENT-WARNING##\ ?(.*)')
-    default_timeout = 300
 
     reboot_timeout = 600
     reboot_settling_time = 60
 
     def __init__(self, host, port = 20701, retryLimit = 5,
             deviceRoot = None, logLevel = mozlog.ERROR, **kwargs):
         DeviceManager.__init__(self, logLevel = logLevel,
                                deviceRoot = deviceRoot)
--- a/testing/mozbase/mozdevice/mozdevice/droid.py
+++ b/testing/mozbase/mozdevice/mozdevice/droid.py
@@ -153,17 +153,17 @@ class DroidMixin(object):
 class DroidADB(DeviceManagerADB, DroidMixin):
 
     _stopApplicationNeedsRoot = False
 
     def getTopActivity(self):
         package = None
         data = None
         try:
-            data = self.shellCheckOutput(["dumpsys", "window", "windows"])
+            data = self.shellCheckOutput(["dumpsys", "window", "windows"], timeout=self.short_timeout)
         except:
             # dumpsys seems to intermittently fail (seen on 4.3 emulator), producing
             # no output.
             return ""
         # "dumpsys window windows" produces many lines of input. The top/foreground
         # activity is indicated by something like:
         #   mFocusedApp=AppWindowToken{483e6db0 token=HistoryRecord{484dcad8 com.mozilla.SUTAgentAndroid/.SUTAgentAndroid}}
         # or, on other devices: