Bug 821865 - Update mozdevice to version 0.18, r=jhammel
authorAndrew Halberstadt <ahalberstadt@mozilla.com>
Thu, 20 Dec 2012 10:24:25 -0500
changeset 125800 3905e1085c13275704f92b5c3b485473f0e16f08
parent 125799 37ad8c11c3fb5772735fc49b5d3030c6327c1988
child 125801 73ce12dc950b5e4965c04f7ae3f480df1a60f6f5
push id2151
push userlsblakk@mozilla.com
push dateTue, 19 Feb 2013 18:06:57 +0000
treeherdermozilla-beta@4952e88741ec [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjhammel
bugs821865
milestone20.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 821865 - Update mozdevice to version 0.18, r=jhammel
testing/mozbase/mozdevice/mozdevice/devicemanager.py
testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
testing/mozbase/mozdevice/setup.py
testing/mozbase/mozdevice/tests/sut_mkdir.py
testing/mozbase/mozdevice/tests/sut_push.py
--- a/testing/mozbase/mozdevice/mozdevice/devicemanager.py
+++ b/testing/mozbase/mozdevice/mozdevice/devicemanager.py
@@ -61,17 +61,17 @@ class DeviceManager:
         retval = self.shell(cmd, buf, env=env, cwd=cwd, timeout=timeout, root=root)
         output = str(buf.getvalue()[0:-1]).rstrip()
         buf.close()
         if retval != 0:
             raise DMError("Non-zero return code for command: %s (output: '%s', retval: '%s')" % (cmd, output, retval))
         return output
 
     @abstractmethod
-    def pushFile(self, localname, destname):
+    def pushFile(self, localname, destname, retryLimit=1):
         """
         Copies localname from the host to destname on the device
         """
 
     @abstractmethod
     def mkDir(self, name):
         """
         Creates a single directory on the device file system
@@ -89,17 +89,17 @@ class DeviceManager:
             for part in parts:
                 if part == parts[-1]:
                     break
                 if part != "":
                     name += '/' + part
                     self.mkDir(name) # mkDir will check previous existence
 
     @abstractmethod
-    def pushDir(self, localDir, remoteDir):
+    def pushDir(self, localDir, remoteDir, retryLimit=1):
         """
         Push localDir from host to remoteDir on the device
         """
 
     @abstractmethod
     def fileExists(self, filepath):
         """
         Checks if filepath exists and is a file on the device file system
--- a/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
+++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py
@@ -18,21 +18,21 @@ class DeviceManagerADB(DeviceManager):
     _useDDCopy = False
     _useZip = False
     _logcatNeedsRoot = False
     _pollingInterval = 0.01
     _packageName = None
     _tempDir = None
     default_timeout = 300
 
-    def __init__(self, host=None, port=20701, retrylimit=5, packageName='fennec',
+    def __init__(self, host=None, port=20701, retryLimit=5, packageName='fennec',
                  adbPath='adb', deviceSerial=None, deviceRoot=None, **kwargs):
         self.host = host
         self.port = port
-        self.retrylimit = retrylimit
+        self.retryLimit = retryLimit
         self.deviceRoot = deviceRoot
 
         # the path to adb, or 'adb' to assume that it's on the PATH
         self._adbPath = adbPath
 
         # The serial number of the device to use with adb, used in cases
         # where multiple devices are being managed by the same adb instance.
         self._deviceSerial = deviceSerial
@@ -159,79 +159,84 @@ class DeviceManagerADB(DeviceManager):
         return None
 
     def _connectRemoteADB(self):
         self._checkCmd(["connect", self.host + ":" + str(self.port)])
 
     def _disconnectRemoteADB(self):
         self._checkCmd(["disconnect", self.host + ":" + str(self.port)])
 
-    def pushFile(self, localname, destname):
+    def pushFile(self, localname, destname, retryLimit=None):
         """
         Copies localname from the host to destname on the device
         """
         # you might expect us to put the file *in* the directory in this case,
         # but that would be different behaviour from devicemanagerSUT. Throw
         # an exception so we have the same behaviour between the two
         # implementations
+        retryLimit = retryLimit or self.retryLimit
         if self.dirExists(destname):
             raise DMError("Attempted to push a file (%s) to a directory (%s)!" %
                           (localname, destname))
 
         if self._useRunAs:
             remoteTmpFile = self.getTempDir() + "/" + os.path.basename(localname)
-            self._checkCmd(["push", os.path.realpath(localname), remoteTmpFile])
+            self._checkCmd(["push", os.path.realpath(localname), remoteTmpFile],
+                    retryLimit=retryLimit)
             if self._useDDCopy:
                 self.shellCheckOutput(["dd", "if=" + remoteTmpFile, "of=" + destname])
             else:
                 self.shellCheckOutput(["cp", remoteTmpFile, destname])
             self.shellCheckOutput(["rm", remoteTmpFile])
         else:
-            self._checkCmd(["push", os.path.realpath(localname), destname])
+            self._checkCmd(["push", os.path.realpath(localname), destname],
+                    retryLimit=retryLimit)
 
     def mkDir(self, name):
         """
         Creates a single directory on the device file system
         """
         result = self._runCmdAs(["shell", "mkdir", name]).stdout.read()
         if 'read-only file system' in result.lower():
             raise DMError("Error creating directory: read only file system")
 
-    def pushDir(self, localDir, remoteDir):
+    def pushDir(self, localDir, remoteDir, retryLimit=None):
         """
         Push localDir from host to remoteDir on the device
         """
         # adb "push" accepts a directory as an argument, but if the directory
         # contains symbolic links, the links are pushed, rather than the linked
         # files; we either zip/unzip or re-copy the directory into a temporary
         # one to get around this limitation
+        retryLimit = retryLimit or self.retryLimit
         if not self.dirExists(remoteDir):
             self.mkDirs(remoteDir+"/x")
         if self._useZip:
             try:
                 localZip = tempfile.mktemp() + ".zip"
                 remoteZip = remoteDir + "/adbdmtmp.zip"
                 subprocess.Popen(["zip", "-r", localZip, '.'], cwd=localDir,
                                  stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
-                self.pushFile(localZip, remoteZip)
+                self.pushFile(localZip, remoteZip, retryLimit=retryLimit)
                 os.remove(localZip)
-                data = self._runCmdAs(["shell", "unzip", "-o", remoteZip, "-d", remoteDir]).stdout.read()
-                self._checkCmdAs(["shell", "rm", remoteZip])
+                data = self._runCmdAs(["shell", "unzip", "-o", remoteZip,
+                                       "-d", remoteDir]).stdout.read()
+                self._checkCmdAs(["shell", "rm", remoteZip], retryLimit=retryLimit)
                 if re.search("unzip: exiting", data) or re.search("Operation not permitted", data):
                     raise Exception("unzip failed, or permissions error")
             except:
                 print "zip/unzip failure: falling back to normal push"
                 self._useZip = False
-                self.pushDir(localDir, remoteDir)
+                self.pushDir(localDir, remoteDir, retryLimit=retryLimit)
         else:
             tmpDir = tempfile.mkdtemp()
             # copytree's target dir must not already exist, so create a subdir
             tmpDirTarget = os.path.join(tmpDir, "tmp")
             shutil.copytree(localDir, tmpDirTarget)
-            self._checkCmd(["push", tmpDirTarget, remoteDir])
+            self._checkCmd(["push", tmpDirTarget, remoteDir], retryLimit=retryLimit)
             shutil.rmtree(tmpDir)
 
     def dirExists(self, remotePath):
         """
         Return True if remotePath is an existing directory on the device.
         """
         p = self._runCmd(["shell", "ls", "-a", remotePath + '/'])
 
@@ -255,28 +260,22 @@ class DeviceManagerADB(DeviceManager):
 
     def removeFile(self, filename):
         """
         Removes filename from the device
         """
         if self.fileExists(filename):
             self._runCmd(["shell", "rm", filename])
 
-    def _removeSingleDir(self, remoteDir):
-        """
-        Deletes a single empty directory
-        """
-        return self._runCmd(["shell", "rmdir", remoteDir]).stdout.read()
-
     def removeDir(self, remoteDir):
         """
         Does a recursive delete of directory on the device: rm -Rf remoteDir
         """
         if (self.dirExists(remoteDir)):
-            self._runCmd(["shell", "rm", "-r", remoteDir])
+            self._runCmd(["shell", "rm", "-r", remoteDir]).wait()
         else:
             self.removeFile(remoteDir.strip())
 
     def listFiles(self, rootdir):
         """
         Lists files on the device rootdir
 
         returns array of filenames, ['file1', 'file2', ...]
@@ -693,61 +692,67 @@ class DeviceManagerADB(DeviceManager):
         """
         if self._useRunAs:
             args.insert(1, "run-as")
             args.insert(2, self._packageName)
         return self._runCmd(args)
 
     # timeout is specified in seconds, and if no timeout is given,
     # we will run until we hit the default_timeout specified in the __init__
-    def _checkCmd(self, args, timeout=None):
+    def _checkCmd(self, args, timeout=None, retryLimit=None):
         """
         Runs a command using adb and waits for the command to finish.
         If timeout is specified, the process is killed after <timeout> seconds.
 
         returns: returncode from subprocess.Popen
         """
+        retryLimit = retryLimit or self.retryLimit
         # use run-as to execute commands as the package we're testing if
         # possible
         finalArgs = [self._adbPath]
         if self._deviceSerial:
             finalArgs.extend(['-s', self._deviceSerial])
         if not self._haveRootShell and self._useRunAs and \
                 args[0] == "shell" and args[1] not in [ "run-as", "am" ]:
             args.insert(1, "run-as")
             args.insert(2, self._packageName)
         finalArgs.extend(args)
         if not timeout:
             # We are asserting that all commands will complete in this time unless otherwise specified
             timeout = self.default_timeout
 
         timeout = int(timeout)
-        proc = subprocess.Popen(finalArgs)
-        start_time = time.time()
-        ret_code = proc.poll()
-        while ((time.time() - start_time) <= timeout) and ret_code == None:
-            time.sleep(self._pollingInterval)
+        retries = 0
+        while retries < retryLimit:
+            proc = subprocess.Popen(finalArgs)
+            start_time = time.time()
             ret_code = proc.poll()
-        if ret_code == None:
-            proc.kill()
-            raise DMError("Timeout exceeded for _checkCmd call")
-        return ret_code
+            while ((time.time() - start_time) <= timeout) and ret_code == None:
+                time.sleep(self._pollingInterval)
+                ret_code = proc.poll()
+            if ret_code == None:
+                proc.kill()
+                retries += 1
+                continue
+            return ret_code
+        raise DMError("Timeout exceeded for _checkCmd call after %d retries." % retries)
 
-    def _checkCmdAs(self, args, timeout=None):
+    def _checkCmdAs(self, args, timeout=None, retryLimit=None):
         """
         Runs a command using adb and waits for command to finish
         If self._useRunAs is True, the command is run-as user specified in self._packageName
         If timeout is specified, the process is killed after <timeout> seconds
 
         returns: returncode from subprocess.Popen
         """
+        retryLimit = retryLimit or self.retryLimit
         if (self._useRunAs):
             args.insert(1, "run-as")
             args.insert(2, self._packageName)
-        return self._checkCmd(args, timeout)
+        return self._checkCmd(args, timeout, retryLimit=retryLimit)
 
     def chmodDir(self, remoteDir, mask="777"):
         """
         Recursively changes file permissions in a directory
         """
         if (self.dirExists(remoteDir)):
             files = self.listFiles(remoteDir.strip())
             for f in files:
--- a/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
+++ b/testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py
@@ -20,20 +20,20 @@ class DeviceManagerSUT(DeviceManager):
     debug = 2
     _base_prompt = '$>'
     _base_prompt_re = '\$\>'
     _prompt_sep = '\x00'
     _prompt_regex = '.*(' + _base_prompt_re + _prompt_sep + ')'
     _agentErrorRE = re.compile('^##AGENT-WARNING##\ ?(.*)')
     default_timeout = 300
 
-    def __init__(self, host, port = 20701, retrylimit = 5, deviceRoot = None, **kwargs):
+    def __init__(self, host, port = 20701, retryLimit = 5, deviceRoot = None, **kwargs):
         self.host = host
         self.port = port
-        self.retrylimit = retrylimit
+        self.retryLimit = retryLimit
         self._sock = None
         self._everConnected = False
         self.deviceRoot = deviceRoot
 
         # Initialize device root
         self.getDeviceRoot()
 
         # Get version
@@ -94,54 +94,56 @@ class DeviceManagerSUT(DeviceManager):
                              re.compile('^rebt.*'),
                              re.compile('^uninst .*$')]
 
         for c in socketClosingCmds:
             if (c.match(cmd)):
                 return True
         return False
 
-    def _sendCmds(self, cmdlist, outputfile, timeout = None):
+    def _sendCmds(self, cmdlist, outputfile, timeout = None, retryLimit = None):
         """
-        Wrapper for _doCmds that loops up to self.retrylimit iterations
+        Wrapper for _doCmds that loops up to retryLimit iterations
         """
         # this allows us to move the retry logic outside of the _doCmds() to make it
         # easier for debugging in the future.
         # note that since cmdlist is a list of commands, they will all be retried if
         # one fails.  this is necessary in particular for pushFile(), where we don't want
         # to accidentally send extra data if a failure occurs during data transmission.
 
+        retryLimit = retryLimit or self.retryLimit
         retries = 0
-        while retries < self.retrylimit:
+        while retries < retryLimit:
             try:
                 self._doCmds(cmdlist, outputfile, timeout)
                 return
             except DMError, err:
                 # re-raise error if it's fatal (i.e. the device got the command but
                 # couldn't execute it). retry otherwise
                 if err.fatal:
                     raise err
                 if self.debug >= 4:
                     print err
                 retries += 1
                 # if we lost the connection or failed to establish one, wait a bit
-                if retries < self.retrylimit and not self._sock:
+                if retries < retryLimit and not self._sock:
                     sleep_time = 5 * retries
                     print 'Could not connect; sleeping for %d seconds.' % sleep_time
                     time.sleep(sleep_time)
 
-        raise DMError("Remote Device Error: unable to connect to %s after %s attempts" % (self.host, self.retrylimit))
+        raise DMError("Remote Device Error: unable to connect to %s after %s attempts" % (self.host, retryLimit))
 
-    def _runCmds(self, cmdlist, timeout = None):
+    def _runCmds(self, cmdlist, timeout = None, retryLimit = None):
         """
         Similar to _sendCmds, but just returns any output as a string instead of
         writing to a file
         """
+        retryLimit = retryLimit or self.retryLimit
         outputfile = StringIO.StringIO()
-        self._sendCmds(cmdlist, outputfile, timeout)
+        self._sendCmds(cmdlist, outputfile, timeout, retryLimit=retryLimit)
         outputfile.seek(0)
         return outputfile.read()
 
     def _doCmds(self, cmdlist, outputfile, timeout):
         promptre = re.compile(self._prompt_regex + '$')
         shouldCloseSocket = False
 
         if not timeout:
@@ -322,27 +324,28 @@ class DeviceManagerSUT(DeviceManager):
         if lastline:
             m = re.search('return code \[([0-9]+)\]', lastline)
             if m:
                 return int(m.group(1))
 
         # woops, we couldn't find an end of line/return value
         raise DMError("Automation Error: Error finding end of line/return value when running '%s'" % cmdline)
 
-    def pushFile(self, localname, destname):
+    def pushFile(self, localname, destname, retryLimit = None):
         """
         Copies localname from the host to destname on the device
         """
+        retryLimit = retryLimit or self.retryLimit
         self.mkDirs(destname)
 
         try:
             filesize = os.path.getsize(localname)
             with open(localname, 'rb') as f:
                 remoteHash = self._runCmds([{ 'cmd': 'push ' + destname + ' ' + str(filesize),
-                                              'data': f.read() }]).strip()
+                                              'data': f.read() }], retryLimit=retryLimit).strip()
         except OSError:
             raise DMError("DeviceManager: Error reading file to push")
 
         if (self.debug >= 3):
             print "push returned: %s" % hash
 
         localHash = self._getLocalHash(localname)
 
@@ -352,20 +355,21 @@ class DeviceManagerSUT(DeviceManager):
 
     def mkDir(self, name):
         """
         Creates a single directory on the device file system
         """
         if not self.dirExists(name):
             self._runCmds([{ 'cmd': 'mkdr ' + name }])
 
-    def pushDir(self, localDir, remoteDir):
+    def pushDir(self, localDir, remoteDir, retryLimit = None):
         """
         Push localDir from host to remoteDir on the device
         """
+        retryLimit = retryLimit or self.retryLimit
         if (self.debug >= 2):
             print "pushing directory: %s to %s" % (localDir, remoteDir)
 
         existentDirectories = []
         for root, dirs, files in os.walk(localDir, followlinks=True):
             parts = root.split(localDir)
             for f in files:
                 remoteRoot = remoteDir + '/' + parts[1]
@@ -377,17 +381,17 @@ class DeviceManagerSUT(DeviceManager):
                 if (parts[1] == ""):
                     remoteRoot = remoteDir
 
                 parent = os.path.dirname(remoteName)
                 if parent not in existentDirectories:
                     self.mkDirs(remoteName)
                     existentDirectories.append(parent)
 
-                self.pushFile(os.path.join(root, f), remoteName)
+                self.pushFile(os.path.join(root, f), remoteName, retryLimit=retryLimit)
 
 
     def dirExists(self, remotePath):
         """
         Return True if remotePath is an existing directory on the device.
         """
         ret = self._runCmds([{ 'cmd': 'isdir ' + remotePath }]).strip()
         if not ret:
--- a/testing/mozbase/mozdevice/setup.py
+++ b/testing/mozbase/mozdevice/setup.py
@@ -1,30 +1,22 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import os
 from setuptools import setup
 
-PACKAGE_VERSION = '0.17'
-
-# take description from README
-here = os.path.dirname(os.path.abspath(__file__))
-try:
-    description = file(os.path.join(here, 'README.md')).read()
-except (OSError, IOError):
-    description = ''
+PACKAGE_VERSION = '0.18'
 
 deps = ['mozprocess == 0.8']
 
 setup(name='mozdevice',
       version=PACKAGE_VERSION,
       description="Mozilla-authored device management",
-      long_description=description,
+      long_description="see http://mozbase.readthedocs.org/",
       classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
       keywords='',
       author='Mozilla Automation and Testing Team',
       author_email='tools@lists.mozilla.org',
       url='https://wiki.mozilla.org/Auto-tools/Projects/MozBase',
       license='MPL',
       packages=['mozdevice'],
       include_package_data=True,
--- a/testing/mozbase/mozdevice/tests/sut_mkdir.py
+++ b/testing/mozbase/mozdevice/tests/sut_mkdir.py
@@ -1,25 +1,27 @@
 from sut import MockAgent
 import mozdevice
 import unittest
 
 class PushTest(unittest.TestCase):
 
     def test_mkdirs(self):
-        subTests = [ { 'cmds': [ ("isdir /mnt", "TRUE"),
+        subTests = [ { 'cmds': [ ("isdir /mnt/sdcard/baz/boop", "FALSE"),
+                                 ("isdir /mnt", "TRUE"),
                                  ("isdir /mnt/sdcard", "TRUE"),
                                  ("isdir /mnt/sdcard/baz", "FALSE"),
                                  ("mkdr /mnt/sdcard/baz",
                                   "/mnt/sdcard/baz successfully created"),
                                  ("isdir /mnt/sdcard/baz/boop", "FALSE"),
                                  ("mkdr /mnt/sdcard/baz/boop",
                                   "/mnt/sdcard/baz/boop successfully created") ],
                        'expectException': False },
-                     { 'cmds': [ ("isdir /mnt", "TRUE"),
+                     { 'cmds': [ ("isdir /mnt/sdcard/baz/boop", "FALSE"),
+                                 ("isdir /mnt", "TRUE"),
                                  ("isdir /mnt/sdcard", "TRUE"),
                                  ("isdir /mnt/sdcard/baz", "FALSE"),
                                  ("mkdr /mnt/sdcard/baz",
                                   "##AGENT-WARNING## Could not create the directory /mnt/sdcard/baz") ],
                        'expectException': True },
                      ]
         for subTest in subTests:
             a = MockAgent(self, commands = subTest['cmds'])
--- a/testing/mozbase/mozdevice/tests/sut_push.py
+++ b/testing/mozbase/mozdevice/tests/sut_push.py
@@ -11,18 +11,17 @@ class PushTest(unittest.TestCase):
         pushfile = "1234ABCD"
         mdsum = hashlib.md5()
         mdsum.update(pushfile)
         expectedResponse = mdsum.hexdigest()
 
         # (good response, no exception), (bad response, exception)
         for response in [ (expectedResponse, False), ("BADHASH", True) ]:
             cmd = "push /mnt/sdcard/foobar %s" % len(pushfile)
-            a = MockAgent(self, commands = [("isdir /mnt", "TRUE"),
-                                            ("isdir /mnt/sdcard", "TRUE"),
+            a = MockAgent(self, commands = [("isdir /mnt/sdcard", "TRUE"),
                                             (cmd, response[0])])
             exceptionThrown = False
             with tempfile.NamedTemporaryFile() as f:
                 try:
                     f.write(pushfile)
                     f.flush()
                     d = mozdevice.DroidSUT("127.0.0.1", port=a.port)
                     d.pushFile(f.name, '/mnt/sdcard/foobar')
@@ -39,37 +38,30 @@ class PushTest(unittest.TestCase):
 
         tempdir = tempfile.mkdtemp()
         complex_path = os.path.join(tempdir, "baz")
         os.mkdir(complex_path)
         f = tempfile.NamedTemporaryFile(dir=complex_path)
         f.write(pushfile)
         f.flush()
 
-        subTests = [ { 'cmds': [ ("isdir /mnt", "TRUE"),
-                                 ("isdir /mnt/sdcard", "TRUE"),
-                                 ("isdir /mnt/sdcard/baz", "TRUE"),
-                                 ("isdir /mnt", "TRUE"),
-                                 ("isdir /mnt/sdcard", "TRUE"),
-                                 ("isdir /mnt/sdcard/baz", "TRUE"),
+        subTests = [ { 'cmds': [ ("isdir /mnt/sdcard//baz", "TRUE"),
+                                 ("isdir /mnt/sdcard//baz", "TRUE"),
                                  ("push /mnt/sdcard//baz/%s %s" %
                                   (os.path.basename(f.name), len(pushfile)),
                                   expectedFileResponse) ],
                        'expectException': False },
-                     { 'cmds': [ ("isdir /mnt", "TRUE"),
-                                 ("isdir /mnt/sdcard", "TRUE"),
-                                 ("isdir /mnt/sdcard/baz", "TRUE"),
-                                 ("isdir /mnt", "TRUE"),
-                                 ("isdir /mnt/sdcard", "TRUE"),
-                                 ("isdir /mnt/sdcard/baz", "TRUE"),
+                     { 'cmds': [ ("isdir /mnt/sdcard//baz", "TRUE"),
+                                 ("isdir /mnt/sdcard//baz", "TRUE"),
                                  ("push /mnt/sdcard//baz/%s %s" %
                                   (os.path.basename(f.name), len(pushfile)),
                                   "BADHASH") ],
                        'expectException': True },
-                     { 'cmds': [ ("isdir /mnt", "FALSE"),
+                     { 'cmds': [ ("isdir /mnt/sdcard//baz", "FALSE"),
+                                 ("isdir /mnt", "FALSE"),
                                  ("mkdr /mnt",
                                   "##AGENT-WARNING## Could not create the directory /mnt") ],
                        'expectException': True },
 
                      ]
 
         for subTest in subTests:
             a = MockAgent(self, commands = subTest['cmds'])