Bug 573281 Update Devicemanager.py for android r=jmaher
authorClint Talbert <ctalbert@mozilla.com>
Mon, 19 Jul 2010 08:55:42 -0700
changeset 47927 ee164ea7d1502db4362252019f0cb48b8f80cf27
parent 47926 659b3ff05e374c5b6dacf38123bcb4b5b1b8b40b
child 47928 093a3e61ea504657f08e8ec3c70e9f231ed063a5
push id14482
push userctalbert@mozilla.com
push dateMon, 19 Jul 2010 15:59:24 +0000
treeherdermozilla-central@093a3e61ea50 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs573281
milestone2.0b2pre
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 573281 Update Devicemanager.py for android r=jmaher
build/mobile/devicemanager.py
--- a/build/mobile/devicemanager.py
+++ b/build/mobile/devicemanager.py
@@ -14,17 +14,18 @@
 # The Original Code is Test Automation Framework.
 #
 # The Initial Developer of the Original Code is Joel Maher.
 #
 # Portions created by the Initial Developer are Copyright (C) 2009
 # the Initial Developer. All Rights Reserved.
 #
 # Contributor(s):
-# Joel Maher <joel.maher@gmail.com> (Original Developer)
+#   Joel Maher <joel.maher@gmail.com> (Original Developer)
+#   Clint Talbert <cmtalbert@gmail.com>
 #
 # Alternatively, the contents of this file may be used under the terms of
 # either the GNU General Public License Version 2 or later (the "GPL"), or
 # the GNU Lesser General Public License Version 2.1 or later (the "LGPL"),
 # in which case the provisions of the GPL or the LGPL are applicable instead
 # of those above. If you wish to allow use of your version of this file only
 # under the terms of either the GPL or the LGPL, and not to allow others to
 # use your version of this file under the terms of the MPL, indicate your
@@ -59,30 +60,68 @@ class DeviceManager:
   port = 0
   debug = 3
   _redo = False
   deviceRoot = None
   tempRoot = os.getcwd()
   base_prompt = '\$\>'
   prompt_sep = '\x00'
   prompt_regex = '.*' + base_prompt + prompt_sep
+  agentErrorRE = re.compile('^##AGENT-ERROR##.*')
 
-  def __init__(self, host, port = 27020):
+  def __init__(self, host, port = 20701):
     self.host = host
     self.port = port
     self._sock = None
     self.getDeviceRoot()
 
-  def sendCMD(self, cmdline, newline = True, sleep = 0):
-    promptre = re.compile(self.prompt_regex + '$')
+  def cmdNeedsResponse(self, cmd):
+    """ Not all commands need a response from the agent:
+        * if the cmd matches the pushRE then it is the first half of push
+          and therefore we want to wait until the second half before looking
+          for a response
+        * rebt obviously doesn't get a response
+        * uninstall performs a reboot to ensure starting in a clean state and
+          so also doesn't look for a response
+    """
+    noResponseCmds = [re.compile('^push .*$'),
+                      re.compile('^rebt'),
+                      re.compile('^uninst .*$')]
+
+    for c in noResponseCmds:
+      if (c.match(cmd)):
+        return False
+    
+    # If the command is not in our list, then it gets a response
+    return True
 
-    # TODO: any commands that don't output anything and quit need to match this RE
-    pushre = re.compile('^push .*$')
+  def shouldCmdCloseSocket(self, cmd):
+    """ Some commands need to close the socket after they are sent:
+    * push
+    * rebt
+    * uninst
+    * quit
+    """
+    
+    socketClosingCmds = [re.compile('^push .*$'),
+                         re.compile('^quit.*'),
+                         re.compile('^rebt.*'),
+                         re.compile('^uninst .*$')]
+
+    for c in socketClosingCmds:
+      if (c.match(cmd)):
+        return True
+
+    return False
+
+  def sendCMD(self, cmdline, newline = True):
+    promptre = re.compile(self.prompt_regex + '$')
     data = ""
-    noQuit = False
+    shouldCloseSocket = False
+    recvGuard = 1000
 
     if (self._sock == None):
       try:
         if (self.debug >= 1):
           print "reconnecting socket"
         self._sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
       except:
         self._redo = True
@@ -96,53 +135,73 @@ class DeviceManager:
         self._sock.recv(1024)
       except:
         self._redo = True
         self._sock.close()
         self._sock = None
         if (self.debug >= 2):
           print "unable to connect socket"
         return None
-      
+    
     for cmd in cmdline:
-      if (cmd == 'quit'): break
       if newline: cmd += '\r\n'
       
       try:
-        self._sock.send(cmd)
+        numbytes = self._sock.send(cmd)
+        if (numbytes != len(cmd)):
+          print "ERROR: our cmd was " + str(len(cmd)) + " bytes and we only sent " + str(numbytes)
+          return None
         if (self.debug >= 4): print "send cmd: " + str(cmd)
       except:
         self._redo = True
         self._sock.close()
         self._sock = None
         return None
       
-      if (pushre.match(cmd) or cmd == 'rebt'):
-        noQuit = True
-      elif noQuit == False:
-        time.sleep(int(sleep))
+      # Check if the command should close the socket
+      shouldCloseSocket = self.shouldCmdCloseSocket(cmd)
+
+      # Handle responses from commands
+      if (self.cmdNeedsResponse(cmd)):
         found = False
-        while (found == False):
+        loopguard = 0
+        # TODO: We had an old sleep here but we don't need it
+
+        while (found == False and (loopguard < recvGuard)):
           if (self.debug >= 4): print "recv'ing..."
-          
+
+          # Get our response
           try:
             temp = self._sock.recv(1024)
+            if (self.debug >= 4): print "response: " + str(temp)
           except:
             self._redo = True
             self._sock.close()
             self._sock = None
             return None
+
+          # If something goes wrong in the agent it will send back a string that
+          # starts with '##AGENT-ERROR##'
+          if (self.agentErrorRE.match(temp)):
+            data = temp
+            break
+
           lines = temp.split('\n')
+
           for line in lines:
             if (promptre.match(line)):
               found = True
           data += temp
 
-    time.sleep(int(sleep))
-    if (noQuit == True):
+          # If we violently lose the connection to the device, this loop tends to spin,
+          # this guard prevents that
+          loopguard = loopguard + 1
+
+    # TODO: We had an old sleep here but we don't need it
+    if (shouldCloseSocket == True):
       try:
         self._sock.close()
         self._sock = None
       except:
         self._redo = True
         self._sock = None
         return None
 
@@ -175,38 +234,49 @@ class DeviceManager:
       return ''
 
     if self.mkDirs(destname) == None:
       print "unable to make dirs: " + destname
       return None
 
     if (self.debug >= 2): print "sending: push " + destname
     
-    # sleep 5 seconds / MB
     filesize = os.path.getsize(localname)
-    sleepsize = 1024 * 1024
-    sleepTime = (int(filesize / sleepsize) * 5) + 2
     f = open(localname, 'rb')
     data = f.read()
     f.close()
-    retVal = self.sendCMD(['push ' + destname + '\r\n', data], newline = False, sleep = sleepTime)
-    if (retVal == None):
-      if (self.debug >= 2): print "Error in sendCMD, not validating push"
-      return None
+    retVal = self.sendCMD(['push ' + destname + ' ' + str(filesize) + '\r\n', data], newline = False)
+    
+    if (self.debug >= 3): print "push returned: " + str(retVal)
 
-    if (self.validateFile(destname, localname) == False):
-      if (self.debug >= 2): print "file did not copy as expected"
+    validated = False
+    if (retVal):
+      retline = self.stripPrompt(retVal).strip() 
+      if (retline == None or self.agentErrorRE.match(retVal)):
+        # Then we failed to get back a hash from agent, try manual validation
+        validated = self.validateFile(destname, localname)
+      else:
+        # Then we obtained a hash from push
+        localHash = self.getLocalHash(localname)
+        if (str(localHash) == str(retline)):
+          validated = True
+    else:
+      # We got nothing back from sendCMD, try manual validation
+      validated = self.validateFile(destname, localname)
+
+    if (validated):
+      if (self.debug >= 2): print "Push File Validated!"
+      return True
+    else:
+      if (self.debug >= 2): print "Push File Failed to Validate!"
       return None
-
-    return retVal
   
   def mkDir(self, name):
     return self.sendCMD(['mkdr ' + name])
   
-  
   # make directory structure on the device
   def mkDirs(self, filename):
     parts = filename.split('/')
     name = ""
     for part in parts:
       if (part == parts[-1]): break
       if (part != ""):
         name += '/' + part
@@ -220,28 +290,25 @@ class DeviceManager:
     if (self.debug >= 2): print "pushing directory: " + localDir + " to " + remoteDir
     for root, dirs, files in os.walk(localDir):
       parts = root.split(localDir)
       for file in files:
         remoteRoot = remoteDir + '/' + parts[1]
         remoteName = remoteRoot + '/' + file
         if (parts[1] == ""): remoteRoot = remoteDir
         if (self.pushFile(os.path.join(root, file), remoteName) == None):
-          time.sleep(5)
           self.removeFile(remoteName)
-          time.sleep(5)
           if (self.pushFile(os.path.join(root, file), remoteName) == None):
             return None
     return True
 
-
   def dirExists(self, dirname):
     match = ".*" + dirname + "$"
     dirre = re.compile(match)
-    data = self.sendCMD(['cd ' + dirname, 'cwd', 'quit'], sleep = 1)
+    data = self.sendCMD(['cd ' + dirname, 'cwd'])
     if (data == None):
       return None
     retVal = self.stripPrompt(data)
     data = retVal.split('\n')
     found = False
     for d in data:
       if (dirre.match(d)): 
         found = True
@@ -258,34 +325,32 @@ class DeviceManager:
       if (f == s[-1]):
         return True
     return False
 
   # list files on the device, requires cd to directory first
   def listFiles(self, rootdir):
     if (self.dirExists(rootdir) == False):
       return []  
-    data = self.sendCMD(['cd ' + rootdir, 'ls', 'quit'], sleep=1)
+    data = self.sendCMD(['cd ' + rootdir, 'ls'])
     if (data == None):
       return None
     retVal = self.stripPrompt(data)
     return retVal.split('\n')
 
   def removeFile(self, filename):
     if (self.debug>= 2): print "removing file: " + filename
-    return self.sendCMD(['rm ' + filename, 'quit'])
-  
-  
+    return self.sendCMD(['rm ' + filename])
+    
   # does a recursive delete of directory on the device: rm -Rf remoteDir
   def removeDir(self, remoteDir):
-    self.sendCMD(['rmdr ' + remoteDir], sleep = 5)
-
+    self.sendCMD(['rmdr ' + remoteDir])
 
   def getProcessList(self):
-    data = self.sendCMD(['ps'], sleep = 3)
+    data = self.sendCMD(['ps'])
     if (data == None):
       return None
       
     retVal = self.stripPrompt(data)
     lines = retVal.split('\n')
     files = []
     for line in lines:
       if (line.strip() != ''):
@@ -293,17 +358,17 @@ class DeviceManager:
         if (len(pidproc) == 2):
           files += [[pidproc[0], pidproc[1]]]
         elif (len(pidproc) == 3):
           #android returns <userID> <procID> <procName>
           files += [[pidproc[1], pidproc[2], pidproc[0]]]     
     return files
 
   def getMemInfo(self):
-    data = self.sendCMD(['mems', 'quit'])
+    data = self.sendCMD(['mems'])
     if (data == None):
       return None
     retVal = self.stripPrompt(data)
     # TODO: this is hardcoded for now
     fhandle = open("memlog.txt", 'a')
     fhandle.write("\n")
     fhandle.write(retVal)
     fhandle.close()
@@ -355,18 +420,16 @@ class DeviceManager:
     try:
       if (self.processExist(process) == None):
         return None
       return 1
     except:
       return None
     return 1
   
-
-
   # iterates process list and returns pid if exists, otherwise ''
   def processExist(self, appname):
     pid = ''
   
     pieces = appname.split(' ')
     parts = pieces[0].split('/')
     app = parts[-1]
     procre = re.compile('.*' + app + '.*')
@@ -376,47 +439,44 @@ class DeviceManager:
       return None
       
     for proc in procList:
       if (procre.match(proc[1])):
         pid = proc[0]
         break
     return pid
 
-
   def killProcess(self, appname):
     if (self.sendCMD(['kill ' + appname]) == None):
       return None
 
     return True
 
   def getTempDir(self):
     retVal = ''
-    data = self.sendCMD(['tmpd', 'quit'])
+    data = self.sendCMD(['tmpd'])
     if (data == None):
       return None
     return self.stripPrompt(data).strip('\n')
-
   
   # copy file from device (remoteFile) to host (localFile)
   def getFile(self, remoteFile, localFile = ''):
     if localFile == '':
         localFile = os.path.join(self.tempRoot, "temp.txt")
   
     promptre = re.compile(self.prompt_regex + '.*')
-    data = self.sendCMD(['cat ' + remoteFile, 'quit'], sleep = 5)
+    data = self.sendCMD(['cat ' + remoteFile])
     if (data == None):
       return None
     retVal = self.stripPrompt(data)
     fhandle = open(localFile, 'wb')
     fhandle.write(retVal)
     fhandle.close()
     return retVal
-  
-  
+    
   # copy directory structure from device (remoteDir) to host (localDir)
   def getDirectory(self, remoteDir, localDir):
     if (self.debug >= 2): print "getting files in '" + remoteDir + "'"
     filelist = self.listFiles(remoteDir)
     if (filelist == None):
       return None
     if (self.debug >= 3): print filelist
     if not os.path.exists(localDir):
@@ -427,40 +487,37 @@ class DeviceManager:
     for f in filelist:
       if (isFile.match(f)):
         if (self.getFile(remoteDir + '/' + f, os.path.join(localDir, f)) == None):
           return None
       else:
         if (self.getDirectory(remoteDir + '/' + f, os.path.join(localDir, f)) == None):
           return None
 
-
   # true/false check if the two files have the same md5 sum
   def validateFile(self, remoteFile, localFile):
     remoteHash = self.getRemoteHash(remoteFile)
     localHash = self.getLocalHash(localFile)
 
     if (remoteHash == localHash):
         return True
 
     return False
-
   
   # return the md5 sum of a remote file
   def getRemoteHash(self, filename):
-      data = self.sendCMD(['hash ' + filename, 'quit'], sleep = 1)
+      data = self.sendCMD(['hash ' + filename])
       if (data == None):
           return ''
       retVal = self.stripPrompt(data)
       if (retVal != None):
         retVal = retVal.strip('\n')
       if (self.debug >= 3): print "remote hash returned: '" + retVal + "'"
       return retVal
     
-
   # return the md5 sum of a file on the host
   def getLocalHash(self, filename):
       file = open(filename, 'rb')
       if (file == None):
           return None
 
       try:
         mdsum = hashlib.md5()
@@ -487,17 +544,17 @@ class DeviceManager:
   # /tests
   #       /<fennec>|<firefox>  --> approot
   #       /profile
   #       /xpcshell
   #       /reftest
   #       /mochitest
   def getDeviceRoot(self):
     if (not self.deviceRoot):
-      data = self.sendCMD(['testroot'], sleep = 1)
+      data = self.sendCMD(['testroot'])
       if (data == None):
         return '/tests'
       self.deviceRoot = self.stripPrompt(data).strip('\n') + '/tests'
 
     if (not self.dirExists(self.deviceRoot)):
       self.mkDir(self.deviceRoot)
 
     return self.deviceRoot
@@ -545,17 +602,16 @@ class DeviceManager:
       dir = '/' + filename
     elif self.fileExists(self.getDeviceRoot() + '/' + filename):
       dir = self.getDeviceRoot() + '/' + filename
     else:
       return None
 
     return self.sendCMD(['cd ' + dir, 'unzp ' + filename])
 
-
   def reboot(self, wait = False):
     self.sendCMD(['rebt'])
 
     if wait == True:
       time.sleep(30)
       timeout = 270
       done = False
       while (not done):
@@ -577,75 +633,86 @@ class DeviceManager:
         remoteRoot = remoteDir + '/' + parts[1]
         remoteRoot = remoteRoot.replace('/', '/')
         if (parts[1] == ""): remoteRoot = remoteDir
         remoteName = remoteRoot + '/' + file
         if (self.validateFile(remoteName, os.path.join(root, file)) <> True):
             return None
     return True
 
-  #TODO: make this simpler by doing a single directive at a time
   # Returns information about the device:
   # Directive indicates the information you want to get, your choices are:
   # os - name of the os
   # id - unique id of the device
   # uptime - uptime of the device
   # systime - system time of the device
   # screen - screen resolution
   # memory - memory stats
   # process - list of running processes (same as ps)
   # disk - total, free, available bytes on disk
   # power - power status (charge, battery temp)
-  # all - all of them
-  def getInfo(self, directive):
+  # all - all of them - or call it with no parameters to get all the information
+  def getInfo(self, directive=None):
     data = None
-    if (directive in ('os','id','uptime','systime','screen','memory','process',
-                      'disk','power')):
-      data = self.sendCMD(['info ' + directive, 'quit'], sleep = 1)
-    else:
-      directive = None
-      data = self.sendCMD(['info', 'quit'], sleep = 1)
+    result = {}
+    collapseSpaces = re.compile('  +')
 
-    if (data is None):
-      return None
-      
-    data = self.stripPrompt(data)
-    result = {}
-        
-    if directive:
-      result[directive] = data.split('\n')
-      for i in range(len(result[directive])):
-        if (len(result[directive][i]) != 0):
-          result[directive][i] = result[directive][i].strip()
-
-      # Get rid of any empty attributes
-      result[directive].remove('')
+    directives = ['os', 'id','uptime','systime','screen','memory','process',
+                  'disk','power']
+    if (directive in directives):
+      directives = [directive]
 
-    else:
-      lines = data.split('\n')
-      result['id'] = lines[0]
-      result['os'] = lines[1]
-      result['systime'] = lines[2]
-      result['uptime'] = lines[3]
-      result['screen'] = lines[4]
-      result['memory'] = lines[5]
-      if (lines[6] == 'Power status'):
-        tmp = []
-        for i in range(4):
-          tmp.append(line[7 + i])
-        result['power'] = tmp
-      tmp = []
+    for d in directives:
+      data = self.sendCMD(['info ' + d])
+      if (data is None):
+        continue
+      data = self.stripPrompt(data)
+      data = collapseSpaces.sub(' ', data)
+      result[d] = data.split('\n')
 
-      # Linenum is the line where the process list begins
-      linenum = 11
-      for j in range(len(lines) - linenum):
-        if (lines[j + linenum].strip() != ''):
-          procline = lines[j + linenum].split('\t')
+    # Get rid of any 0 length members of the arrays
+    for v in result.itervalues():
+      while '' in v:
+        v.remove('')
+    
+    # Format the process output
+    if 'process' in result:
+      proclist = []
+      for l in result['process']:
+        if l:
+          proclist.append(l.split('\t'))
+      result['process'] = proclist
 
-          if len(procline) == 2:
-            tmp.append([procline[0], procline[1]])
-          elif len(procline) == 3:
-            # Android has <userid> <procid> <procname>
-            # We put the userid to achieve a common format
-            tmp.append([procline[1], procline[2], procline[0]])
-      result['process'] = tmp
+    print "results: " + str(result)
     return result
 
+  """
+  Installs the application onto the device
+  Application bundle - path to the application bundle on the device
+  Destination - destination directory of where application should be
+                installed to (optional)
+  Returns True or False depending on what we get back
+  TODO: we need a real way to know if this works or not
+  """
+  def installApp(self, appBundlePath, destPath=None):
+    cmd = 'inst ' + appBundlePath
+    if destPath:
+      cmd += ' ' + destPath
+    data = self.sendCMD([cmd])
+    if (data is None):
+      return False
+    else:
+      return True
+
+  """
+  Uninstalls the named application from device and causes a reboot.
+  Takes an optional argument of installation path - the path to where the application
+  was installed.
+  Returns True, but it doesn't mean anything other than the command was sent,
+  the reboot happens and we don't know if this succeeds or not.
+  """
+  def uninstallAppAndReboot(self, appName, installPath=None):
+    cmd = 'uninst ' + appName
+    if installPath:
+      cmd += ' ' + installPath
+    self.sendCMD([cmd])
+    return True
+