Bug 617815 - Enable the use of envrionment variables for remote automation on android r=ctalbert a=NPOTB
authorJoel Maher <jmaher@mozilla.com>
Thu, 16 Dec 2010 15:28:35 -0800
changeset 59424 18004ceae9ce0573eaaf6df1c2cc30a61fcd2401
parent 59423 a6525c0a9d7a4957ab0e6080fc878a3eba4116dd
child 59425 c6f86b5978e785136955204f24be6b8ed610f47c
push idunknown
push userunknown
push dateunknown
reviewersctalbert, NPOTB
bugs617815
milestone2.0b9pre
Bug 617815 - Enable the use of envrionment variables for remote automation on android r=ctalbert a=NPOTB
build/mobile/devicemanager.py
build/mobile/remoteautomation.py
--- a/build/mobile/devicemanager.py
+++ b/build/mobile/devicemanager.py
@@ -58,21 +58,23 @@ class FileError(Exception):
 
 class DeviceManager:
   host = ''
   port = 0
   debug = 2 
   _redo = False
   deviceRoot = None
   tempRoot = os.getcwd()
-  base_prompt = '\$\>'
+  base_prompt = '$>'
+  base_prompt_re = '\$\>'
   prompt_sep = '\x00'
-  prompt_regex = '.*' + base_prompt + prompt_sep
+  prompt_regex = '.*(' + base_prompt_re + prompt_sep + ')'
   agentErrorRE = re.compile('^##AGENT-WARNING##.*')
 
+
   def __init__(self, host, port = 20701):
     self.host = host
     self.port = port
     self._sock = None
     self.getDeviceRoot()
 
   def cmdNeedsResponse(self, cmd):
     """ Not all commands need a response from the agent:
@@ -80,17 +82,18 @@ class DeviceManager:
           and therefore we want to wait until the second half before looking
           for a response
         * rebt obviously doesn't get a response
         * uninstall performs a reboot to ensure starting in a clean state and
           so also doesn't look for a response
     """
     noResponseCmds = [re.compile('^push .*$'),
                       re.compile('^rebt'),
-                      re.compile('^uninst .*$')]
+                      re.compile('^uninst .*$'),
+                      re.compile('^pull .*$')]
 
     for c in noResponseCmds:
       if (c.match(cmd)):
         return False
     
     # If the command is not in our list, then it gets a response
     return True
 
@@ -331,23 +334,28 @@ class DeviceManager:
     listfiles = self.listFiles(containingpath)
     for f in listfiles:
       if (f == s[-1]):
         return True
     return False
 
   # list files on the device, requires cd to directory first
   def listFiles(self, rootdir):
+    rootdir = rootdir.rstrip('/')
     if (self.dirExists(rootdir) == False):
       return []  
     data = self.sendCMD(['cd ' + rootdir, 'ls'])
     if (data == None):
       return None
     retVal = self.stripPrompt(data)
-    return retVal.split('\n')
+    files = filter(lambda x: x, retVal.split('\n'))
+    if len(files) == 1 and files[0] == '<empty>':
+      # special case on the agent: empty directories return just the string "<empty>"
+      return []
+    return files
 
   def removeFile(self, filename):
     if (self.debug>= 2): print "removing file: " + filename
     return self.sendCMD(['rm ' + filename])
     
   # does a recursive delete of directory on the device: rm -Rf remoteDir
   def removeDir(self, remoteDir):
     self.sendCMD(['rmdr ' + remoteDir])
@@ -390,21 +398,24 @@ class DeviceManager:
     self.sendCMD(['exec ' + appname])
 
     #NOTE: we sleep for 30 seconds to allow the application to startup
     time.sleep(30)
 
     self.process = self.processExist(appname)
     if (self.debug >= 4): print "got pid: " + str(self.process) + " for process: " + str(appname)
 
-  def launchProcess(self, cmd, outputFile = "process.txt", cwd = ''):
+  def launchProcess(self, cmd, outputFile = "process.txt", cwd = '', env = ''):
     cmdline = subprocess.list2cmdline(cmd)
     if (outputFile == "process.txt" or outputFile == None):
       outputFile = self.getDeviceRoot() + '/' + "process.txt"
       cmdline += " > " + outputFile
+    
+    # Prepend our env to the command 
+    cmdline = ('%s ' % self.formatEnvString(env)) + cmdline
 
     self.fireProcess(cmdline)
     return outputFile
   
   #hardcoded: sleep interval of 5 seconds, timeout of 10 minutes
   def communicate(self, process, timeout = 600):
     interval = 5
     timed_out = True
@@ -420,26 +431,41 @@ class DeviceManager:
     if (timed_out == True):
       return None
 
     return [self.getFile(process, "temp.txt"), None]
 
 
   def poll(self, process):
     try:
-      if (self.processExist(process) == None):
+      if (self.processExist(process) == ''):
         return None
       return 1
     except:
       return None
     return 1
   
   # iterates process list and returns pid if exists, otherwise ''
   def processExist(self, appname):
     pid = ''
+
+    #remove the environment variables in the cli if they exist
+    parts = appname.split(' ')
+    for p in parts:
+      if (p is ''):
+        parts.remove(p)
+
+    if len(parts[0].strip('"').split('=')) > 1:
+      envvars = parts[0].strip('"').split(',')
+      for e in envvars:
+        env = e.split('=')
+        if (len(env) > 1):
+          os.environ[env[0]] = str(env[1])
+      appname = ' '.join(parts[1:])
+
   
     pieces = appname.split(' ')
     parts = pieces[0].split('/')
     app = parts[-1]
     procre = re.compile('.*' + app + '.*')
 
     procList = self.getProcessList()
     if (procList == None):
@@ -458,51 +484,144 @@ class DeviceManager:
     return True
 
   def getTempDir(self):
     retVal = ''
     data = self.sendCMD(['tmpd'])
     if (data == None):
       return None
     return self.stripPrompt(data).strip('\n')
+
+  def catFile(self, remoteFile):
+    data = self.sendCMD(['cat ' + remoteFile])
+    if data == None:
+        return None
+    return self.stripPrompt(data)
   
+  def pullFile(self, remoteFile):
+    def err(error_msg):
+        err_str = 'bad response to pull: %s!' % error_msg
+        print err_str
+        self._sock = None
+        raise FileError(err_str) 
+
+    def read(to_recv, error_msg):
+      data = self._sock.recv(to_recv)
+      if not data:
+        err(error_msg)
+        return None
+      return data
+
+    self.sendCMD(['pull ' + remoteFile])
+    buffer = ''
+    while not '\n' in buffer:
+      data = read(1024, 'could not find metadata')
+      if data == None:
+        return
+      buffer += data
+    nl = buffer.find('\n')
+    metadata = buffer[:nl]
+    print 'metadata: %s' % metadata
+    filedata = buffer[nl+1:]  # skip newline
+    sep = metadata.rfind(',')
+    if sep == -1:
+      err('could not find file size')
+      return None
+    filename = metadata[:sep]
+    filesizestr = metadata[sep+1:]
+    prompt = self.base_prompt + self.prompt_sep
+    try:
+        filesize = int(filesizestr)
+    except ValueError:
+      err('invalid file size')
+      return None
+    if filesize == -1:
+      while not '\n' in filedata:
+        data = read(1024, 'could not find metadata')
+        if data == None:
+          return None
+        filedata += data
+      nl = filedata.find('\n')
+      error_str = filedata[:nl]
+      filedata = filedata[nl+1:]
+      while filedata < len(prompt):
+        data = read(1024, 'could not find metadata')
+        if data == None:
+          return None
+      print 'error pulling file: %s' % error_str
+      return None
+
+    total_to_recv = filesize + len(prompt)
+    while len(filedata) < total_to_recv:
+      to_recv = min(total_to_recv - len(filedata), 1024)
+      data = read(to_recv, 'could not get all file data')
+      if data == None:
+        return None
+      filedata += data
+    if filedata[-len(prompt):] != prompt:
+      err('no prompt')
+      return filedata
+    return filedata[:-len(prompt)]
+
   # copy file from device (remoteFile) to host (localFile)
   def getFile(self, remoteFile, localFile = ''):
     if localFile == '':
-        localFile = os.path.join(self.tempRoot, "temp.txt")
+      localFile = os.path.join(self.tempRoot, "temp.txt")
   
-    promptre = re.compile(self.prompt_regex + '.*')
-    data = self.sendCMD(['cat ' + remoteFile])
-    if (data == None):
+    retVal = self.pullFile(remoteFile)
+    if retVal == None:
       return None
-    retVal = self.stripPrompt(data)
     fhandle = open(localFile, 'wb')
     fhandle.write(retVal)
     fhandle.close()
+    if not self.validateFile(remoteFile, localFile):
+      print 'failed to validate file when downloading %s!' % remoteFile
+      return None
     return retVal
     
   # copy directory structure from device (remoteDir) to host (localDir)
   def getDirectory(self, remoteDir, localDir):
     if (self.debug >= 2): print "getting files in '" + remoteDir + "'"
     filelist = self.listFiles(remoteDir)
     if (filelist == None):
       return None
     if (self.debug >= 3): print filelist
     if not os.path.exists(localDir):
       os.makedirs(localDir)
   
-    # TODO: is this a comprehensive file regex?
-    isFile = re.compile('^([a-zA-Z0-9_\-\. ]+)\.([a-zA-Z0-9]+)$')
     for f in filelist:
-      if (isFile.match(f)):
-        if (self.getFile(remoteDir + '/' + f, os.path.join(localDir, f)) == None):
+      if f == '.' or f == '..':
+        continue
+      remotePath = remoteDir + '/' + f
+      localPath = os.path.join(localDir, f)
+      print 'remotePath is %s' % remotePath
+      print 'localPath is %s' % localPath
+      try:
+        is_dir = self.isDir(remotePath)
+      except FileError:
+        print 'bad file "%s"!' % remotePath
+        continue
+      if is_dir:
+        if (self.getDirectory(remotePath, localPath) == None):
+          print 'aborted when getting directory'
           return None
       else:
-        if (self.getDirectory(remoteDir + '/' + f, os.path.join(localDir, f)) == None):
-          return None
+        # It's sometimes acceptable to have getFile() return None, such as
+        # when the agent encounters broken symlinks.
+        # FIXME: This should be improved so we know when a file transfer really
+        # failed.
+        self.getFile(remotePath, localPath)
+    return filelist
+
+  def isDir(self, remotePath):
+    data = self.sendCMD(['isdir ' + remotePath])
+    retVal = self.stripPrompt(data).strip()
+    if not retVal:
+      raise FileError('isdir returned null')
+    return retVal == 'TRUE'
 
   # true/false check if the two files have the same md5 sum
   def validateFile(self, remoteFile, localFile):
     remoteHash = self.getRemoteHash(remoteFile)
     localHash = self.getLocalHash(localFile)
 
     if (remoteHash == localHash):
         return True
@@ -741,39 +860,40 @@ class DeviceManager:
   port - port to await a callback ping to let us know that the device has updated properly
          defaults to 30000, and counts up from there if it finds a conflict
   Returns True if succeeds, False if not
   
   NOTE: We have no real way to know if the device gets updated or not due to the
         reboot that the udpate call forces on us.  We can't install our own heartbeat
         listener here because we run the risk of racing with other heartbeat listeners.
   """
-  def updateApp(self, appBundlePath, processName=None, destPath=None, ipAddr=None, port=None):
+  def updateApp(self, appBundlePath, processName=None, destPath=None, ipAddr=None, port=30000):
     status = None
     cmd = 'updt '
     if (processName == None):
       # Then we pass '' for processName
       cmd += "'' " + appBundlePath
     else:
       cmd += processName + ' ' + appBundlePath
 
     if (destPath):
       cmd += " " + destPath
 
-    ip, port = self.getCallbackIpAndPort(ipAddr, 30000)
+    if (ipAddr is not None):
+      ip, port = self.getCallbackIpAndPort(ipAddr, port)
 
-    cmd += " %s %s" % (ip, port)
-
-    if (self.debug > 3): print "updateApp using command: " + str(cmd)
+      cmd += " %s %s" % (ip, port)
 
-    # Set up our callback server
-    callbacksvr = callbackServer(ip, port, self.debug)
-    data = self.sendCMD([cmd])
-    status = callbacksvr.disconnect()
-    if (self.debug > 3): print "got status back: " + str(status)
+      if (self.debug > 3): print "updateApp using command: " + str(cmd)
+
+      # Set up our callback server
+      callbacksvr = callbackServer(ip, port, self.debug)
+      data = self.sendCMD([cmd])
+      status = callbacksvr.disconnect()
+      if (self.debug > 3): print "got status back: " + str(status)
 
     return status
 
   """
     return the current time on the device
   """
   def getCurrentTime(self):
     data = self.sendCMD(['clok'])
@@ -792,16 +912,37 @@ class DeviceManager:
     if (ip == None):
       ip = nettools.getLanIp()
     if (aPort != None):
       port = nettools.findOpenPort(ip, aPort)
     else:
       port = nettools.findOpenPort(ip, 30000)
     return ip, port
 
+  """
+    Returns a properly formatted env string for the agent.
+    Input - env, which is either None, '', or a dict
+    Output - a quoted string of the form: '"envvar1=val1,envvar2=val2..."'
+    If env is None or '' return '""' (empty quoted string)
+  """
+  def formatEnvString(self, env):
+    if (env == None or env == ''):
+      return '""'
+
+    envstr = '"'
+    # TODO: I believe this is inefficient for large dicts
+    for k, v in env.items():
+      envstr += ('%s=%s,' % (k, v))
+    
+    # kill the trailing comma, add the last quote
+    envstr = envstr.rstrip(',')
+    envstr += '"'
+
+    return envstr
+
 gCallbackData = ''
 
 class callbackServer():
   def __init__(self, ip, port, debuglevel):
     self.ip = ip
     self.port = port
     self.connected = False
     self.debug = debuglevel
--- a/build/mobile/remoteautomation.py
+++ b/build/mobile/remoteautomation.py
@@ -67,16 +67,31 @@ class RemoteAutomation(Automation):
         self._remoteProfile = remoteProfile
 
     def setProduct(self, product):
         self._product = product
         
     def setRemoteLog(self, logfile):
         self._remoteLog = logfile
 
+    # Set up what we need for the remote environment
+    def environment(self, env = None, xrePath = None, crashreporter = True):
+        # Because we are running remote, we don't want to mimic the local env
+        # so no copying of os.environ
+        if env is None:
+            env = {}
+
+        if crashreporter:
+            env['MOZ_CRASHREPORTER_NO_REPORT'] = '1'
+            env['MOZ_CRASHREPORTER'] = '1'
+        else:
+            env['MOZ_CRASHREPORTER_DISABLE'] = '1'
+
+        return env
+
     def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime, debuggerInfo, symbolsDir):
         # maxTime is used to override the default timeout, we should honor that
         status = proc.wait(timeout = maxTime)
 
         print proc.stdout
 
         if (status == 1 and self._devicemanager.processExist(proc.procName)):
             # Then we timed out, make sure Fennec is dead
@@ -110,17 +125,17 @@ class RemoteAutomation(Automation):
         return self.RProcess(self._devicemanager, cmd, stdout, stderr, env, cwd)
 
     # be careful here as this inner class doesn't have access to outer class members    
     class RProcess(object):
         # device manager process
         dm = None
         def __init__(self, dm, cmd, stdout = None, stderr = None, env = None, cwd = '.'):
             self.dm = dm
-            self.proc = dm.launchProcess(cmd, stdout)
+            self.proc = dm.launchProcess(cmd, stdout, cwd, env)
             exepath = cmd[0]
             name = exepath.split('/')[-1]
             self.procName = name
 
             # Setting timeout at 1 hour since on a remote device this takes much longer
             self.timeout = 3600
             time.sleep(15)