Bug 650205 - Implement devicemanager using adb. r=jmaher, a=test-only
authorBrad Lassey <blassey@mozilla.com>
Fri, 06 May 2011 18:17:55 -0400
changeset 69401 43949eb48546a92d6d7f017ed4c0b8825fefd477
parent 69400 8ad0dfefa6fa95d7cfcae02a57826e93bddfc632
child 69402 57af04c3dec40909895e694892936f8beb481a1d
push id19961
push userjmaher@mozilla.com
push dateThu, 12 May 2011 17:10:23 +0000
treeherdermozilla-central@43949eb48546 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher, test-only
bugs650205
milestone6.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 650205 - Implement devicemanager using adb. r=jmaher, a=test-only
build/mobile/devicemanager-utils.py
build/mobile/devicemanager.py
build/mobile/devicemanagerADB.py
layout/tools/reftest/Makefile.in
layout/tools/reftest/remotereftest.py
testing/mochitest/Makefile.in
testing/mochitest/runtestsremote.py
--- a/build/mobile/devicemanager-utils.py
+++ b/build/mobile/devicemanager-utils.py
@@ -32,17 +32,16 @@
 # the provisions above, a recipient may use your version of this file under
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
 import devicemanager
 import sys
 import os
-import devicemanagerSUT
 
 def copy(dm, gre_path):
     file = sys.argv[2]
     if len(sys.argv) >= 4:
         path = sys.argv[3]
     else:
         path = gre_path
     if os.path.isdir(file):
--- a/build/mobile/devicemanager.py
+++ b/build/mobile/devicemanager.py
@@ -36,16 +36,17 @@
 # the terms of any one of the MPL, the GPL or the LGPL.
 #
 # ***** END LICENSE BLOCK *****
 
 import time
 import hashlib
 import socket
 import os
+import re
 
 class FileError(Exception):
   " Signifies an error which occurs while doing a file operation."
 
   def __init__(self, msg = ''):
     self.msg = msg
 
   def __str__(self):
new file mode 100644
--- /dev/null
+++ b/build/mobile/devicemanagerADB.py
@@ -0,0 +1,439 @@
+import subprocess
+from devicemanager import DeviceManager, DMError
+import re
+
+class DeviceManagerADB(DeviceManager):
+
+  def __init__(self, host = None, port = 20701, retrylimit = 5):
+    self.host = host
+    self.port = port
+    self.retrylimit = retrylimit
+    self.retries = 0
+    self._sock = None
+    self.getDeviceRoot()
+    try:
+      # a test to see if we have root privs
+      self.checkCmd(["shell", "ls", "/sbin"])
+    except:
+      try:
+        self.checkCmd(["root"])
+      except:
+        print "restarting as root failed"
+
+  # external function
+  # returns:
+  #  success: True
+  #  failure: False
+  def pushFile(self, localname, destname):
+    try:
+      self.checkCmd(["push", localname, destname])
+      self.chmodDir(destname)
+      return True
+    except:
+      return False
+
+  # external function
+  # returns:
+  #  success: directory name
+  #  failure: None
+  def mkDir(self, name):
+    try:
+      self.checkCmd(["shell", "mkdir", name])
+      return name
+    except:
+      return None
+
+  # make directory structure on the device
+  # external function
+  # returns:
+  #  success: directory structure that we created
+  #  failure: None
+  def mkDirs(self, filename):
+    self.checkCmd(["shell", "mkdir", "-p ", name])
+    return filename
+
+  # push localDir from host to remoteDir on the device
+  # external function
+  # returns:
+  #  success: remoteDir
+  #  failure: None
+  def pushDir(self, localDir, remoteDir):
+    try:
+      self.checkCmd(["push", localDir, remoteDir])
+      self.chmodDir(remoteDir)
+      return True
+    except:
+      print "pushing " + localDir + " to " + remoteDir + " failed"
+      return False
+
+  # external function
+  # returns:
+  #  success: True
+  #  failure: False
+  def dirExists(self, dirname):
+    try:
+      self.checkCmd(["shell", "ls", dirname])
+      return True
+    except:
+      return False
+
+  # Because we always have / style paths we make this a lot easier with some
+  # assumptions
+  # external function
+  # returns:
+  #  success: True
+  #  failure: False
+  def fileExists(self, filepath):
+    self.checkCmd(["shell", "ls", filepath])
+    return True
+
+  def removeFile(self, filename):
+    return self.runCmd(["shell", "rm", filename]).stdout.read()
+
+  # does a recursive delete of directory on the device: rm -Rf remoteDir
+  # external function
+  # returns:
+  #  success: output of telnet, i.e. "removing file: /mnt/sdcard/tests/test.txt"
+  #  failure: None
+  def removeSingleDir(self, remoteDir):
+    return self.runCmd(["shell", "rmdir", remoteDir]).stdout.read()
+
+  # does a recursive delete of directory on the device: rm -Rf remoteDir
+  # external function
+  # returns:
+  #  success: output of telnet, i.e. "removing file: /mnt/sdcard/tests/test.txt"
+  #  failure: None
+  def removeDir(self, remoteDir):
+      out = ""
+      if (self.isDir(remoteDir)):
+          files = self.listFiles(remoteDir.strip())
+          for f in files:
+              if (self.isDir(remoteDir.strip() + "/" + f.strip())):
+                  out += self.removeDir(remoteDir.strip() + "/" + f.strip())
+              else:
+                  out += self.removeFile(remoteDir.strip())
+          out += self.removeSingleDir(remoteDir)
+      else:
+          out += self.removeFile(remoteDir.strip())
+      return out
+
+  def isDir(self, remotePath):
+      p = self.runCmd(["shell", "ls", remotePath])
+      data = p.stdout.readlines()
+      if (len(data) == 0):
+          return True
+      if (len(data) == 1):
+          if (data[0] == remotePath):
+              return False
+          if (data[0].find("No such file or directory") != -1):
+              return False
+          if (data[0].find("Not a directory") != -1):
+              return False
+      return True
+
+  def listFiles(self, rootdir):
+      p = self.runCmd(["shell", "ls", rootdir])
+      data = p.stdout.readlines()
+      if (len(data) == 1):
+          if (data[0] == rootdir):
+              return []
+          if (data[0].find("No such file or directory") != -1):
+              return []
+          if (data[0].find("Not a directory") != -1):
+              return []
+      return data
+
+  # external function
+  # returns:
+  #  success: array of process tuples
+  #  failure: []
+  def getProcessList(self):
+    p = self.runCmd(["shell", "ps"])
+      # first line is the headers
+    p.stdout.readline()
+    proc = p.stdout.readline()
+    ret = []
+    while (proc):
+      els = proc.split()
+      ret.append(list([els[1], els[len(els) - 1], els[0]]))
+      proc =  p.stdout.readline()
+    return ret
+
+  # external function
+  # returns:
+  #  success: pid
+  #  failure: None
+  def fireProcess(self, appname, failIfRunning=False):
+    return self.runCmd(["shell", appname]).pid
+
+  # external function
+  # returns:
+  #  success: output filename
+  #  failure: None
+  def launchProcess(self, cmd, outputFile = "process.txt", cwd = '', env = '', failIfRunning=False):
+    acmd = ["shell", "am","start"]
+    cmd = ' '.join(cmd)
+    i = cmd.find(" ")
+    acmd.append("-n")
+    acmd.append(cmd[0:i] + "/.App")
+    acmd.append("--es")
+    acmd.append("args")
+    acmd.append(cmd[i:])
+    print acmd
+    self.checkCmd(acmd)
+    return outputFile;
+
+  # external function
+  # returns:
+  #  success: output from testagent
+  #  failure: None
+  def killProcess(self, appname):
+    procs = self.getProcessList()
+    for proc in procs:
+      if (proc[1] == appname):
+        p = self.runCmd(["shell", "ps"])
+        return p.stdout.read()
+      return None
+
+  # external function
+  # returns:
+  #  success: filecontents
+  #  failure: None
+  def catFile(self, remoteFile):
+    #p = self.runCmd(["shell", "cat", remoteFile])
+    #return p.stdout.read()
+    return self.getFile(remoteFile)
+
+  # external function
+  # returns:
+  #  success: output of pullfile, string
+  #  failure: None
+  def pullFile(self, remoteFile):
+    #return self.catFile(remoteFile)
+    return self.getFile(remoteFile)
+
+  # copy file from device (remoteFile) to host (localFile)
+  # external function
+  # returns:
+  #  success: output of pullfile, string
+  #  failure: None
+  def getFile(self, remoteFile, localFile = 'tmpfile_dm_adb'):
+    try:
+      self.checkCmd(["pull",  remoteFile, localFile])
+      f = open(localFile)
+      ret = f.read()
+      f.close()
+      return ret;      
+    except:
+      return None
+
+  # copy directory structure from device (remoteDir) to host (localDir)
+  # external function
+  # checkDir exists so that we don't create local directories if the
+  # remote directory doesn't exist but also so that we don't call isDir
+  # twice when recursing.
+  # returns:
+  #  success: list of files, string
+  #  failure: None
+  def getDirectory(self, remoteDir, localDir, checkDir=True):
+    ret = []
+    p = self.runCmd(["pull", remoteDir, localDir])
+    p.stderr.readline()
+    line = p.stderr.readline()
+    while (line):
+      els = line.split()
+      f = els[len(els) - 1]
+      i = f.find(localDir)
+      if (i != -1):
+        if (localDir[len(localDir) - 1] != '/'):
+          i = i + 1
+        f = f[i + len(localDir):]
+      i = f.find("/")
+      if (i > 0):
+        f = f[0:i]
+      ret.append(f)
+      line =  p.stderr.readline()
+    #the last line is a summary
+    ret.pop(len(ret) - 1)
+    return ret
+
+
+
+  # true/false check if the two files have the same md5 sum
+  # external function
+  # returns:
+  #  success: True
+  #  failure: False
+  def validateFile(self, remoteFile, localFile):
+    return self.getRemoteHash(remoteFile) == self.getLocalHash(localFile)
+
+  # return the md5 sum of a remote file
+  # internal function
+  # returns:
+  #  success: MD5 hash for given filename
+  #  failure: None
+  def getRemoteHash(self, filename):
+    data = p = self.runCmd(["shell", "ls", "-l", filename]).stdout.read()
+    return data.split()[3]
+
+  def getLocalHash(self, filename):
+    data = p = subprocess.Popen(["ls", "-l", filename], stdout=subprocess.PIPE).stdout.read()
+    return data.split()[4]
+
+  # Gets the device root for the testing area on the device
+  # For all devices we will use / type slashes and depend on the device-agent
+  # to sort those out.  The agent will return us the device location where we
+  # should store things, we will then create our /tests structure relative to
+  # that returned path.
+  # Structure on the device is as follows:
+  # /tests
+  #       /<fennec>|<firefox>  --> approot
+  #       /profile
+  #       /xpcshell
+  #       /reftest
+  #       /mochitest
+  #
+  # external function
+  # returns:
+  #  success: path for device root
+  #  failure: None
+  def getDeviceRoot(self):
+    if (not self.dirExists("/data/local/tests")):
+      self.mkDir("/data/local/tests")
+    return "/data/local/tests"
+
+  # Either we will have /tests/fennec or /tests/firefox but we will never have
+  # both.  Return the one that exists
+  # TODO: ensure we can support org.mozilla.firefox
+  # external function
+  # returns:
+  #  success: path for app root
+  #  failure: None
+  def getAppRoot(self):
+    devroot = self.getDeviceRoot()
+    if (devroot == None):
+      return None
+
+    if (self.dirExists(devroot + '/fennec')):
+      return devroot + '/fennec'
+    elif (self.dirExists(devroot + '/firefox')):
+      return devroot + '/firefox'
+    elif (self.dirExsts('/data/data/org.mozilla.fennec')):
+      return 'org.mozilla.fennec'
+    elif (self.dirExists('/data/data/org.mozilla.firefox')):
+      return 'org.mozilla.firefox'
+
+    # Failure (either not installed or not a recognized platform)
+    return None
+
+  # Gets the directory location on the device for a specific test type
+  # Type is one of: xpcshell|reftest|mochitest
+  # external function
+  # returns:
+  #  success: path for test root
+  #  failure: None
+  def getTestRoot(self, type):
+    devroot = self.getDeviceRoot()
+    if (devroot == None):
+      return None
+
+    if (re.search('xpcshell', type, re.I)):
+      self.testRoot = devroot + '/xpcshell'
+    elif (re.search('?(i)reftest', type)):
+      self.testRoot = devroot + '/reftest'
+    elif (re.search('?(i)mochitest', type)):
+      self.testRoot = devroot + '/mochitest'
+    return self.testRoot
+
+
+  # external function
+  # returns:
+  #  success: status from test agent
+  #  failure: None
+  def reboot(self, wait = False):
+    ret = self.runCmd(["reboot"]).stdout.read()
+    if (not wait):
+      return "Success"
+    countdown = 40
+    while (countdown > 0):
+      countdown
+      try:
+        self.checkCmd(["wait-for-device", "shell", "ls", "/sbin"])
+        return ret
+      except:
+        try:
+          self.checkCmd(["root"])
+        except:
+          time.sleep(1)
+          print "couldn't get root"
+    return "Success"
+
+  # external function
+  # returns:
+  #  success: text status from command or callback server
+  #  failure: None
+  def updateApp(self, appBundlePath, processName=None, destPath=None, ipAddr=None, port=30000):
+    return self.runCmd(["install", "-r", appBundlePath]).stdout.read()
+    
+  # Returns information about the device:
+  # Directive indicates the information you want to get, your choices are:
+  # os - name of the os
+  # id - unique id of the device
+  # uptime - uptime of the device
+  # systime - system time of the device
+  # screen - screen resolution
+  # memory - memory stats
+  # process - list of running processes (same as ps)
+  # disk - total, free, available bytes on disk
+  # power - power status (charge, battery temp)
+  # all - all of them - or call it with no parameters to get all the information
+  # returns:
+  #   success: dict of info strings by directive name
+  #   failure: {}
+  def getInfo(self, directive="all"):
+    ret = {}
+    if (directive == "id" or directive == "all"):
+      ret["id"] = self.runCmd(["get-serialno"]).stdout.read()
+    if (directive == "os" or directive == "all"):
+      ret["os"] = self.runCmd(["shell", "getprop", "ro.build.display.id"]).stdout.read()
+    if (directive == "uptime" or directive == "all"):
+      utime = self.runCmd(["shell", "uptime"]).stdout.read()
+      if (not utime):
+        raise DMError("error getting uptime")
+      utime = utime[9:]
+      hours = utime[0:utime.find(":")]
+      utime = utime[utime[1:].find(":") + 2:]
+      minutes = utime[0:utime.find(":")]
+      utime = utime[utime[1:].find(":") +  2:]
+      seconds = utime[0:utime.find(",")]
+      ret["uptime"] = ["0 days " + hours + " hours " + minutes + " minutes " + seconds + " seconds"]
+    if (directive == "process" or directive == "all"):
+      ret["process"] = self.runCmd(["shell", "ps"]).stdout.read()
+    if (directive == "systime" or directive == "all"):
+      ret["systime"] = self.runCmd(["shell", "date"]).stdout.read()
+    print ret
+    return ret
+
+  def runCmd(self, args):
+    args.insert(0, "adb")
+    return subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+
+  def checkCmd(self, args):
+    args.insert(0, "adb")
+    return subprocess.check_call(args)
+
+  def chmodDir(self, remoteDir):
+    print "called chmodDir"
+    if (self.isDir(remoteDir)):
+      files = self.listFiles(remoteDir.strip())
+      for f in files:
+        if (self.isDir(remoteDir.strip() + "/" + f.strip())):
+          self.chmodDir(remoteDir.strip() + "/" + f.strip())
+        else:
+          self.checkCmd(["shell", "chmod", "777", remoteDir.strip()])
+          print "chmod " + remoteDir.strip()
+      self.checkCmd(["shell", "chmod", "777", remoteDir])
+      print "chmod " + remoteDir
+    else:
+      self.checkCmd(["shell", "chmod", "777", remoteDir.strip()])
+      print "chmod " + remoteDir
--- a/layout/tools/reftest/Makefile.in
+++ b/layout/tools/reftest/Makefile.in
@@ -76,16 +76,18 @@ copy-harness: make-xpi
 libs:: copy-harness
 endif
 
 _HARNESS_FILES = \
   $(srcdir)/runreftest.py \
   $(srcdir)/remotereftest.py \
   automation.py \
   $(topsrcdir)/build/mobile/devicemanager.py \
+  $(topsrcdir)/build/mobile/devicemanagerADB.py \
+  $(topsrcdir)/build/mobile/devicemanagerSUT.py \
   $(topsrcdir)/build/automationutils.py \
   $(topsrcdir)/build/poster.zip \
   $(topsrcdir)/build/mobile/remoteautomation.py \
   $(topsrcdir)/testing/mochitest/server.js \
   $(topsrcdir)/build/pgo/server-locations.txt \
   $(NULL)
 
 $(_DEST_DIR):
--- a/layout/tools/reftest/remotereftest.py
+++ b/layout/tools/reftest/remotereftest.py
@@ -42,17 +42,17 @@ import time
 import tempfile
 
 # We need to know our current directory so that we can serve our test files from it.
 SCRIPT_DIRECTORY = os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0])))
 
 from runreftest import RefTest
 from runreftest import ReftestOptions
 from automation import Automation
-from devicemanager import DeviceManager
+import devicemanager, devicemanagerADB, devicemanagerSUT
 from remoteautomation import RemoteAutomation
 
 class RemoteOptions(ReftestOptions):
     def __init__(self, automation):
         ReftestOptions.__init__(self, automation)
 
         defaults = {}
         defaults["logFile"] = "reftest.log"
@@ -106,16 +106,21 @@ class RemoteOptions(ReftestOptions):
                     help = "add webserver and port to the user.js file for remote script access and universalXPConnect")
         defaults["enablePrivilege"] = False
 
         self.add_option("--pidfile", action = "store",
                     type = "string", dest = "pidFile",
                     help = "name of the pidfile to generate")
         defaults["pidFile"] = ""
 
+        self.add_option("--dm_trans", action="store",
+                    type = "string", dest = "dm_trans",
+                    help = "the transport to use to communicate with device: [adb|sut]; default=sut")
+        defaults["dm_trans"] = "sut"
+
         defaults["localLogName"] = None
 
         self.set_defaults(**defaults)
 
     def verifyRemoteOptions(self, options):
         # Ensure our defaults are set properly for everything we can infer
         options.remoteTestRoot = self._automation._devicemanager.getDeviceRoot() + '/reftest'
         options.remoteProfile = options.remoteTestRoot + "/profile"
@@ -370,26 +375,32 @@ user_pref("capability.principal.codebase
         if (self.pidFile != ""):
             try:
                 os.remove(self.pidFile)
                 os.remove(self.pidFile + ".xpcshell.pid")
             except:
                 print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % self.pidFile
 
 def main():
-    dm_none = DeviceManager(None, None)
+    dm_none = devicemanagerADB.DeviceManagerADB(None, None)
     automation = RemoteAutomation(dm_none)
     parser = RemoteOptions(automation)
     options, args = parser.parse_args()
 
     if (options.deviceIP == None):
         print "Error: you must provide a device IP to connect to via the --device option"
         sys.exit(1)
 
-    dm = DeviceManager(options.deviceIP, options.devicePort)
+    if (options.dm_trans == "adb"):
+        if (options.deviceIP):
+            dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort)
+        else:
+            dm = dm_auto
+    else:
+         dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort)
     automation.setDeviceManager(dm)
 
     if (options.remoteProductName != None):
         automation.setProduct(options.remoteProductName)
 
     # Set up the defaults and ensure options are set
     options = parser.verifyRemoteOptions(options)
     if (options == None):
--- a/testing/mochitest/Makefile.in
+++ b/testing/mochitest/Makefile.in
@@ -74,16 +74,18 @@ include $(topsrcdir)/build/automation-bu
 
 # files that get copied into $objdir/_tests/
 _SERV_FILES = 	\
 		runtests.py \
 		automation.py \
 		runtestsremote.py \
 		runtestsvmware.py \
 		$(topsrcdir)/build/mobile/devicemanager.py \
+		$(topsrcdir)/build/mobile/devicemanagerADB.py \
+		$(topsrcdir)/build/mobile/devicemanagerSUT.py \
 		$(topsrcdir)/build/automationutils.py \
 		$(topsrcdir)/build/poster.zip \
 		$(topsrcdir)/build/mobile/remoteautomation.py \
 		gen_template.pl \
 		server.js \
 		harness-a11y.xul \
 		harness-overlay.xul \
 		harness.xul \
--- a/testing/mochitest/runtestsremote.py
+++ b/testing/mochitest/runtestsremote.py
@@ -43,17 +43,17 @@ import tempfile
 sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))))
 
 from automation import Automation
 from remoteautomation import RemoteAutomation
 from runtests import Mochitest
 from runtests import MochitestOptions
 from runtests import MochitestServer
 
-import devicemanager
+import devicemanager, devicemanagerADB, devicemanagerSUT
 
 class RemoteOptions(MochitestOptions):
 
     def __init__(self, automation, scriptdir, **kwargs):
         defaults = {}
         MochitestOptions.__init__(self, automation, scriptdir)
 
         self.add_option("--remote-app-path", action="store",
@@ -61,16 +61,21 @@ class RemoteOptions(MochitestOptions):
                     help = "Path to remote executable relative to device root using only forward slashes. Either this or app must be specified but not both")
         defaults["remoteAppPath"] = None
 
         self.add_option("--deviceIP", action="store",
                     type = "string", dest = "deviceIP",
                     help = "ip address of remote device to test")
         defaults["deviceIP"] = None
 
+        self.add_option("--dm_trans", action="store",
+                    type = "string", dest = "dm_trans",
+                    help = "the transport to use to communicate with device: [adb|sut]; default=sut")
+        defaults["dm_trans"] = "sut"
+
         self.add_option("--devicePort", action="store",
                     type = "string", dest = "devicePort",
                     help = "port of remote device to test")
         defaults["devicePort"] = 20701
 
         self.add_option("--remote-product-name", action="store",
                     type = "string", dest = "remoteProductName",
                     help = "The executable's name of remote product to test - either fennec or firefox, defaults to fennec")
@@ -297,22 +302,27 @@ class MochiRemote(Mochitest):
             raise devicemanager.FileError("Unable to install Chrome files on device.")
         return manifest
 
     def getLogFilePath(self, logFile):             
         return logFile
 
 def main():
     scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
-    dm_none = devicemanager.DeviceManager(None, None)
+    dm_none = devicemanagerADB.DeviceManagerADB()
     auto = RemoteAutomation(dm_none, "fennec")
     parser = RemoteOptions(auto, scriptdir)
     options, args = parser.parse_args()
-
-    dm = devicemanager.DeviceManager(options.deviceIP, options.devicePort)
+    if (options.dm_trans == "adb"):
+        if (options.deviceIP):
+            dm = devicemanagerADB.DeviceManagerADB(options.deviceIP, options.devicePort)
+        else:
+            dm = dm_auto
+    else:
+         dm = devicemanagerSUT.DeviceManagerSUT(options.deviceIP, options.devicePort)
     auto.setDeviceManager(dm)
     options = parser.verifyRemoteOptions(options, auto)
     if (options == None):
         print "ERROR: Invalid options specified, use --help for a list of valid options"
         sys.exit(1)
 
     productPieces = options.remoteProductName.split('.')
     if (productPieces != None):