Bug 749011 - Add support for running mochitest-plain on B2G, r=jmaher
authorJonathan Griffin <jgriffin@mozilla.com>
Fri, 27 Apr 2012 11:44:59 -0700
changeset 92640 acd7146a04845988ea30990749651e253423f188
parent 92639 29f55d65a931d10e6f20093fcfadbf5f3252f546
child 92641 1f7740a01fe63845eb5a44aa0df901a6f7ec6a66
push id8787
push userjgriffin@mozilla.com
push dateFri, 27 Apr 2012 18:48:46 +0000
treeherdermozilla-inbound@acd7146a0484 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs749011
milestone15.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 749011 - Add support for running mochitest-plain on B2G, r=jmaher
build/mobile/b2gautomation.py
testing/mochitest/Makefile.in
testing/mochitest/runtestsb2g.py
new file mode 100644
--- /dev/null
+++ b/build/mobile/b2gautomation.py
@@ -0,0 +1,232 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import automationutils
+import os
+import re
+import socket
+import shutil
+import sys
+import tempfile
+import time
+
+from automation import Automation
+from devicemanager import DeviceManager, NetworkTools
+
+class B2GRemoteAutomation(Automation):
+    _devicemanager = None
+
+    def __init__(self, deviceManager, appName='', remoteLog=None,
+                 marionette=None):
+        self._devicemanager = deviceManager
+        self._appName = appName
+        self._remoteProfile = None
+        self._remoteLog = remoteLog
+        self.marionette = marionette
+
+        # Default our product to b2g
+        self._product = "b2g"
+        Automation.__init__(self)
+
+    def setDeviceManager(self, deviceManager):
+        self._devicemanager = deviceManager
+
+    def setAppName(self, appName):
+        self._appName = appName
+
+    def setRemoteProfile(self, remoteProfile):
+        self._remoteProfile = remoteProfile
+
+    def setProduct(self, product):
+        self._product = product
+
+    def setRemoteLog(self, logfile):
+        self._remoteLog = logfile
+
+    # Set up what we need for the remote environment
+    def environment(self, env=None, xrePath=None, crashreporter=True):
+        # Because we are running remote, we don't want to mimic the local env
+        # so no copying of os.environ
+        if env is None:
+            env = {}
+
+        return env
+
+    def checkForCrashes(self, directory, symbolsPath):
+        # XXX: This will have to be updated after crash reporting on b2g
+        # is in place.
+        dumpDir = tempfile.mkdtemp()
+        self._devicemanager.getDirectory(self._remoteProfile + '/minidumps/', dumpDir)
+        automationutils.checkForCrashes(dumpDir, symbolsPath, self.lastTestSeen)
+        try:
+          shutil.rmtree(dumpDir)
+        except:
+          print "WARNING: unable to remove directory: %s" % (dumpDir)
+
+    def buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs):
+        # if remote profile is specified, use that instead
+        if (self._remoteProfile):
+            profileDir = self._remoteProfile
+
+        cmd, args = Automation.buildCommandLine(self, app, debuggerInfo, profileDir, testURL, extraArgs)
+
+        return app, args
+
+    def getLanIp(self):
+        nettools = NetworkTools()
+        return nettools.getLanIp()
+
+    def waitForFinish(self, proc, utilityPath, timeout, maxTime, startTime,
+                      debuggerInfo, symbolsPath, logger):
+        """ Wait for mochitest to finish (as evidenced by a signature string
+            in logcat), or for a given amount of time to elapse with no
+            output.
+        """
+        timeout = timeout or 120
+
+        didTimeout = False
+
+        done = time.time() + timeout
+        while True:
+            currentlog = proc.stdout
+            if currentlog:
+                done = time.time() + timeout
+                print currentlog
+                if 'INFO SimpleTest FINISHED' in currentlog:
+                    return 0
+            else:
+                if time.time() > done:
+                    self.log.info("TEST-UNEXPECTED-FAIL | %s | application timed "
+                                  "out after %d seconds with no output",
+                                  self.lastTestSeen, int(timeout))
+                    return 1
+
+    def getDeviceStatus(self, serial=None):
+        # Get the current status of the device.  If we know the device
+        # serial number, we look for that, otherwise we use the (presumably
+        # only) device shown in 'adb devices'.
+        serial = serial or self._devicemanager.deviceSerial
+        status = 'unknown'
+
+        for line in self._devicemanager.runCmd(['devices']).stdout.readlines():
+            result =  re.match('(.*?)\t(.*)', line)
+            if result:
+                thisSerial = result.group(1)
+                if not serial or thisSerial == serial:
+                    serial = thisSerial
+                    status = result.group(2)
+
+        return (serial, status)
+
+    def rebootDevice(self):
+        # find device's current status and serial number
+        serial, status = self.getDeviceStatus()
+
+        # reboot!
+        self._devicemanager.checkCmd(['reboot'])
+
+        # wait for device to come back to previous status
+        print 'waiting for device to come back online after reboot'
+        start = time.time()
+        rserial, rstatus = self.getDeviceStatus(serial)
+        while rstatus != 'device':
+            if time.time() - start > 120:
+                # device hasn't come back online in 2 minutes, something's wrong
+                raise Exception("Device %s (status: %s) not back online after reboot" % (serial, rstatus))
+            time.sleep(5)
+            rserial, rstatus = self.getDeviceStatus(serial)
+        print 'device:', serial, 'status:', rstatus
+
+    def Process(self, cmd, stdout=None, stderr=None, env=None, cwd=None):
+        # On a desktop or fennec run, the Process method invokes a gecko
+        # process in which to run mochitests.  For B2G, we simply
+        # reboot the device (which was configured with a test profile
+        # already), wait for B2G to start up, and then navigate to the
+        # test url using Marionette.  There doesn't seem to be any way
+        # to pass env variables into the B2G process, but this doesn't 
+        # seem to matter.
+
+        instance = self.B2GInstance(self._devicemanager)
+
+        # reboot device so it starts up with the mochitest profile
+        # XXX:  We could potentially use 'stop b2g' + 'start b2g' to achieve
+        # a similar effect; will see which is more stable while attempting
+        # to bring up the continuous integration.
+        self.rebootDevice()
+
+        # Infrequently, gecko comes up before networking does, so wait a little
+        # bit to give the network time to become available.
+        # XXX:  need a more robust mechanism for this
+        time.sleep(20)
+
+        # Set up port forwarding again for Marionette, since any that
+        # existed previously got wiped out by the reboot.
+        self._devicemanager.checkCmd(['forward',
+                                      'tcp:%s' % self.marionette.port,
+                                      'tcp:%s' % self.marionette.port])
+
+        # start a marionette session
+        session = self.marionette.start_session()
+        if 'b2g' not in session:
+            raise Exception("bad session value %s returned by start_session" % session)
+
+        # start the tests by navigating to the mochitest url
+        self.marionette.execute_script("window.location.href='%s';" % self.testURL)
+
+        return instance
+
+    # be careful here as this inner class doesn't have access to outer class members
+    class B2GInstance(object):
+        """Represents a B2G instance running on a device, and exposes
+           some process-like methods/properties that are expected by the
+           automation.
+        """
+
+        def __init__(self, dm):
+            self.dm = dm
+            self.lastloglines = []
+
+        @property
+        def pid(self):
+            # a dummy value to make the automation happy
+            return 0
+
+        @property
+        def stdout(self):
+            # Return any part of logcat output that wasn't returned
+            # previously.  This is done by fetching about the last 50
+            # lines of the log (logcat -t 50 generally fetches 50-58 lines),
+            # and comparing against the previous set to find new lines.
+            t = self.dm.runCmd(['logcat', '-t', '50']).stdout.read()
+            if t == None: return ''
+
+            t = t.strip('\n').strip()
+            loglines = t.split('\n')
+            line_index = 0
+
+            # If there are more than 50 lines, we skip the first 20; this
+            # is because the number of lines returned
+            # by logcat -t 50 varies (usually between 50 and 58).
+            log_index = 20 if len(loglines) > 50 else 0
+
+            for index, line in enumerate(loglines[log_index:]):
+                line_index = index + log_index + 1
+                try:
+                    self.lastloglines.index(line)
+                except ValueError:
+                    break
+
+            newoutput = '\n'.join(loglines[line_index:])
+            self.lastloglines = loglines
+
+            return newoutput
+
+        def wait(self, timeout = None):
+            # this should never happen
+            raise Exception("'wait' called on B2GInstance")
+
+        def kill(self):
+            # this should never happen
+            raise Exception("'kill' called on B2GInstance")
+
--- a/testing/mochitest/Makefile.in
+++ b/testing/mochitest/Makefile.in
@@ -74,24 +74,26 @@ include $(topsrcdir)/config/rules.mk
 # necessary for relative objdir paths.
 TARGET_DEPTH = ../../..
 include $(topsrcdir)/build/automation-build.mk
 
 # files that get copied into $objdir/_tests/
 _SERV_FILES = 	\
 		runtests.py \
 		automation.py \
+		runtestsb2g.py \
 		runtestsremote.py \
 		runtestsvmware.py \
 		$(topsrcdir)/build/mobile/devicemanager.py \
 		$(topsrcdir)/build/mobile/devicemanagerADB.py \
 		$(topsrcdir)/build/mobile/devicemanagerSUT.py \
 		$(topsrcdir)/build/automationutils.py \
 		$(topsrcdir)/build/manifestparser.py \
 		$(topsrcdir)/build/mobile/remoteautomation.py \
+		$(topsrcdir)/build/mobile/b2gautomation.py \
 		gen_template.pl \
 		server.js \
 		harness-overlay.xul \
 		harness.xul \
 		browser-test-overlay.xul \
 		browser-test.js \
 		chrome-harness.js \
 		browser-harness.xul \
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/runtestsb2g.py
@@ -0,0 +1,412 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import ConfigParser
+import os
+import re
+import sys
+import tempfile
+import time
+import urllib
+
+sys.path.insert(0, os.path.abspath(os.path.realpath(os.path.dirname(sys.argv[0]))))
+
+from automation import Automation
+from b2gautomation import B2GRemoteAutomation
+from runtests import Mochitest
+from runtests import MochitestOptions
+from runtests import MochitestServer
+
+import devicemanagerADB
+import manifestparser
+
+from marionette import Marionette
+
+
+class B2GOptions(MochitestOptions):
+
+    def __init__(self, automation, scriptdir, **kwargs):
+        defaults = {}
+        MochitestOptions.__init__(self, automation, scriptdir)
+
+        self.add_option("--b2gpath", action="store",
+                    type = "string", dest = "b2gPath",
+                    help = "path to B2G repo or qemu dir")
+        defaults["b2gPath"] = None
+
+        self.add_option("--marionette", action="store",
+                    type = "string", dest = "marionette",
+                    help = "host:port to use when connecting to Marionette")
+        defaults["marionette"] = None
+
+        self.add_option("--emulator", action="store_true",
+                    dest = "emulator",
+                    help = "True if using a b2g emulator")
+        defaults["emulator"] = False
+
+        self.add_option("--adbpath", action="store",
+                    type = "string", dest = "adbPath",
+                    help = "path to adb")
+        defaults["adbPath"] = "adb"
+
+        self.add_option("--deviceIP", action="store",
+                    type = "string", dest = "deviceIP",
+                    help = "ip address of remote device to test")
+        defaults["deviceIP"] = None
+
+        self.add_option("--devicePort", action="store",
+                    type = "string", dest = "devicePort",
+                    help = "port of remote device to test")
+        defaults["devicePort"] = 20701
+
+        self.add_option("--remote-logfile", action="store",
+                    type = "string", dest = "remoteLogFile",
+                    help = "Name of log file on the device relative to the device root.  PLEASE ONLY USE A FILENAME.")
+        defaults["remoteLogFile"] = None
+
+        self.add_option("--remote-webserver", action = "store",
+                    type = "string", dest = "remoteWebServer",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["remoteWebServer"] = None
+
+        self.add_option("--http-port", action = "store",
+                    type = "string", dest = "httpPort",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["httpPort"] = automation.DEFAULT_HTTP_PORT
+
+        self.add_option("--ssl-port", action = "store",
+                    type = "string", dest = "sslPort",
+                    help = "ip address where the remote web server is hosted at")
+        defaults["sslPort"] = automation.DEFAULT_SSL_PORT
+
+        self.add_option("--pidfile", action = "store",
+                    type = "string", dest = "pidFile",
+                    help = "name of the pidfile to generate")
+        defaults["pidFile"] = ""
+
+        defaults["remoteTestRoot"] = None
+        defaults["logFile"] = "mochitest.log"
+        defaults["autorun"] = True
+        defaults["closeWhenDone"] = True
+        defaults["testPath"] = ""
+
+        self.set_defaults(**defaults)
+
+    def verifyRemoteOptions(self, options, automation):
+        options.remoteTestRoot = automation._devicemanager.getDeviceRoot()
+        productRoot = options.remoteTestRoot + "/" + automation._product
+
+        if options.utilityPath == self._automation.DIST_BIN:
+            options.utilityPath = productRoot + "/bin"
+
+        if options.remoteWebServer == None:
+            if os.name != "nt":
+                options.remoteWebServer = automation.getLanIp()
+            else:
+                print "ERROR: you must specify a --remote-webserver=<ip address>\n"
+                return None
+
+        options.webServer = options.remoteWebServer
+
+        #if not options.emulator and not options.deviceIP:
+        #    print "ERROR: you must provide a device IP"
+        #    return None
+
+        if options.remoteLogFile == None:
+            options.remoteLogFile = options.remoteTestRoot + '/logs/mochitest.log'
+
+        if options.remoteLogFile.count('/') < 1:
+            options.remoteLogFile = options.remoteTestRoot + '/' + options.remoteLogFile
+
+        # Only reset the xrePath if it wasn't provided
+        if options.xrePath == None:
+            options.xrePath = options.utilityPath
+
+        if options.pidFile != "":
+            f = open(options.pidFile, 'w')
+            f.write("%s" % os.getpid())
+            f.close()
+
+        return options
+
+    def verifyOptions(self, options, mochitest):
+        # since we are reusing verifyOptions, it will exit if App is not found
+        temp = options.app
+        options.app = sys.argv[0]
+        tempPort = options.httpPort
+        tempSSL = options.sslPort
+        tempIP = options.webServer
+        options = MochitestOptions.verifyOptions(self, options, mochitest)
+        options.webServer = tempIP
+        options.app = temp
+        options.sslPort = tempSSL
+        options.httpPort = tempPort
+
+        return options 
+
+
+class ProfileConfigParser(ConfigParser.RawConfigParser):
+    """Subclass of RawConfigParser that outputs .ini files in the exact
+       format expected for profiles.ini, which is slightly different
+       than the default format.
+    """
+
+    def optionxform(self, optionstr):
+        return optionstr
+
+    def write(self, fp):
+        if self._defaults:
+            fp.write("[%s]\n" % ConfigParser.DEFAULTSECT)
+            for (key, value) in self._defaults.items():
+                fp.write("%s=%s\n" % (key, str(value).replace('\n', '\n\t')))
+            fp.write("\n")
+        for section in self._sections:
+            fp.write("[%s]\n" % section)
+            for (key, value) in self._sections[section].items():
+                if key == "__name__":
+                    continue
+                if (value is not None) or (self._optcre == self.OPTCRE):
+                    key = "=".join((key, str(value).replace('\n', '\n\t')))
+                fp.write("%s\n" % (key))
+            fp.write("\n")
+
+
+class B2GMochitest(Mochitest):
+
+    _automation = None
+    _dm = None
+    localProfile = None
+
+    def __init__(self, automation, devmgr, options):
+        self._automation = automation
+        Mochitest.__init__(self, self._automation)
+        self._dm = devmgr
+        self.runSSLTunnel = False
+        self.remoteProfile = options.remoteTestRoot + '/profile'
+        self._automation.setRemoteProfile(self.remoteProfile)
+        self.remoteLog = options.remoteLogFile
+        self.remoteProfilesIniPath = '/data/b2g/mozilla/profiles.ini'
+        self.originalProfilesIni = None
+
+    def cleanup(self, manifest, options):
+        self._dm.getFile(self.remoteLog, self.localLog)
+        self._dm.removeFile(self.remoteLog)
+        self._dm.removeDir(self.remoteProfile)
+
+        if self.originalProfilesIni:
+            try:
+                self.restoreProfilesIni()
+                os.remove(self.originalProfilesIni)
+            except:
+                pass
+
+        if options.pidFile != "":
+            try:
+                os.remove(options.pidFile)
+                os.remove(options.pidFile + ".xpcshell.pid")
+            except:
+                print "Warning: cleaning up pidfile '%s' was unsuccessful from the test harness" % options.pidFile
+
+        # We've restored the original profile, so reboot the device so that
+        # it gets picked up.
+        self._automation.rebootDevice()
+
+    def findPath(self, paths, filename = None):
+        for path in paths:
+            p = path
+            if filename:
+                p = os.path.join(p, filename)
+            if os.path.exists(self.getFullPath(p)):
+                return path
+        return None
+
+    def startWebServer(self, options):
+        """ Create the webserver on the host and start it up """
+        remoteXrePath = options.xrePath
+        remoteProfilePath = options.profilePath
+        remoteUtilityPath = options.utilityPath
+        localAutomation = Automation()
+        localAutomation.IS_WIN32 = False
+        localAutomation.IS_LINUX = False
+        localAutomation.IS_MAC = False
+        localAutomation.UNIXISH = False
+        hostos = sys.platform
+        if hostos in ['mac', 'darwin']:
+            localAutomation.IS_MAC = True
+        elif hostos in ['linux', 'linux2']:
+            localAutomation.IS_LINUX = True
+            localAutomation.UNIXISH = True
+        elif hostos in ['win32', 'win64']:
+            localAutomation.BIN_SUFFIX = ".exe"
+            localAutomation.IS_WIN32 = True
+
+        paths = [options.xrePath,
+                 localAutomation.DIST_BIN,
+                 self._automation._product,
+                 os.path.join('..', self._automation._product)]
+        options.xrePath = self.findPath(paths)
+        if options.xrePath == None:
+            print "ERROR: unable to find xulrunner path for %s, please specify with --xre-path" % (os.name)
+            sys.exit(1)
+        paths.append("bin")
+        paths.append(os.path.join("..", "bin"))
+
+        xpcshell = "xpcshell"
+        if (os.name == "nt"):
+            xpcshell += ".exe"
+
+        if (options.utilityPath):
+            paths.insert(0, options.utilityPath)
+        options.utilityPath = self.findPath(paths, xpcshell)
+        if options.utilityPath == None:
+            print "ERROR: unable to find utility path for %s, please specify with --utility-path" % (os.name)
+            sys.exit(1)
+
+        options.profilePath = tempfile.mkdtemp()
+        self.server = MochitestServer(localAutomation, options)
+        self.server.start()
+
+        if (options.pidFile != ""):
+            f = open(options.pidFile + ".xpcshell.pid", 'w')
+            f.write("%s" % self.server._process.pid)
+            f.close()
+        self.server.ensureReady(self.SERVER_STARTUP_TIMEOUT)
+
+        options.xrePath = remoteXrePath
+        options.utilityPath = remoteUtilityPath
+        options.profilePath = remoteProfilePath
+
+    def stopWebServer(self, options):
+        self.server.stop()
+
+    def buildProfile(self, options):
+        if self.localProfile:
+            options.profilePath = self.localProfile
+        print 'buildProfile', repr(options)
+        manifest = Mochitest.buildProfile(self, options)
+        self.localProfile = options.profilePath
+
+        # Profile isn't actually copied to device until
+        # buildURLOptions is called.
+
+        options.profilePath = self.remoteProfile
+        return manifest
+
+    def restoreProfilesIni(self):
+        # restore profiles.ini on the device to its previous state
+        if not self.originalProfilesIni or not os.access(self.originalProfilesIni, os.F_OK):
+            raise DMError('Unable to install original profiles.ini; file not found: %s',
+                          self.originalProfilesIni)
+
+        self._dm.pushFile(self.originalProfilesIni, self.remoteProfilesIniPath)
+
+    def updateProfilesIni(self, profilePath):
+        # update profiles.ini on the device to point to the test profile
+        self.originalProfilesIni = tempfile.mktemp()
+        self._dm.getFile(self.remoteProfilesIniPath, self.originalProfilesIni)
+
+        config = ProfileConfigParser()
+        config.read(self.originalProfilesIni)
+        for section in config.sections():
+            if 'Profile' in section:
+                config.set(section, 'IsRelative', 0)
+                config.set(section, 'Path', profilePath)
+
+        newProfilesIni = tempfile.mktemp()
+        with open(newProfilesIni, 'wb') as configfile:
+            config.write(configfile)
+
+        self._dm.pushFile(newProfilesIni, self.remoteProfilesIniPath)
+
+    def buildURLOptions(self, options, env):
+        self.localLog = options.logFile
+        options.logFile = self.remoteLog
+        options.profilePath = self.localProfile
+        retVal = Mochitest.buildURLOptions(self, options, env)
+
+        # set the testURL
+        testURL = self.buildTestPath(options)
+        if len(self.urlOpts) > 0:
+            testURL += "?" + "&".join(self.urlOpts)
+        self._automation.testURL = testURL
+
+        # Set the B2G homepage as a static local page, since wi-fi generally
+        # isn't available as soon as the device boots.
+        f = open(os.path.join(options.profilePath, "user.js"), "a")
+        f.write('user_pref("browser.homescreenURL", "data:text/html,mochitest-plain should start soon");\n')
+        f.close()
+
+        self._dm.removeDir(self.remoteProfile)
+        if self._dm.pushDir(options.profilePath, self.remoteProfile) == None:
+            raise devicemanager.FileError("Unable to copy profile to device.")
+
+        self.updateProfilesIni(self.remoteProfile)
+
+        options.profilePath = self.remoteProfile
+        options.logFile = self.localLog
+        return retVal
+
+
+def main():
+    scriptdir = os.path.abspath(os.path.realpath(os.path.dirname(__file__)))
+    auto = B2GRemoteAutomation(None, "fennec")
+    parser = B2GOptions(auto, scriptdir)
+    options, args = parser.parse_args()
+
+    # create our Marionette instance
+    kwargs = {'emulator': options.emulator}
+    if options.b2gPath:
+        kwargs['homedir'] = options.b2gPath
+    if options.marionette:
+        host,port = options.marionette.split(':')
+        kwargs['host'] = host
+        kwargs['port'] = int(port)
+    marionette = Marionette(**kwargs)
+
+    auto.marionette = marionette
+
+    # create the DeviceManager
+    kwargs = {'adbPath': options.adbPath}
+    if options.deviceIP:
+        kwargs.update({'host': options.deviceIP,
+                       'port': options.devicePort})
+    dm = devicemanagerADB.DeviceManagerADB(**kwargs)
+
+    auto.setDeviceManager(dm)
+    options = parser.verifyRemoteOptions(options, auto)
+    if (options == None):
+        print "ERROR: Invalid options specified, use --help for a list of valid options"
+        sys.exit(1)
+
+    auto.setProduct("b2g")
+
+    mochitest = B2GMochitest(auto, dm, options)
+
+    options = parser.verifyOptions(options, mochitest)
+    if (options == None):
+        sys.exit(1)
+
+    logParent = os.path.dirname(options.remoteLogFile)
+    dm.mkDir(logParent);
+    auto.setRemoteLog(options.remoteLogFile)
+    auto.setServerInfo(options.webServer, options.httpPort, options.sslPort)
+
+    retVal = 1
+    try:
+        retVal = mochitest.runTests(options)
+    except:
+        print "TEST-UNEXPECTED-FAIL | %s | Exception caught while running tests." % sys.exc_info()[1]
+        mochitest.stopWebServer(options)
+        mochitest.stopWebSocketServer(options)
+        try:
+            mochitest.cleanup(None, options)
+        except:
+            pass
+            sys.exit(1)
+
+    sys.exit(retVal)
+
+if __name__ == "__main__":
+    main()
+