Bug 811411 - Add ability to run C++ unit tests on mobile; r=ted.mielczarek
authorGeoff Brown <gbrown@mozilla.com>
Thu, 03 Jan 2013 13:01:54 -0700
changeset 126588 d834b07541e05cc33e80943d6fdb3747f76e98e6
parent 126587 958f452fc41ed816fb56b29e56245063469e2092
child 126589 79e3ae50976460a55e57bd6ba56bef594bc0dc24
push id2151
push userlsblakk@mozilla.com
push dateTue, 19 Feb 2013 18:06:57 +0000
treeherdermozilla-beta@4952e88741ec [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersted
bugs811411
milestone20.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 811411 - Add ability to run C++ unit tests on mobile; r=ted.mielczarek
testing/remotecppunittests.py
testing/runcppunittests.py
testing/testsuite-targets.mk
new file mode 100644
--- /dev/null
+++ b/testing/remotecppunittests.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os, sys
+import runcppunittests as cppunittests
+import mozcrash, mozlog
+import StringIO
+import posixpath
+from mozdevice import devicemanager, devicemanagerADB, devicemanagerSUT
+
+log = mozlog.getLogger('remotecppunittests')
+
+class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
+    def __init__(self, devmgr, options, progs):
+        cppunittests.CPPUnitTests.__init__(self)
+        self.options = options
+        self.device = devmgr
+        self.remote_test_root = self.device.getDeviceRoot() + "/cppunittests"
+        self.remote_bin_dir = posixpath.join(self.remote_test_root, "b")
+        self.remote_tmp_dir = posixpath.join(self.remote_test_root, "tmp")
+        self.remote_profile_dir = posixpath.join(self.remote_test_root, "p")
+        if options.setup:
+            self.setup_bin(progs)
+
+    def setup_bin(self, progs):
+        if not self.device.dirExists(self.remote_test_root):
+            self.device.mkDir(self.remote_test_root)
+        if self.device.dirExists(self.remote_tmp_dir):
+            self.device.removeDir(self.remote_tmp_dir)
+        self.device.mkDir(self.remote_tmp_dir)
+        if self.device.dirExists(self.remote_bin_dir):
+            self.device.removeDir(self.remote_bin_dir)
+        self.device.mkDir(self.remote_bin_dir)
+        self.push_libs()
+        self.push_progs(progs)
+        self.device.chmodDir(self.remote_bin_dir)
+
+    def push_libs(self):
+        for file in os.listdir(self.options.local_lib):
+            if file.endswith(".so"):
+                print >> sys.stderr, "Pushing %s.." % file
+                remote_file = posixpath.join(self.remote_bin_dir, file)
+                self.device.pushFile(os.path.join(self.options.local_lib, file), remote_file)
+        # Additional libraries may be found in a sub-directory such as "lib/armeabi-v7a"
+        local_arm_lib = os.path.join(self.options.local_lib, "lib")
+        if os.path.isdir(local_arm_lib):
+            for root, dirs, files in os.walk(local_arm_lib):
+                for file in files:
+                    if (file.endswith(".so")):
+                        remote_file = posixpath.join(self.remote_bin_dir, file)
+                        self.device.pushFile(os.path.join(root, file), remote_file)
+
+    def push_progs(self, progs):
+        for local_file in progs:
+            remote_file = posixpath.join(self.remote_bin_dir, os.path.basename(local_file))
+            self.device.pushFile(local_file, remote_file)
+
+    def build_environment(self):
+        env = self.build_core_environment()
+        env['LD_LIBRARY_PATH'] = self.remote_bin_dir
+        return env
+
+    def run_one_test(self, prog, env, symbols_path=None):
+        """
+        Run a single C++ unit test program remotely.
+
+        Arguments:
+        * prog: The path to the test program to run.
+        * env: The environment to use for running the program.
+        * symbols_path: A path to a directory containing Breakpad-formatted
+                        symbol files for producing stack traces on crash.
+
+        Return True if the program exits with a zero status, False otherwise.
+        """
+        basename = os.path.basename(prog)
+        remote_bin = posixpath.join(self.remote_bin_dir, basename)
+        log.info("Running test %s", basename)
+        buf = StringIO.StringIO()
+        returncode = self.device.shell([remote_bin], buf, env=env, cwd=self.remote_tmp_dir,
+                                       timeout=cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT)
+        print >> sys.stdout, buf.getvalue()
+        with cppunittests.TemporaryDirectory() as tempdir:
+            self.device.getDirectory(self.remote_tmp_dir, tempdir)
+            if mozcrash.check_for_crashes(tempdir, symbols_path,
+                                          test_name=basename):
+                log.testFail("%s | test crashed", basename)
+                return False
+        result = returncode == 0
+        if not result:
+            log.testFail("%s | test failed with return code %s",
+                         basename, returncode)
+        return result
+
+class RemoteCPPUnittestOptions(cppunittests.CPPUnittestOptions):
+    def __init__(self):
+        cppunittests.CPPUnittestOptions.__init__(self)
+        defaults = {}
+
+        self.add_option("--deviceIP", action="store",
+                        type = "string", dest = "device_ip",
+                        help = "ip address of remote device to test")
+        defaults["device_ip"] = None
+
+        self.add_option("--devicePort", action="store",
+                        type = "string", dest = "device_port",
+                        help = "port of remote device to test")
+        defaults["device_port"] = 20701
+
+        self.add_option("--dm_trans", action="store",
+                        type = "string", dest = "dm_trans",
+                        help = "the transport to use to communicate with device: [adb|sut]; default=sut")
+        defaults["dm_trans"] = "sut"
+
+        self.add_option("--noSetup", action="store_false",
+                        dest = "setup",
+                        help = "do not copy any files to device (to be used only if device is already setup)")
+        defaults["setup"] = True
+
+        self.add_option("--localLib", action="store",
+                        type = "string", dest = "local_lib",
+                        help = "location of libraries to push -- preferably stripped")
+        defaults["local_lib"] = None
+
+        self.set_defaults(**defaults)
+
+def main():
+    parser = RemoteCPPUnittestOptions()
+    options, args = parser.parse_args()
+    if not args:
+        print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
+        sys.exit(1)
+    if not options.local_lib:
+        print >>sys.stderr, """Error: --localLib is required"""
+        sys.exit(1)
+    if not os.path.isdir(options.local_lib):
+        print >>sys.stderr, """Error: --localLib directory %s not found""" % options.local_lib
+        sys.exit(1)
+    if not options.xre_path:
+        print >>sys.stderr, """Error: --xre-path is required"""
+        sys.exit(1)
+    if options.dm_trans == "adb":
+        if options.device_ip:
+            dm = devicemanagerADB.DeviceManagerADB(options.device_ip, options.device_port, packageName=None)
+        else:
+            dm = devicemanagerADB.DeviceManagerADB(packageName=None)
+    else:
+        dm = devicemanagerSUT.DeviceManagerSUT(options.device_ip, options.device_port)
+        if not options.device_ip:
+            print "Error: you must provide a device IP to connect to via the --deviceIP option"
+            sys.exit(1)
+    options.xre_path = os.path.abspath(options.xre_path)
+    progs = [os.path.abspath(p) for p in args]
+    tester = RemoteCPPUnitTests(dm, options, progs)
+    try:
+        result = tester.run_tests(progs, options.xre_path, options.symbols_path)
+    except Exception, e:
+        log.error(str(e))
+        result = False
+    sys.exit(0 if result else 1)
+
+if __name__ == '__main__':
+    main()
--- a/testing/runcppunittests.py
+++ b/testing/runcppunittests.py
@@ -1,120 +1,151 @@
 #!/usr/bin/env python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import with_statement
-import sys, optparse, os, tempfile, shutil
+import sys, os, tempfile, shutil
+from optparse import OptionParser
 import mozprocess, mozinfo, mozlog, mozcrash
 from contextlib import contextmanager
 
 log = mozlog.getLogger('cppunittests')
 
 @contextmanager
 def TemporaryDirectory():
     tempdir = tempfile.mkdtemp()
     yield tempdir
     shutil.rmtree(tempdir)
 
-def run_one_test(prog, env, symbols_path=None):
-    """
-    Run a single C++ unit test program.
+class CPPUnitTests(object):
+    # Time (seconds) to wait for test process to complete
+    TEST_PROC_TIMEOUT = 300
+
+    def run_one_test(self, prog, env, symbols_path=None):
+        """
+        Run a single C++ unit test program.
+
+        Arguments:
+        * prog: The path to the test program to run.
+        * env: The environment to use for running the program.
+        * symbols_path: A path to a directory containing Breakpad-formatted
+                        symbol files for producing stack traces on crash.
 
-    Arguments:
-    * prog: The path to the test program to run.
-    * env: The environment to use for running the program.
-    * symbols_path: A path to a directory containing Breakpad-formatted
-                    symbol files for producing stack traces on crash.
+        Return True if the program exits with a zero status, False otherwise.
+        """
+        basename = os.path.basename(prog)
+        log.info("Running test %s", basename)
+        with TemporaryDirectory() as tempdir:
+            proc = mozprocess.ProcessHandler([prog],
+                                             cwd=tempdir,
+                                             env=env)
+            #TODO: After bug 811320 is fixed, don't let .run() kill the process,
+            # instead use a timeout in .wait() and then kill to get a stack.
+            proc.run(timeout=CPPUnitTests.TEST_PROC_TIMEOUT)
+            proc.wait()
+            if proc.timedOut:
+                log.testFail("%s | timed out after %d seconds",
+                             basename, CPPUnitTests.TEST_PROC_TIMEOUT)
+                return False
+            if mozcrash.check_for_crashes(tempdir, symbols_path,
+                                          test_name=basename):
+                log.testFail("%s | test crashed", basename)
+                return False
+            result = proc.proc.returncode == 0
+            if not result:
+                log.testFail("%s | test failed with return code %d",
+                             basename, proc.proc.returncode)
+            return result
 
-    Return True if the program exits with a zero status, False otherwise.
-    """
-    basename = os.path.basename(prog)
-    log.info("Running test %s", basename)
-    with TemporaryDirectory() as tempdir:
-        proc = mozprocess.ProcessHandler([prog],
-                                         cwd=tempdir,
-                                         env=env)
-        timeout = 300
-        #TODO: After bug 811320 is fixed, don't let .run() kill the process,
-        # instead use a timeout in .wait() and then kill to get a stack.
-        proc.run(timeout=timeout)
-        proc.wait()
-        if proc.timedOut:
-            log.testFail("%s | timed out after %d seconds",
-                         basename, timeout)
-            return False
-        if mozcrash.check_for_crashes(tempdir, symbols_path,
-                                      test_name=basename):
-            log.testFail("%s | test crashed", basename)
-            return False
-        result = proc.proc.returncode == 0
-        if not result:
-            log.testFail("%s | test failed with return code %d",
-                         basename, proc.proc.returncode)
+    def build_core_environment(self, env = {}):
+        """
+        Add environment variables likely to be used across all platforms, including remote systems.
+        """
+        env["MOZILLA_FIVE_HOME"] = self.xre_path
+        env["MOZ_XRE_DIR"] = self.xre_path
+        #TODO: switch this to just abort once all C++ unit tests have
+        # been fixed to enable crash reporting
+        env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
+        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+        env["MOZ_CRASHREPORTER"] = "1"
+        return env
+
+    def build_environment(self):
+        """
+        Create and return a dictionary of all the appropriate env variables and values.
+        On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
+        """
+        if not os.path.isdir(self.xre_path):
+            raise Exception("xre_path does not exist: %s", self.xre_path)
+        env = dict(os.environ)
+        env = self.build_core_environment(env)
+        pathvar = ""
+        if mozinfo.os == "linux":
+            pathvar = "LD_LIBRARY_PATH"
+        elif mozinfo.os == "mac":
+            pathvar = "DYLD_LIBRARY_PATH"
+        elif mozinfo.os == "win":
+            pathvar = "PATH"
+        if pathvar:
+            if pathvar in env:
+                env[pathvar] = "%s%s%s" % (self.xre_path, os.pathsep, env[pathvar])
+            else:
+                env[pathvar] = self.xre_path
+        return env
+
+    def run_tests(self, programs, xre_path, symbols_path=None):
+        """
+        Run a set of C++ unit test programs.
+
+        Arguments:
+        * programs: An iterable containing paths to test programs.
+        * xre_path: A path to a directory containing a XUL Runtime Environment.
+        * symbols_path: A path to a directory containing Breakpad-formatted
+                        symbol files for producing stack traces on crash.
+
+        Returns True if all test programs exited with a zero status, False
+        otherwise.
+        """
+        self.xre_path = xre_path
+        env = self.build_environment()
+        result = True
+        for prog in programs:
+            single_result = self.run_one_test(prog, env, symbols_path)
+            result = result and single_result
         return result
 
-def run_tests(programs, xre_path, symbols_path=None):
-    """
-    Run a set of C++ unit test programs.
-
-    Arguments:
-    * programs: An iterable containing paths to test programs.
-    * xre_path: A path to a directory containing a XUL Runtime Environment.
-    * symbols_path: A path to a directory containing Breakpad-formatted
-                    symbol files for producing stack traces on crash.
+class CPPUnittestOptions(OptionParser):
+    def __init__(self):
+        OptionParser.__init__(self)
+        self.add_option("--xre-path",
+                        action = "store", type = "string", dest = "xre_path",
+                        default = None,
+                        help = "absolute path to directory containing XRE (probably xulrunner)")
+        self.add_option("--symbols-path",
+                        action = "store", type = "string", dest = "symbols_path",
+                        default = None,
+                        help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
 
-    Returns True if all test programs exited with a zero status, False
-    otherwise.
-    """
-    if not os.path.isdir(xre_path):
-        log.error("xre_path does not exist: %s", xre_path)
-        return False
-    #TODO: stick this in a module?
-    env = dict(os.environ)
-    pathvar = ""
-    if mozinfo.os == "linux":
-        pathvar = "LD_LIBRARY_PATH"
-    elif mozinfo.os == "mac":
-        pathvar = "DYLD_LIBRARY_PATH"
-    elif mozinfo.os == "win":
-        pathvar = "PATH"
-    if pathvar:
-        if pathvar in env:
-            env[pathvar] = "%s%s%s" % (xre_path, os.pathsep, env[pathvar])
-        else:
-            env[pathvar] = xre_path
-    env["MOZILLA_FIVE_HOME"] = xre_path
-    env["MOZ_XRE_DIR"] = xre_path
-    #TODO: switch this to just abort once all C++ unit tests have
-    # been fixed to enable crash reporting
-    env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
-    env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
-    env["MOZ_CRASHREPORTER"] = "1"
-    result = True
-    for prog in programs:
-        single_result = run_one_test(prog, env, symbols_path)
-        result = result and single_result
-    return result
-
-if __name__ == '__main__':
-    parser = optparse.OptionParser()
-    parser.add_option("--xre-path",
-                      action = "store", type = "string", dest = "xre_path",
-                      default = None,
-                      help = "absolute path to directory containing XRE (probably xulrunner)")
-    parser.add_option("--symbols-path",
-                      action = "store", type = "string", dest = "symbols_path",
-                      default = None,
-                      help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+def main():
+    parser = CPPUnittestOptions()
     options, args = parser.parse_args()
     if not args:
         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
         sys.exit(1)
     if not options.xre_path:
         print >>sys.stderr, """Error: --xre-path is required"""
         sys.exit(1)
     progs = [os.path.abspath(p) for p in args]
     options.xre_path = os.path.abspath(options.xre_path)
-    result = run_tests(progs, options.xre_path, options.symbols_path)
+    tester = CPPUnitTests()
+    try:
+        result = tester.run_tests(progs, options.xre_path, options.symbols_path)
+    except Exception, e:
+        log.error(str(e))
+        result = False
     sys.exit(0 if result else 1)
+
+if __name__ == '__main__':
+    main()
+
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -322,16 +322,33 @@ RUN_PEPTEST = \
           --proxy-host-dirs \
           --server-path=_tests/peptest/tests/firefox/server \
           --log-file=./$@.log $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS)
 
 peptest:
 	$(RUN_PEPTEST)
 	$(CHECK_TEST_ERROR)
 
+REMOTE_CPPUNITTESTS = \
+	$(PYTHON) -u $(topsrcdir)/testing/remotecppunittests.py \
+	  --xre-path=$(DEPTH)/dist/bin \
+	  --localLib=$(DEPTH)/dist/fennec \
+	  --dm_trans=$(DM_TRANS) \
+	  --deviceIP=${TEST_DEVICE} \
+	  $(TEST_PATH) $(EXTRA_TEST_ARGS)
+
+# Usage: |make [TEST_PATH=...] [EXTRA_TEST_ARGS=...] cppunittests-remote|.
+cppunittests-remote: DM_TRANS?=adb
+cppunittests-remote:
+	@if [ "${TEST_DEVICE}" != "" -o "$(DM_TRANS)" = "adb" ]; \
+          then $(call REMOTE_CPPUNITTESTS); \
+        else \
+          echo "please prepare your host with environment variables for TEST_DEVICE"; \
+        fi
+
 # Package up the tests and test harnesses
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 ifndef UNIVERSAL_BINARY
 PKG_STAGE = $(DIST)/test-package-stage
 package-tests: \
   stage-mochitest \
   stage-reftest \