Bug 811411 - Add ability to run C++ unit tests on mobile; r=ted.mielczarek
authorGeoff Brown <gbrown@mozilla.com>
Thu, 03 Jan 2013 13:01:54 -0700
changeset 123057 d834b07541e05cc33e80943d6fdb3747f76e98e6
parent 123056 958f452fc41ed816fb56b29e56245063469e2092
child 123058 79e3ae50976460a55e57bd6ba56bef594bc0dc24
push idunknown
push userunknown
push dateunknown
reviewersted
bugs811411
milestone20.0a1
Bug 811411 - Add ability to run C++ unit tests on mobile; r=ted.mielczarek
testing/remotecppunittests.py
testing/runcppunittests.py
testing/testsuite-targets.mk
new file mode 100644
--- /dev/null
+++ b/testing/remotecppunittests.py
@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os, sys
+import runcppunittests as cppunittests
+import mozcrash, mozlog
+import StringIO
+import posixpath
+from mozdevice import devicemanager, devicemanagerADB, devicemanagerSUT
+
+log = mozlog.getLogger('remotecppunittests')
+
+class RemoteCPPUnitTests(cppunittests.CPPUnitTests):
+    def __init__(self, devmgr, options, progs):
+        cppunittests.CPPUnitTests.__init__(self)
+        self.options = options
+        self.device = devmgr
+        self.remote_test_root = self.device.getDeviceRoot() + "/cppunittests"
+        self.remote_bin_dir = posixpath.join(self.remote_test_root, "b")
+        self.remote_tmp_dir = posixpath.join(self.remote_test_root, "tmp")
+        self.remote_profile_dir = posixpath.join(self.remote_test_root, "p")
+        if options.setup:
+            self.setup_bin(progs)
+
+    def setup_bin(self, progs):
+        if not self.device.dirExists(self.remote_test_root):
+            self.device.mkDir(self.remote_test_root)
+        if self.device.dirExists(self.remote_tmp_dir):
+            self.device.removeDir(self.remote_tmp_dir)
+        self.device.mkDir(self.remote_tmp_dir)
+        if self.device.dirExists(self.remote_bin_dir):
+            self.device.removeDir(self.remote_bin_dir)
+        self.device.mkDir(self.remote_bin_dir)
+        self.push_libs()
+        self.push_progs(progs)
+        self.device.chmodDir(self.remote_bin_dir)
+
+    def push_libs(self):
+        for file in os.listdir(self.options.local_lib):
+            if file.endswith(".so"):
+                print >> sys.stderr, "Pushing %s.." % file
+                remote_file = posixpath.join(self.remote_bin_dir, file)
+                self.device.pushFile(os.path.join(self.options.local_lib, file), remote_file)
+        # Additional libraries may be found in a sub-directory such as "lib/armeabi-v7a"
+        local_arm_lib = os.path.join(self.options.local_lib, "lib")
+        if os.path.isdir(local_arm_lib):
+            for root, dirs, files in os.walk(local_arm_lib):
+                for file in files:
+                    if (file.endswith(".so")):
+                        remote_file = posixpath.join(self.remote_bin_dir, file)
+                        self.device.pushFile(os.path.join(root, file), remote_file)
+
+    def push_progs(self, progs):
+        for local_file in progs:
+            remote_file = posixpath.join(self.remote_bin_dir, os.path.basename(local_file))
+            self.device.pushFile(local_file, remote_file)
+
+    def build_environment(self):
+        env = self.build_core_environment()
+        env['LD_LIBRARY_PATH'] = self.remote_bin_dir
+        return env
+
+    def run_one_test(self, prog, env, symbols_path=None):
+        """
+        Run a single C++ unit test program remotely.
+
+        Arguments:
+        * prog: The path to the test program to run.
+        * env: The environment to use for running the program.
+        * symbols_path: A path to a directory containing Breakpad-formatted
+                        symbol files for producing stack traces on crash.
+
+        Return True if the program exits with a zero status, False otherwise.
+        """
+        basename = os.path.basename(prog)
+        remote_bin = posixpath.join(self.remote_bin_dir, basename)
+        log.info("Running test %s", basename)
+        buf = StringIO.StringIO()
+        returncode = self.device.shell([remote_bin], buf, env=env, cwd=self.remote_tmp_dir,
+                                       timeout=cppunittests.CPPUnitTests.TEST_PROC_TIMEOUT)
+        print >> sys.stdout, buf.getvalue()
+        with cppunittests.TemporaryDirectory() as tempdir:
+            self.device.getDirectory(self.remote_tmp_dir, tempdir)
+            if mozcrash.check_for_crashes(tempdir, symbols_path,
+                                          test_name=basename):
+                log.testFail("%s | test crashed", basename)
+                return False
+        result = returncode == 0
+        if not result:
+            log.testFail("%s | test failed with return code %s",
+                         basename, returncode)
+        return result
+
+class RemoteCPPUnittestOptions(cppunittests.CPPUnittestOptions):
+    def __init__(self):
+        cppunittests.CPPUnittestOptions.__init__(self)
+        defaults = {}
+
+        self.add_option("--deviceIP", action="store",
+                        type = "string", dest = "device_ip",
+                        help = "ip address of remote device to test")
+        defaults["device_ip"] = None
+
+        self.add_option("--devicePort", action="store",
+                        type = "string", dest = "device_port",
+                        help = "port of remote device to test")
+        defaults["device_port"] = 20701
+
+        self.add_option("--dm_trans", action="store",
+                        type = "string", dest = "dm_trans",
+                        help = "the transport to use to communicate with device: [adb|sut]; default=sut")
+        defaults["dm_trans"] = "sut"
+
+        self.add_option("--noSetup", action="store_false",
+                        dest = "setup",
+                        help = "do not copy any files to device (to be used only if device is already setup)")
+        defaults["setup"] = True
+
+        self.add_option("--localLib", action="store",
+                        type = "string", dest = "local_lib",
+                        help = "location of libraries to push -- preferably stripped")
+        defaults["local_lib"] = None
+
+        self.set_defaults(**defaults)
+
+def main():
+    parser = RemoteCPPUnittestOptions()
+    options, args = parser.parse_args()
+    if not args:
+        print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
+        sys.exit(1)
+    if not options.local_lib:
+        print >>sys.stderr, """Error: --localLib is required"""
+        sys.exit(1)
+    if not os.path.isdir(options.local_lib):
+        print >>sys.stderr, """Error: --localLib directory %s not found""" % options.local_lib
+        sys.exit(1)
+    if not options.xre_path:
+        print >>sys.stderr, """Error: --xre-path is required"""
+        sys.exit(1)
+    if options.dm_trans == "adb":
+        if options.device_ip:
+            dm = devicemanagerADB.DeviceManagerADB(options.device_ip, options.device_port, packageName=None)
+        else:
+            dm = devicemanagerADB.DeviceManagerADB(packageName=None)
+    else:
+        dm = devicemanagerSUT.DeviceManagerSUT(options.device_ip, options.device_port)
+        if not options.device_ip:
+            print "Error: you must provide a device IP to connect to via the --deviceIP option"
+            sys.exit(1)
+    options.xre_path = os.path.abspath(options.xre_path)
+    progs = [os.path.abspath(p) for p in args]
+    tester = RemoteCPPUnitTests(dm, options, progs)
+    try:
+        result = tester.run_tests(progs, options.xre_path, options.symbols_path)
+    except Exception, e:
+        log.error(str(e))
+        result = False
+    sys.exit(0 if result else 1)
+
+if __name__ == '__main__':
+    main()
--- a/testing/runcppunittests.py
+++ b/testing/runcppunittests.py
@@ -1,120 +1,151 @@
 #!/usr/bin/env python
 #
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from __future__ import with_statement
-import sys, optparse, os, tempfile, shutil
+import sys, os, tempfile, shutil
+from optparse import OptionParser
 import mozprocess, mozinfo, mozlog, mozcrash
 from contextlib import contextmanager
 
 log = mozlog.getLogger('cppunittests')
 
 @contextmanager
 def TemporaryDirectory():
     tempdir = tempfile.mkdtemp()
     yield tempdir
     shutil.rmtree(tempdir)
 
-def run_one_test(prog, env, symbols_path=None):
-    """
-    Run a single C++ unit test program.
+class CPPUnitTests(object):
+    # Time (seconds) to wait for test process to complete
+    TEST_PROC_TIMEOUT = 300
+
+    def run_one_test(self, prog, env, symbols_path=None):
+        """
+        Run a single C++ unit test program.
+
+        Arguments:
+        * prog: The path to the test program to run.
+        * env: The environment to use for running the program.
+        * symbols_path: A path to a directory containing Breakpad-formatted
+                        symbol files for producing stack traces on crash.
 
-    Arguments:
-    * prog: The path to the test program to run.
-    * env: The environment to use for running the program.
-    * symbols_path: A path to a directory containing Breakpad-formatted
-                    symbol files for producing stack traces on crash.
+        Return True if the program exits with a zero status, False otherwise.
+        """
+        basename = os.path.basename(prog)
+        log.info("Running test %s", basename)
+        with TemporaryDirectory() as tempdir:
+            proc = mozprocess.ProcessHandler([prog],
+                                             cwd=tempdir,
+                                             env=env)
+            #TODO: After bug 811320 is fixed, don't let .run() kill the process,
+            # instead use a timeout in .wait() and then kill to get a stack.
+            proc.run(timeout=CPPUnitTests.TEST_PROC_TIMEOUT)
+            proc.wait()
+            if proc.timedOut:
+                log.testFail("%s | timed out after %d seconds",
+                             basename, CPPUnitTests.TEST_PROC_TIMEOUT)
+                return False
+            if mozcrash.check_for_crashes(tempdir, symbols_path,
+                                          test_name=basename):
+                log.testFail("%s | test crashed", basename)
+                return False
+            result = proc.proc.returncode == 0
+            if not result:
+                log.testFail("%s | test failed with return code %d",
+                             basename, proc.proc.returncode)
+            return result
 
-    Return True if the program exits with a zero status, False otherwise.
-    """
-    basename = os.path.basename(prog)
-    log.info("Running test %s", basename)
-    with TemporaryDirectory() as tempdir:
-        proc = mozprocess.ProcessHandler([prog],
-                                         cwd=tempdir,
-                                         env=env)
-        timeout = 300
-        #TODO: After bug 811320 is fixed, don't let .run() kill the process,
-        # instead use a timeout in .wait() and then kill to get a stack.
-        proc.run(timeout=timeout)
-        proc.wait()
-        if proc.timedOut:
-            log.testFail("%s | timed out after %d seconds",
-                         basename, timeout)
-            return False
-        if mozcrash.check_for_crashes(tempdir, symbols_path,
-                                      test_name=basename):
-            log.testFail("%s | test crashed", basename)
-            return False
-        result = proc.proc.returncode == 0
-        if not result:
-            log.testFail("%s | test failed with return code %d",
-                         basename, proc.proc.returncode)
+    def build_core_environment(self, env = {}):
+        """
+        Add environment variables likely to be used across all platforms, including remote systems.
+        """
+        env["MOZILLA_FIVE_HOME"] = self.xre_path
+        env["MOZ_XRE_DIR"] = self.xre_path
+        #TODO: switch this to just abort once all C++ unit tests have
+        # been fixed to enable crash reporting
+        env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
+        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+        env["MOZ_CRASHREPORTER"] = "1"
+        return env
+
+    def build_environment(self):
+        """
+        Create and return a dictionary of all the appropriate env variables and values.
+        On a remote system, we overload this to set different values and are missing things like os.environ and PATH.
+        """
+        if not os.path.isdir(self.xre_path):
+            raise Exception("xre_path does not exist: %s", self.xre_path)
+        env = dict(os.environ)
+        env = self.build_core_environment(env)
+        pathvar = ""
+        if mozinfo.os == "linux":
+            pathvar = "LD_LIBRARY_PATH"
+        elif mozinfo.os == "mac":
+            pathvar = "DYLD_LIBRARY_PATH"
+        elif mozinfo.os == "win":
+            pathvar = "PATH"
+        if pathvar:
+            if pathvar in env:
+                env[pathvar] = "%s%s%s" % (self.xre_path, os.pathsep, env[pathvar])
+            else:
+                env[pathvar] = self.xre_path
+        return env
+
+    def run_tests(self, programs, xre_path, symbols_path=None):
+        """
+        Run a set of C++ unit test programs.
+
+        Arguments:
+        * programs: An iterable containing paths to test programs.
+        * xre_path: A path to a directory containing a XUL Runtime Environment.
+        * symbols_path: A path to a directory containing Breakpad-formatted
+                        symbol files for producing stack traces on crash.
+
+        Returns True if all test programs exited with a zero status, False
+        otherwise.
+        """
+        self.xre_path = xre_path
+        env = self.build_environment()
+        result = True
+        for prog in programs:
+            single_result = self.run_one_test(prog, env, symbols_path)
+            result = result and single_result
         return result
 
-def run_tests(programs, xre_path, symbols_path=None):
-    """
-    Run a set of C++ unit test programs.
-
-    Arguments:
-    * programs: An iterable containing paths to test programs.
-    * xre_path: A path to a directory containing a XUL Runtime Environment.
-    * symbols_path: A path to a directory containing Breakpad-formatted
-                    symbol files for producing stack traces on crash.
+class CPPUnittestOptions(OptionParser):
+    def __init__(self):
+        OptionParser.__init__(self)
+        self.add_option("--xre-path",
+                        action = "store", type = "string", dest = "xre_path",
+                        default = None,
+                        help = "absolute path to directory containing XRE (probably xulrunner)")
+        self.add_option("--symbols-path",
+                        action = "store", type = "string", dest = "symbols_path",
+                        default = None,
+                        help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
 
-    Returns True if all test programs exited with a zero status, False
-    otherwise.
-    """
-    if not os.path.isdir(xre_path):
-        log.error("xre_path does not exist: %s", xre_path)
-        return False
-    #TODO: stick this in a module?
-    env = dict(os.environ)
-    pathvar = ""
-    if mozinfo.os == "linux":
-        pathvar = "LD_LIBRARY_PATH"
-    elif mozinfo.os == "mac":
-        pathvar = "DYLD_LIBRARY_PATH"
-    elif mozinfo.os == "win":
-        pathvar = "PATH"
-    if pathvar:
-        if pathvar in env:
-            env[pathvar] = "%s%s%s" % (xre_path, os.pathsep, env[pathvar])
-        else:
-            env[pathvar] = xre_path
-    env["MOZILLA_FIVE_HOME"] = xre_path
-    env["MOZ_XRE_DIR"] = xre_path
-    #TODO: switch this to just abort once all C++ unit tests have
-    # been fixed to enable crash reporting
-    env["XPCOM_DEBUG_BREAK"] = "stack-and-abort"
-    env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
-    env["MOZ_CRASHREPORTER"] = "1"
-    result = True
-    for prog in programs:
-        single_result = run_one_test(prog, env, symbols_path)
-        result = result and single_result
-    return result
-
-if __name__ == '__main__':
-    parser = optparse.OptionParser()
-    parser.add_option("--xre-path",
-                      action = "store", type = "string", dest = "xre_path",
-                      default = None,
-                      help = "absolute path to directory containing XRE (probably xulrunner)")
-    parser.add_option("--symbols-path",
-                      action = "store", type = "string", dest = "symbols_path",
-                      default = None,
-                      help = "absolute path to directory containing breakpad symbols, or the URL of a zip file containing symbols")
+def main():
+    parser = CPPUnittestOptions()
     options, args = parser.parse_args()
     if not args:
         print >>sys.stderr, """Usage: %s <test binary> [<test binary>...]""" % sys.argv[0]
         sys.exit(1)
     if not options.xre_path:
         print >>sys.stderr, """Error: --xre-path is required"""
         sys.exit(1)
     progs = [os.path.abspath(p) for p in args]
     options.xre_path = os.path.abspath(options.xre_path)
-    result = run_tests(progs, options.xre_path, options.symbols_path)
+    tester = CPPUnitTests()
+    try:
+        result = tester.run_tests(progs, options.xre_path, options.symbols_path)
+    except Exception, e:
+        log.error(str(e))
+        result = False
     sys.exit(0 if result else 1)
+
+if __name__ == '__main__':
+    main()
+
--- a/testing/testsuite-targets.mk
+++ b/testing/testsuite-targets.mk
@@ -322,16 +322,33 @@ RUN_PEPTEST = \
           --proxy-host-dirs \
           --server-path=_tests/peptest/tests/firefox/server \
           --log-file=./$@.log $(SYMBOLS_PATH) $(EXTRA_TEST_ARGS)
 
 peptest:
 	$(RUN_PEPTEST)
 	$(CHECK_TEST_ERROR)
 
+REMOTE_CPPUNITTESTS = \
+	$(PYTHON) -u $(topsrcdir)/testing/remotecppunittests.py \
+	  --xre-path=$(DEPTH)/dist/bin \
+	  --localLib=$(DEPTH)/dist/fennec \
+	  --dm_trans=$(DM_TRANS) \
+	  --deviceIP=${TEST_DEVICE} \
+	  $(TEST_PATH) $(EXTRA_TEST_ARGS)
+
+# Usage: |make [TEST_PATH=...] [EXTRA_TEST_ARGS=...] cppunittests-remote|.
+cppunittests-remote: DM_TRANS?=adb
+cppunittests-remote:
+	@if [ "${TEST_DEVICE}" != "" -o "$(DM_TRANS)" = "adb" ]; \
+          then $(call REMOTE_CPPUNITTESTS); \
+        else \
+          echo "please prepare your host with environment variables for TEST_DEVICE"; \
+        fi
+
 # Package up the tests and test harnesses
 include $(topsrcdir)/toolkit/mozapps/installer/package-name.mk
 
 ifndef UNIVERSAL_BINARY
 PKG_STAGE = $(DIST)/test-package-stage
 package-tests: \
   stage-mochitest \
   stage-reftest \