Bug 1445716 - Add runjunit.py, a test harness for junit tests on Android devices; r=jmaher
authorGeoff Brown <gbrown@mozilla.com>
Wed, 18 Apr 2018 14:43:33 -0600
changeset 467959 5247cde8ef0c1cc2c817fe693e83854579f27253
parent 467958 36688a93952f2fd8175a3893fc469d64c3d4784a
child 467960 05412fcc00a740e9a718dfbd34e1011007335423
push id9165
push userasasaki@mozilla.com
push dateThu, 26 Apr 2018 21:04:54 +0000
treeherdermozilla-beta@064c3804de2e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1445716
milestone61.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1445716 - Add runjunit.py, a test harness for junit tests on Android devices; r=jmaher
testing/mochitest/moz.build
testing/mochitest/rungeckoview.py
testing/mochitest/runjunit.py
--- a/testing/mochitest/moz.build
+++ b/testing/mochitest/moz.build
@@ -51,16 +51,17 @@ TEST_HARNESS_FILES.testing.mochitest += 
     'mach_test_package_commands.py',
     'manifest.webapp',
     'manifestLibrary.js',
     'mochitest_options.py',
     'nested_setup.js',
     'pywebsocket_wrapper.py',
     'redirect.html',
     'rungeckoview.py',
+    'runjunit.py',
     'runrobocop.py',
     'runtests.py',
     'runtestsremote.py',
     'server.js',
     'start_desktop.js',
 ]
 
 TEST_HARNESS_FILES.testing.mochitest.embed += [
--- a/testing/mochitest/rungeckoview.py
+++ b/testing/mochitest/rungeckoview.py
@@ -142,19 +142,19 @@ class GeckoviewTestRunner:
         fail_count = 0
         for test in all_tests:
             self.test_name = test.__doc__.strip()
             self.log.test_start(self.test_name)
 
             expected = 'PASS'
             (passed, message) = test()
             if passed:
-                pass_count = pass_count + 1
+                pass_count += 1
             else:
-                fail_count = fail_count + 1
+                fail_count += 1
             status = 'PASS' if passed else 'FAIL'
 
             self.log.test_end(self.test_name, status, expected, message)
 
         crashed = self.check_for_crashes()
         if crashed:
             fail_count = 1
         else:
@@ -190,17 +190,17 @@ class GeckoviewTestRunner:
                 # Whilst no crash was found, the run should still display as a failure
                 return True
             self.device.pull(remote_dir, dump_dir)
             crashed = mozcrash.log_crashes(self.log, dump_dir, symbols_path, test=self.test_name)
         finally:
             try:
                 shutil.rmtree(dump_dir)
             except Exception:
-                self.log.warn("unable to remove directory: %s" % dump_dir)
+                self.log.warning("unable to remove directory: %s" % dump_dir)
         return crashed
 
     def cleanup(self):
         """
            Cleanup at end of job run.
         """
         self.log.debug("Cleaning up...")
         self.device.stop_application(self.appname)
new file mode 100644
--- /dev/null
+++ b/testing/mochitest/runjunit.py
@@ -0,0 +1,405 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import argparse
+import json
+import os
+import posixpath
+import re
+import shutil
+import sys
+import tempfile
+import traceback
+
+import mozcrash
+import mozinfo
+import mozlog
+import moznetwork
+from mozdevice import ADBAndroid
+from mozprofile import Profile, Preferences, DEFAULT_PORTS
+from runtests import MochitestDesktop, update_mozinfo
+
+here = os.path.abspath(os.path.dirname(__file__))
+
+try:
+    from mozbuild.base import (
+        MozbuildObject,
+        MachCommandConditions as conditions,
+    )
+    build_obj = MozbuildObject.from_environment(cwd=here)
+except ImportError:
+    build_obj = None
+    conditions = None
+
+
+class JUnitTestRunner(MochitestDesktop):
+    """
+       A test harness to run geckoview junit tests on a remote device.
+    """
+
+    def __init__(self, log, options):
+        self.log = log
+        verbose = False
+        if options.log_tbpl_level == 'debug' or options.log_mach_level == 'debug':
+            verbose = True
+        self.device = ADBAndroid(adb=options.adbPath or 'adb',
+                                 device=options.deviceSerial,
+                                 test_root=options.remoteTestRoot,
+                                 verbose=verbose)
+        self.options = options
+        self.log.debug("options=%s" % vars(options))
+        update_mozinfo()
+        self.remote_profile = posixpath.join(self.device.test_root, 'junit-profile')
+        self.server_init()
+
+        self.cleanup()
+        self.device.clear_logcat()
+        self.build_profile()
+        self.startServers(
+            self.options,
+            debuggerInfo=None,
+            ignoreSSLTunnelExts=True)
+        self.log.debug("Servers started")
+
+    def server_init(self):
+        """
+           Additional initialization required to satisfy MochitestDesktop.startServers
+        """
+        self._locations = None
+        self.server = None
+        self.wsserver = None
+        self.websocketProcessBridge = None
+        self.SERVER_STARTUP_TIMEOUT = 180 if mozinfo.info.get('debug') else 90
+        if self.options.remoteWebServer is None:
+            if os.name != "nt":
+                self.options.remoteWebServer = moznetwork.get_ip()
+            else:
+                raise Exception("--remote-webserver must be specified")
+        self.options.webServer = self.options.remoteWebServer
+        self.options.webSocketPort = '9988'
+        self.options.httpdPath = None
+        self.options.keep_open = False
+        self.options.pidFile = ""
+        self.options.subsuite = None
+        self.options.xrePath = None
+        if build_obj and 'MOZ_HOST_BIN' in os.environ:
+            self.options.xrePath = os.environ['MOZ_HOST_BIN']
+            if not self.options.utilityPath:
+                self.options.utilityPath = self.options.xrePath
+        if not self.options.xrePath:
+            self.options.xrePath = self.options.utilityPath
+        if build_obj:
+            self.options.certPath = os.path.join(build_obj.topsrcdir,
+                                                 'build', 'pgo', 'certs')
+
+    def build_profile(self):
+        """
+           Create a local profile with test prefs and proxy definitions and
+           push it to the remote device.
+        """
+        preferences = [
+            os.path.join(
+                here,
+                'profile_data',
+                'prefs_general.js')]
+
+        prefs = {}
+        for path in preferences:
+            prefs.update(Preferences.read_prefs(path))
+
+        interpolation = {
+            "server": "%s:%s" %
+            (self.options.webServer, self.options.httpPort)}
+
+        prefs = json.loads(json.dumps(prefs) % interpolation)
+        for pref in prefs:
+            prefs[pref] = Preferences.cast(prefs[pref])
+
+        proxy = {'remote': self.options.webServer,
+                 'http': self.options.httpPort,
+                 'https': self.options.sslPort,
+                 'ws': self.options.sslPort
+                 }
+
+        self.profile = Profile(preferences=prefs, proxy=proxy)
+        self.options.profilePath = self.profile.profile
+
+        if self.fillCertificateDB(self.options):
+            self.log.error("Certificate integration failed")
+
+        self.device.mkdir(self.remote_profile, parents=True)
+        self.device.push(self.profile.profile, self.remote_profile)
+        self.log.debug("profile %s -> %s" %
+                       (str(self.profile.profile), str(self.remote_profile)))
+
+    def cleanup(self):
+        try:
+            self.stopServers()
+            self.log.debug("Servers stopped")
+            self.device.stop_application(self.options.app)
+            self.device.rm(self.remote_profile, force=True, recursive=True)
+            if hasattr(self, 'profile'):
+                del self.profile
+        except Exception:
+            traceback.print_exc()
+            self.log.info("Caught and ignored an exception during cleanup")
+
+    def build_command_line(self, test_filters):
+        """
+           Construct and return the 'am instrument' command line.
+        """
+        cmd = "am instrument -w -r"
+        # profile location
+        cmd = cmd + " -e args '-profile %s'" % self.remote_profile
+        # multi-process
+        e10s = 'true' if self.options.e10s else 'false'
+        cmd = cmd + " -e use_multiprocess %s" % e10s
+        # test filters: limit run to specific test(s)
+        for f in test_filters:
+            # filter can be class-name or 'class-name#method-name' (single test)
+            cmd = cmd + " -e class %s" % f
+        # environment
+        env = {}
+        env["MOZ_CRASHREPORTER"] = "1"
+        env["MOZ_CRASHREPORTER_NO_REPORT"] = "1"
+        env["MOZ_CRASHREPORTER_SHUTDOWN"] = "1"
+        env["XPCOM_DEBUG_BREAK"] = "stack"
+        env["DISABLE_UNSAFE_CPOW_WARNINGS"] = "1"
+        env["MOZ_DISABLE_NONLOCAL_CONNECTIONS"] = "1"
+        env["MOZ_IN_AUTOMATION"] = "1"
+        env["R_LOG_VERBOSE"] = "1"
+        env["R_LOG_LEVEL"] = "6"
+        env["R_LOG_DESTINATION"] = "stderr"
+        for (env_count, (env_key, env_val)) in enumerate(env.iteritems()):
+            cmd = cmd + " -e env%d %s=%s" % (env_count, env_key, env_val)
+        # runner
+        cmd = cmd + " %s/%s" % (self.options.app, self.options.runner)
+        return cmd
+
+    def run_tests(self, test_filters=None):
+        """
+           Run the tests.
+        """
+        if not self.device.is_app_installed(self.options.app):
+            raise Exception("%s is not installed" %
+                            self.options.app)
+        if self.device.process_exist(self.options.app):
+            raise Exception("%s already running before starting tests" %
+                            self.options.app)
+
+        self.test_started = False
+        self.pass_count = 0
+        self.fail_count = 0
+        self.class_name = ""
+        self.test_name = ""
+        self.current_full_name = ""
+
+        def callback(line):
+            # Output callback: Parse the raw junit log messages, translating into
+            # treeherder-friendly test start/pass/fail messages.
+
+            self.log.process_output(self.options.app, str(line))
+            expected = 'PASS'
+            # Expect per-test info like: "INSTRUMENTATION_STATUS: class=something"
+            match = re.match(r'INSTRUMENTATION_STATUS:\s*class=(.*)', line)
+            if match:
+                self.class_name = match.group(1)
+            # Expect per-test info like: "INSTRUMENTATION_STATUS: test=something"
+            match = re.match(r'INSTRUMENTATION_STATUS:\s*test=(.*)', line)
+            if match:
+                self.test_name = match.group(1)
+            # Expect per-test info like: "INSTRUMENTATION_STATUS_CODE: 0|1|..."
+            match = re.match(r'INSTRUMENTATION_STATUS_CODE:\s*([+-]?\d+)', line)
+            if match:
+                status = match.group(1)
+                full_name = "%s.%s" % (self.class_name, self.test_name)
+                if full_name == self.current_full_name:
+                    if status == '0':
+                        message = ''
+                        status = 'PASS'
+                        self.pass_count += 1
+                    else:
+                        message = 'status %s' % status
+                        status = 'FAIL'
+                        self.fail_count += 1
+                    self.log.test_end(full_name, status, expected, message)
+                    self.test_started = False
+                else:
+                    if self.test_started:
+                        # next test started without reporting previous status
+                        self.fail_count += 1
+                        self.log.test_end(self.current_full_name, 'FAIL', expected,
+                                          "missing test completion status")
+                    self.log.test_start(full_name)
+                    self.test_started = True
+                    self.current_full_name = full_name
+
+        # Ideally all test names should be reported to suite_start, but these test
+        # names are not known in advance.
+        self.log.suite_start(["geckoview-junit"])
+        try:
+            cmd = self.build_command_line(test_filters)
+            self.log.info("launching %s" % cmd)
+            self.device.shell(cmd, timeout=self.options.max_time, stdout_callback=callback)
+            self.log.info("Passed: %d" % self.pass_count)
+            self.log.info("Failed: %d" % self.fail_count)
+        finally:
+            self.log.suite_end()
+
+        if self.check_for_crashes():
+            self.fail_count = 1
+
+        return 1 if self.fail_count else 0
+
+    def check_for_crashes(self):
+        logcat = self.device.get_logcat()
+        if logcat:
+            if mozcrash.check_for_java_exception(logcat, self.current_full_name):
+                return True
+        symbols_path = self.options.symbolsPath
+        try:
+            dump_dir = tempfile.mkdtemp()
+            remote_dir = posixpath.join(self.remote_profile, 'minidumps')
+            if not self.device.is_dir(remote_dir):
+                # If crash reporting is enabled (MOZ_CRASHREPORTER=1), the
+                # minidumps directory is automatically created when the app
+                # (first) starts, so its lack of presence is a hint that
+                # something went wrong.
+                print "Automation Error: No crash directory (%s) found on remote device" % \
+                    remote_dir
+                return True
+            self.device.pull(remote_dir, dump_dir)
+            crashed = mozcrash.log_crashes(self.log, dump_dir, symbols_path,
+                                           test=self.current_full_name)
+        finally:
+            try:
+                shutil.rmtree(dump_dir)
+            except Exception:
+                self.log.warning("unable to remove directory: %s" % dump_dir)
+        return crashed
+
+
+class JunitArgumentParser(argparse.ArgumentParser):
+    """
+       An argument parser for geckoview-junit.
+    """
+    def __init__(self, **kwargs):
+        super(JunitArgumentParser, self).__init__(**kwargs)
+
+        self.add_argument("--appname",
+                          action="store",
+                          type=str,
+                          dest="app",
+                          default="org.mozilla.geckoview.test",
+                          help="Test package name.")
+        self.add_argument("--adbpath",
+                          action="store",
+                          type=str,
+                          dest="adbPath",
+                          default=None,
+                          help="Path to adb executable.")
+        self.add_argument("--deviceSerial",
+                          action="store",
+                          type=str,
+                          dest="deviceSerial",
+                          help="adb serial number of remote device.")
+        self.add_argument("--remoteTestRoot",
+                          action="store",
+                          type=str,
+                          dest="remoteTestRoot",
+                          help="Remote directory to use as test root "
+                               "(eg. /mnt/sdcard/tests or /data/local/tests).")
+        self.add_argument("--disable-e10s",
+                          action="store_false",
+                          dest="e10s",
+                          default=True,
+                          help="Disable multiprocess mode in test app.")
+        self.add_argument("--max-time",
+                          action="store",
+                          type=str,
+                          dest="max_time",
+                          default="2400",
+                          help="Max time in seconds to wait for tests (default 2400s).")
+        self.add_argument("--runner",
+                          action="store",
+                          type=str,
+                          dest="runner",
+                          default="android.support.test.runner.AndroidJUnitRunner",
+                          help="Test runner name.")
+        self.add_argument("--symbols-path",
+                          action="store",
+                          type=str,
+                          dest="symbolsPath",
+                          default=None,
+                          help="Path to directory containing breakpad symbols, "
+                               "or the URL of a zip file containing symbols.")
+        self.add_argument("--utility-path",
+                          action="store",
+                          type=str,
+                          dest="utilityPath",
+                          default=None,
+                          help="Path to directory containing host utility programs.")
+        # Additional options for server.
+        self.add_argument("--certificate-path",
+                          action="store",
+                          type=str,
+                          dest="certPath",
+                          default=None,
+                          help="Path to directory containing certificate store."),
+        self.add_argument("--http-port",
+                          action="store",
+                          type=str,
+                          dest="httpPort",
+                          default=DEFAULT_PORTS['http'],
+                          help="Port of the web server for http traffic.")
+        self.add_argument("--remote-webserver",
+                          action="store",
+                          type=str,
+                          dest="remoteWebServer",
+                          help="IP address of the webserver.")
+        self.add_argument("--ssl-port",
+                          action="store",
+                          type=str,
+                          dest="sslPort",
+                          default=DEFAULT_PORTS['https'],
+                          help="Port of the web server for https traffic.")
+        # Remaining arguments are test filters.
+        self.add_argument("test_filters",
+                          nargs="*",
+                          help="Test filter(s): class and/or method names of test(s) to run.")
+
+        mozlog.commandline.add_logging_group(self)
+
+
+def run_test_harness(parser, options):
+    if hasattr(options, 'log'):
+        log = options.log
+    else:
+        log = mozlog.commandline.setup_logging("runjunit", options,
+                                               {"tbpl": sys.stdout})
+    runner = JUnitTestRunner(log, options)
+    result = -1
+    try:
+        result = runner.run_tests(options.test_filters)
+    except KeyboardInterrupt:
+        log.info("runjunit.py | Received keyboard interrupt")
+        result = -1
+    except Exception:
+        traceback.print_exc()
+        log.error(
+            "runjunit.py | Received unexpected exception while running tests")
+        result = 1
+    finally:
+        runner.cleanup()
+    return result
+
+
+def main(args=sys.argv[1:]):
+    parser = JunitArgumentParser()
+    options = parser.parse_args()
+    return run_test_harness(parser, options)
+
+
+if __name__ == "__main__":
+    sys.exit(main())