Bug 1426586 - Add mozharness script and config for android hardware tests, r=gbrown
authorBob Clary <bclary@bclary.com>
Tue, 19 Dec 2017 00:26:18 -0800
changeset 809875 13c99a5c12fffe8a32d73d2ca43bb76d5628bc75
parent 809874 baace08171da87195d064a7d543488bb75bda956
child 809876 7f5be9c3698ef7280e498a7eedc393ad153785ef
push id113833
push userbmo:ato@sny.no
push dateSat, 23 Jun 2018 12:17:51 +0000
reviewersgbrown
bugs1426586
milestone62.0a1
Bug 1426586 - Add mozharness script and config for android hardware tests, r=gbrown
testing/mozharness/configs/android/android_hw.py
testing/mozharness/scripts/android_hardware_unittest.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/android/android_hw.py
@@ -0,0 +1,343 @@
+import os
+
+config = {
+    "robocop_package_name": "org.mozilla.roboexample.test",
+    "marionette_address": "%(device_ip)s:2828",
+    "marionette_test_manifest": "unit-tests.ini",
+    "exes": {},
+    "log_tbpl_level": "info",
+    "log_raw_level": "info",
+    "env": {
+        "DISPLAY": ":0.0",
+        "PATH": "%(PATH)s",
+        "MINIDUMP_SAVEPATH": "%(abs_work_dir)s/../minidumps"
+    },
+    "default_actions": [
+        'clobber',
+        'download-and-extract',
+        'create-virtualenv',
+        'verify-device',
+        'install',
+        'run-tests',
+    ],
+    # from android_common.py
+    "download_tooltool": True,
+    "download_minidump_stackwalk": True,
+    # hostutils_manifest_path is relative to branch's root in hg.mozilla.org.
+    "hostutils_manifest_path": "testing/config/tooltool-manifests/linux64/hostutils.manifest",
+    "tooltool_cache": "/builds/worker/tooltool_cache",
+    "tooltool_servers": ['https://api.pub.build.mozilla.org/tooltool/'],
+    # minidump_tooltool_manifest_path is relative to workspace/build/tests/
+    "minidump_tooltool_manifest_path": "config/tooltool-manifests/linux64/releng.manifest",
+    "find_links": [
+        "https://pypi.pub.build.mozilla.org/pub",
+    ],
+    "pip_index": False,
+    "suite_definitions": {
+        "mochitest": {
+            "run_filename": "runtestsremote.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--app=%(app)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--certificate-path=%(certs_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--extra-profile-file=fonts",
+                "--extra-profile-file=hyphenation",
+                "--screenshot-on-fail",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "mochitest-gl": {
+            "run_filename": "runtestsremote.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--app=%(app)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--certificate-path=%(certs_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--screenshot-on-fail",
+                "--subsuite=webgl",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "mochitest-chrome": {
+            "run_filename": "runtestsremote.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--app=%(app)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--certificate-path=%(certs_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--extra-profile-file=fonts",
+                "--extra-profile-file=hyphenation",
+                "--screenshot-on-fail",
+                "--flavor=chrome",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "mochitest-plain-gpu": {
+            "run_filename": "runtestsremote.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--app=%(app)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--certificate-path=%(certs_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--screenshot-on-fail",
+                "--subsuite=gpu",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "mochitest-plain-clipboard": {
+            "run_filename": "runtestsremote.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--app=%(app)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--certificate-path=%(certs_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--screenshot-on-fail",
+                "--subsuite=clipboard",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "mochitest-media": {
+            "run_filename": "runtestsremote.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--app=%(app)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--certificate-path=%(certs_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--screenshot-on-fail",
+                "--chunk-by-runtime",
+                "--subsuite=media",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "robocop": {
+            "run_filename": "runrobocop.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--app=%(app)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--certificate-path=%(certs_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--quiet",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--robocop-apk=../../robocop.apk",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "reftest": {
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest",
+            "options": [
+                "--app=%(app)s",
+                "--ignore-window-size",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--httpd-path", "%(modules_dir)s",
+                "--symbols-path=%(symbols_path)s",
+                "--extra-profile-file=fonts",
+                "--extra-profile-file=hyphenation",
+                "--suite=reftest",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--deviceSerial=%(device_serial)s",
+            ],
+            "tests": ["tests/layout/reftests/reftest.list",],
+        },
+        "crashtest": {
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest",
+            "options": [
+                "--app=%(app)s",
+                "--ignore-window-size",
+                "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s",
+                "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s",
+                "--httpd-path",
+                "%(modules_dir)s",
+                "--symbols-path=%(symbols_path)s",
+                "--suite=crashtest",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--deviceSerial=%(device_serial)s",
+            ],
+            "tests": ["tests/testing/crashtest/crashtests.list",],
+        },
+        "jittest": {
+            "run_filename": "jit_test.py",
+            "testsdir": "jit-test/jit-test",
+            "options": [
+                "../../bin/js",
+                "--remote",
+                "-j",
+                "1",
+                "--localLib=../../bin",
+                "--no-slow",
+                "--no-progress",
+                "--format=automation",
+                "--jitflags=all",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "jsreftest": {
+            "run_filename": "remotereftest.py",
+            "testsdir": "reftest",
+            "options": [
+                "--app=%(app)s",
+                "--ignore-window-size",
+                "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s", "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s", "--httpd-path", "%(modules_dir)s",
+                "--symbols-path=%(symbols_path)s",
+                "--extra-profile-file=jsreftest/tests/user.js",
+                "--suite=jstestbrowser",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--deviceSerial=%(device_serial)s",
+            ],
+            "tests": ["../jsreftest/tests/jstests.list",],
+        },
+        "xpcshell": {
+            "run_filename": "remotexpcshelltests.py",
+            "testsdir": "xpcshell",
+            "install": False,
+            "options": [
+                "--xre-path=%(xre_path)s",
+                "--testing-modules-dir=%(modules_dir)s",
+                "--apk=%(installer_path)s",
+                "--no-logfiles",
+                "--symbols-path=%(symbols_path)s",
+                "--manifest=tests/xpcshell.ini",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--test-plugin-path=none",
+                "--deviceSerial=%(device_serial)s",
+                "--remoteTestRoot=/data/local/tests",
+            ],
+        },
+        "cppunittest": {
+            "run_filename": "remotecppunittests.py",
+            "testsdir": "cppunittest",
+            "install": False,
+            "options": [
+                "--symbols-path=%(symbols_path)s",
+                "--xre-path=%(xre_path)s",
+                "--localBinDir=../bin",
+                "--apk=%(installer_path)s",
+                ".",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "marionette": {
+            "run_filename": os.path.join("harness", "marionette_harness", "runtests.py"),
+            "testsdir": "marionette",
+            "options": [
+                "--app=fennec",
+                "--package=%(app)s",
+                "--address=%(address)s",
+                "%(test_manifest)s",
+                "--disable-e10s",
+                "--gecko-log=%(gecko_log)s",
+                "--log-raw=%(raw_log_file)s",
+                "--log-raw-level=%(log_raw_level)s",
+                "--log-errorsummary=%(error_summary_file)s",
+                "--log-tbpl-level=%(log_tbpl_level)s",
+                "--symbols-path=%(symbols_path)s",
+                "--startup-timeout=300",
+                "--device=%(device_serial)s",
+            ],
+        },
+        "geckoview": {
+            "run_filename": "rungeckoview.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--utility-path=%(utility_path)s",
+                "--symbols-path=%(symbols_path)s",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+        "geckoview-junit": {
+            "run_filename": "runjunit.py",
+            "testsdir": "mochitest",
+            "options": [
+                "--certificate-path=%(certs_path)s",
+                "--remote-webserver=%(remote_webserver)s",
+                "--symbols-path=%(symbols_path)s",
+                "--utility-path=%(utility_path)s",
+                "--deviceSerial=%(device_serial)s",
+            ],
+        },
+
+    },  # end suite_definitions
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/scripts/android_hardware_unittest.py
@@ -0,0 +1,723 @@
+#!/usr/bin/env python
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+
+import copy
+import datetime
+import glob
+import os
+import re
+import sys
+import signal
+import subprocess
+import time
+import tempfile
+
+# load modules from parent dir
+sys.path.insert(1, os.path.dirname(sys.path[0]))
+
+from mozharness.base.log import FATAL
+from mozharness.base.script import BaseScript, PreScriptAction, PostScriptAction
+from mozharness.mozilla.automation import TBPL_RETRY, EXIT_STATUS_DICT
+from mozharness.mozilla.mozbase import MozbaseMixin
+from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
+from mozharness.mozilla.testing.codecoverage import CodeCoverageMixin
+
+
+class AndroidHardwareTest(TestingMixin, BaseScript, MozbaseMixin,
+                          CodeCoverageMixin):
+    config_options = [[
+        ["--test-suite"],
+        {"action": "store",
+         "dest": "test_suite",
+         "default": None
+         }
+    ], [
+        ["--adb-path"],
+        {"action": "store",
+         "dest": "adb_path",
+         "default": None,
+         "help": "Path to adb",
+         }
+    ], [
+        ["--total-chunk"],
+        {"action": "store",
+         "dest": "total_chunks",
+         "default": None,
+         "help": "Number of total chunks",
+         }
+    ], [
+        ["--this-chunk"],
+        {"action": "store",
+         "dest": "this_chunk",
+         "default": None,
+         "help": "Number of this chunk",
+         }
+    ], [
+        ["--log-raw-level"],
+        {"action": "store",
+         "dest": "log_raw_level",
+         "default": "info",
+         "help": "Set log level (debug|info|warning|error|critical|fatal)",
+         }
+    ], [
+        ["--log-tbpl-level"],
+        {"action": "store",
+         "dest": "log_tbpl_level",
+         "default": "info",
+         "help": "Set log level (debug|info|warning|error|critical|fatal)",
+         }
+    ]] + copy.deepcopy(testing_config_options)
+
+    app_name = None
+
+    def __init__(self, require_config_file=False):
+        super(AndroidHardwareTest, self).__init__(
+            config_options=self.config_options,
+            all_actions=['clobber',
+                         'download-and-extract',
+                         'create-virtualenv',
+                         'verify-device',
+                         'install',
+                         'run-tests',
+                         ],
+            require_config_file=require_config_file,
+            config={
+                'virtualenv_modules': [],
+                'virtualenv_requirements': [],
+                'require_test_zip': True,
+                # IP address of the host as seen from the device.
+                'remote_webserver': os.environ['HOST_IP'],
+            }
+        )
+
+        # these are necessary since self.config is read only
+        c = self.config
+        abs_dirs = self.query_abs_dirs()
+        self.adb_path = self.query_exe('adb')
+        self.logcat_file = None
+        self.logcat_proc = None
+        self.installer_url = c.get('installer_url')
+        self.installer_path = c.get('installer_path')
+        self.test_url = c.get('test_url')
+        self.test_packages_url = c.get('test_packages_url')
+        self.test_manifest = c.get('test_manifest')
+        self.robocop_path = os.path.join(abs_dirs['abs_work_dir'], "robocop.apk")
+        self.minidump_stackwalk_path = c.get("minidump_stackwalk_path")
+        self.device_name = os.environ['DEVICE_NAME']
+        self.device_serial = os.environ['DEVICE_SERIAL']
+        self.device_ip = os.environ['DEVICE_IP']
+        self.test_suite = c.get('test_suite')
+        self.this_chunk = c.get('this_chunk')
+        self.total_chunks = c.get('total_chunks')
+        if self.test_suite and self.test_suite not in self.config["suite_definitions"]:
+            # accept old-style test suite name like "mochitest-3"
+            m = re.match("(.*)-(\d*)", self.test_suite)
+            if m:
+                self.test_suite = m.group(1)
+                if self.this_chunk is None:
+                    self.this_chunk = m.group(2)
+        self.sdk_level = None
+        self.xre_path = None
+        self.log_raw_level = c.get('log_raw_level')
+        self.log_tbpl_level = c.get('log_tbpl_level')
+
+    def _query_tests_dir(self):
+        dirs = self.query_abs_dirs()
+        try:
+            test_dir = self.config["suite_definitions"][self.test_suite]["testsdir"]
+        except Exception:
+            test_dir = self.test_suite
+        return os.path.join(dirs['abs_test_install_dir'], test_dir)
+
+    def query_abs_dirs(self):
+        if self.abs_dirs:
+            return self.abs_dirs
+        abs_dirs = super(AndroidHardwareTest, self).query_abs_dirs()
+        dirs = {}
+        dirs['abs_test_install_dir'] = os.path.join(
+            abs_dirs['abs_work_dir'], 'tests')
+        dirs['abs_test_bin_dir'] = os.path.join(
+            abs_dirs['abs_work_dir'], 'tests', 'bin')
+        dirs['abs_xre_dir'] = os.path.join(
+            abs_dirs['abs_work_dir'], 'hostutils')
+        dirs['abs_modules_dir'] = os.path.join(
+            dirs['abs_test_install_dir'], 'modules')
+        dirs['abs_blob_upload_dir'] = os.path.join(
+            abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+        dirs['abs_mochitest_dir'] = os.path.join(
+            dirs['abs_test_install_dir'], 'mochitest')
+        dirs['abs_reftest_dir'] = os.path.join(
+            dirs['abs_test_install_dir'], 'reftest')
+        dirs['abs_xpcshell_dir'] = os.path.join(
+            dirs['abs_test_install_dir'], 'xpcshell')
+        dirs['abs_marionette_dir'] = os.path.join(
+            dirs['abs_test_install_dir'], 'marionette', 'harness', 'marionette_harness')
+        dirs['abs_marionette_tests_dir'] = os.path.join(
+            dirs['abs_test_install_dir'], 'marionette', 'tests', 'testing',
+            'marionette', 'harness', 'marionette_harness', 'tests')
+
+        for key in dirs.keys():
+            if key not in abs_dirs:
+                abs_dirs[key] = dirs[key]
+        self.abs_dirs = abs_dirs
+        return self.abs_dirs
+
+    @PreScriptAction('create-virtualenv')
+    def _pre_create_virtualenv(self, action):
+        dirs = self.query_abs_dirs()
+        requirements = None
+        if self.test_suite == 'mochitest-media':
+            # mochitest-media is the only thing that needs this
+            requirements = os.path.join(dirs['abs_mochitest_dir'],
+                                        'websocketprocessbridge',
+                                        'websocketprocessbridge_requirements.txt')
+        elif self.test_suite == 'marionette':
+            requirements = os.path.join(dirs['abs_test_install_dir'],
+                                        'config', 'marionette_requirements.txt')
+        if requirements:
+            self.register_virtualenv_module(requirements=[requirements],
+                                            two_pass=True)
+
+    def _retry(self, max_attempts, interval, func, description, max_time=0):
+        '''
+        Execute func until it returns True, up to max_attempts times, waiting for
+        interval seconds between each attempt. description is logged on each attempt.
+        If max_time is specified, no further attempts will be made once max_time
+        seconds have elapsed; this provides some protection for the case where
+        the run-time for func is long or highly variable.
+        '''
+        status = False
+        attempts = 0
+        if max_time > 0:
+            end_time = datetime.datetime.now() + datetime.timedelta(seconds=max_time)
+        else:
+            end_time = None
+        while attempts < max_attempts and not status:
+            if (end_time is not None) and (datetime.datetime.now() > end_time):
+                self.info("Maximum retry run-time of %d seconds exceeded; "
+                          "remaining attempts abandoned" % max_time)
+                break
+            if attempts != 0:
+                self.info("Sleeping %d seconds" % interval)
+                time.sleep(interval)
+            attempts += 1
+            self.info(">> %s: Attempt #%d of %d" % (description, attempts, max_attempts))
+            status = func()
+        return status
+
+    def _run_with_timeout(self, timeout, cmd, quiet=False):
+        timeout_cmd = ['timeout', '%s' % timeout] + cmd
+        return self._run_proc(timeout_cmd, quiet=quiet)
+
+    def _run_proc(self, cmd, quiet=False):
+        self.info('Running %s' % subprocess.list2cmdline(cmd))
+        p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+        out, err = p.communicate()
+        if out and not quiet:
+            self.info('%s' % str(out.strip()))
+        if err and not quiet:
+            self.info('stderr: %s' % str(err.strip()))
+        return out, err
+
+    def _verify_adb(self):
+        self.info('Verifying adb connectivity')
+        self._run_with_timeout(180, [self.adb_path,
+                                     '-s',
+                                     self.device_serial,
+                                     'wait-for-device'])
+        return True
+
+    def _verify_adb_device(self):
+        out, _ = self._run_with_timeout(30, [self.adb_path, 'devices'])
+        if (self.device_serial in out) and ("device" in out):
+            return True
+        return False
+
+    def _is_boot_completed(self):
+        boot_cmd = [self.adb_path, '-s', self.device_serial,
+                    'shell', 'getprop', 'sys.boot_completed']
+        out, _ = self._run_with_timeout(30, boot_cmd)
+        if out.strip() == '1':
+            return True
+        return False
+
+    def _verify_device(self):
+        adb_ok = self._verify_adb()
+        if not adb_ok:
+            self.warning('Unable to communicate with adb')
+            return False
+        adb_device_ok = self._retry(4, 30, self._verify_adb_device,
+                                    "Verify device visible to adb")
+        if not adb_device_ok:
+            self.warning('Unable to communicate with device via adb')
+            return False
+        boot_ok = self._retry(30, 10, self._is_boot_completed, "Verify Android boot completed",
+                              max_time=330)
+        if not boot_ok:
+            self.warning('Unable to verify Android boot completion')
+            return False
+        return True
+
+    def _install_fennec_apk(self):
+        package = self._query_package_name()
+        if package:
+            cmd = [self.adb_path, '-s', self.device_serial,
+                   'uninstall', package]
+            out, err = self._run_with_timeout(300, cmd, True)
+        install_ok = False
+        if int(self.sdk_level) >= 23:
+            cmd = [self.adb_path, '-s', self.device_serial, 'install', '-r', '-g',
+                   self.installer_path]
+        else:
+            cmd = [self.adb_path, '-s', self.device_serial, 'install', '-r',
+                   self.installer_path]
+        out, err = self._run_with_timeout(300, cmd, True)
+        if 'Success' in out or 'Success' in err:
+            install_ok = True
+        return install_ok
+
+    def _install_robocop_apk(self):
+        install_ok = False
+        if int(self.sdk_level) >= 23:
+            cmd = [self.adb_path, '-s', self.device_serial, 'install', '-r', '-g',
+                   self.robocop_path]
+        else:
+            cmd = [self.adb_path, '-s', self.device_serial, 'install', '-r',
+                   self.robocop_path]
+        out, err = self._run_with_timeout(300, cmd, True)
+        if 'Success' in out or 'Success' in err:
+            install_ok = True
+        return install_ok
+
+    def _dump_host_state(self):
+        self._run_proc(['ps', '-ef'])
+        self._run_proc(['netstat', '-a', '-p', '-n', '-t', '-u'])
+
+    def _kill_processes(self, process_name):
+        p = subprocess.Popen(['ps', '-A'], stdout=subprocess.PIPE)
+        out, err = p.communicate()
+        self.info("Let's kill every process called %s" % process_name)
+        for line in out.splitlines():
+            if process_name in line:
+                pid = int(line.split(None, 1)[0])
+                self.info("Killing pid %d." % pid)
+                os.kill(pid, signal.SIGKILL)
+
+    def _restart_adbd(self):
+        self._run_with_timeout(30, [self.adb_path, 'kill-server'])
+        self._run_with_timeout(30, [self.adb_path, 'root'])
+
+    def _screenshot(self, prefix):
+        """
+           Save a screenshot of the entire screen to the blob upload directory.
+        """
+        dirs = self.query_abs_dirs()
+        utility = os.path.join(self.xre_path, "screentopng")
+        if not os.path.exists(utility):
+            self.warning("Unable to take screenshot: %s does not exist" % utility)
+            return
+        try:
+            tmpfd, filename = tempfile.mkstemp(prefix=prefix, suffix='.png',
+                                               dir=dirs['abs_blob_upload_dir'])
+            os.close(tmpfd)
+            self.info("Taking screenshot with %s; saving to %s" % (utility, filename))
+            subprocess.call([utility, filename], env=self.query_env())
+        except OSError, err:
+            self.warning("Failed to take screenshot: %s" % err.strerror)
+
+    def _query_package_name(self):
+        if self.app_name is None:
+            # For convenience, assume geckoview.test/geckoview_example when install
+            # target looks like geckoview.
+            if 'androidTest' in self.installer_path:
+                self.app_name = 'org.mozilla.geckoview.test'
+            elif 'geckoview' in self.installer_path:
+                self.app_name = 'org.mozilla.geckoview_example'
+        if self.app_name is None:
+            # Find appname from package-name.txt - assumes download-and-extract
+            # has completed successfully.
+            # The app/package name will typically be org.mozilla.fennec,
+            # but org.mozilla.firefox for release builds, and there may be
+            # other variations. 'aapt dump badging <apk>' could be used as an
+            # alternative to package-name.txt, but introduces a dependency
+            # on aapt, found currently in the Android SDK build-tools component.
+            apk_dir = self.abs_dirs['abs_work_dir']
+            self.apk_path = os.path.join(apk_dir, self.installer_path)
+            unzip = self.query_exe("unzip")
+            package_path = os.path.join(apk_dir, 'package-name.txt')
+            unzip_cmd = [unzip, '-q', '-o',  self.apk_path]
+            self.run_command(unzip_cmd, cwd=apk_dir, halt_on_failure=True)
+            self.app_name = str(self.read_from_file(package_path, verbose=True)).rstrip()
+        return self.app_name
+
+    def preflight_install(self):
+        # in the base class, this checks for mozinstall, but we don't use it
+        pass
+
+    def _build_command(self):
+        c = self.config
+        dirs = self.query_abs_dirs()
+
+        if self.test_suite not in self.config["suite_definitions"]:
+            self.fatal("Key '%s' not defined in the config!" % self.test_suite)
+
+        cmd = [
+            self.query_python_path('python'),
+            '-u',
+            os.path.join(
+                self._query_tests_dir(),
+                self.config["suite_definitions"][self.test_suite]["run_filename"]
+            ),
+        ]
+
+        raw_log_file = os.path.join(dirs['abs_blob_upload_dir'],
+                                    '%s_raw.log' % self.test_suite)
+
+        error_summary_file = os.path.join(dirs['abs_blob_upload_dir'],
+                                          '%s_errorsummary.log' % self.test_suite)
+        str_format_values = {
+            'device_serial': self.device_serial,
+            'remote_webserver': c['remote_webserver'],
+            'xre_path': self.xre_path,
+            'utility_path': self.xre_path,
+            'http_port': '8854',  # starting http port  to use for the mochitest server
+            'ssl_port': '4454',  # starting ssl port to use for the server
+            'certs_path': os.path.join(dirs['abs_work_dir'], 'tests/certs'),
+            # TestingMixin._download_and_extract_symbols() will set
+            # self.symbols_path when downloading/extracting.
+            'symbols_path': self.symbols_path,
+            'modules_dir': dirs['abs_modules_dir'],
+            'installer_path': self.installer_path,
+            'raw_log_file': raw_log_file,
+            'log_tbpl_level': self.log_tbpl_level,
+            'log_raw_level': self.log_raw_level,
+            'error_summary_file': error_summary_file,
+            # marionette options
+            'address': c.get('marionette_address') % {'device_ip': self.device_ip},
+            'gecko_log': os.path.join(dirs["abs_blob_upload_dir"], 'gecko.log'),
+            'test_manifest': os.path.join(
+                dirs['abs_marionette_tests_dir'],
+                self.config.get('marionette_test_manifest', '')
+            ),
+        }
+
+        user_paths = os.environ.get('MOZHARNESS_TEST_PATHS')
+        for option in self.config["suite_definitions"][self.test_suite]["options"]:
+            opt = option.split('=')[0]
+            # override configured chunk options with script args, if specified
+            if opt in ('--this-chunk', '--total-chunks'):
+                if user_paths or getattr(self, opt.replace('-', '_').strip('_'), None) is not None:
+                    continue
+
+            if '%(app)' in option:
+                # only query package name if requested
+                cmd.extend([option % {'app': self._query_package_name()}])
+            else:
+                cmd.extend([option % str_format_values])
+
+        if user_paths:
+            cmd.extend(user_paths.split(':'))
+        elif not self.verify_enabled:
+            if self.this_chunk is not None:
+                cmd.extend(['--this-chunk', self.this_chunk])
+            if self.total_chunks is not None:
+                cmd.extend(['--total-chunks', self.total_chunks])
+
+        try_options, try_tests = self.try_args(self.test_suite)
+        cmd.extend(try_options)
+        if not self.verify_enabled and not self.per_test_coverage:
+            cmd.extend(self.query_tests_args(
+                self.config["suite_definitions"][self.test_suite].get("tests"),
+                None,
+                try_tests))
+
+        return cmd
+
+    def _get_repo_url(self, path):
+        """
+           Return a url for a file (typically a tooltool manifest) in this hg repo
+           and using this revision (or mozilla-central/default if repo/rev cannot
+           be determined).
+
+           :param path specifies the directory path to the file of interest.
+        """
+        if 'GECKO_HEAD_REPOSITORY' in os.environ and 'GECKO_HEAD_REV' in os.environ:
+            # probably taskcluster
+            repo = os.environ['GECKO_HEAD_REPOSITORY']
+            revision = os.environ['GECKO_HEAD_REV']
+        else:
+            # something unexpected!
+            repo = 'https://hg.mozilla.org/mozilla-central'
+            revision = 'default'
+            self.warning('Unable to find repo/revision for manifest; '
+                         'using mozilla-central/default')
+        url = '%s/raw-file/%s/%s' % (
+            repo,
+            revision,
+            path)
+        return url
+
+    def _tooltool_fetch(self, url, dir):
+        c = self.config
+
+        manifest_path = self.download_file(
+            url,
+            file_name='releng.manifest',
+            parent_dir=dir
+        )
+
+        if not os.path.exists(manifest_path):
+            self.fatal("Could not retrieve manifest needed to retrieve "
+                       "artifacts from %s" % manifest_path)
+
+        self.tooltool_fetch(manifest_path,
+                            output_dir=dir,
+                            cache=c.get("tooltool_cache", None))
+
+    ##########################################
+    # Actions for AndroidHardwareTest        #
+    ##########################################
+    def _dump_perf_info(self):
+        '''
+        Dump some host and device performance-related information
+        to an artifact file, to help understand why jobs run slowly
+        sometimes. This is hopefully a temporary diagnostic.
+        See bug 1321605.
+        '''
+        dir = self.query_abs_dirs()['abs_blob_upload_dir']
+        perf_path = os.path.join(dir, "android-performance.log")
+        with open(perf_path, "w") as f:
+
+            f.write('\n\nHost /proc/cpuinfo:\n')
+            out, _ = self._run_proc(['cat', '/proc/cpuinfo'], quiet=True)
+            f.write(out)
+
+            f.write('\n\nHost /proc/meminfo:\n')
+            out, _ = self._run_proc(['cat', '/proc/meminfo'], quiet=True)
+            f.write(out)
+
+            f.write('\n\nHost process list:\n')
+            out, _ = self._run_proc(['ps', '-ef'], quiet=True)
+            f.write(out)
+
+            f.write('\n\nDevice /proc/cpuinfo:\n')
+            cmd = [self.adb_path, '-s', self.device_serial,
+                   'shell', 'cat', '/proc/cpuinfo']
+            out, _ = self._run_with_timeout(30, cmd, quiet=True)
+            f.write(out)
+            cpuinfo = out
+
+            f.write('\n\nDevice /proc/meminfo:\n')
+            cmd = [self.adb_path, '-s', self.device_serial,
+                   'shell', 'cat', '/proc/meminfo']
+            out, _ = self._run_with_timeout(30, cmd, quiet=True)
+            f.write(out)
+
+            f.write('\n\nDevice process list:\n')
+            cmd = [self.adb_path, '-s', self.device_serial,
+                   'shell', 'ps']
+            out, _ = self._run_with_timeout(30, cmd, quiet=True)
+            f.write(out)
+
+        for line in cpuinfo.split('\n'):
+            m = re.match("BogoMIPS.*: (\d*)", line)
+            if m:
+                bogomips = int(m.group(1))
+                self.info("Found Android bogomips: %d" % bogomips)
+                break
+
+    def verify_device(self):
+        '''
+        Check to see if the device can be contacted via adb.
+        '''
+        self.mkdir_p(self.query_abs_dirs()['abs_blob_upload_dir'])
+        self._dump_perf_info()
+        # Start logcat for the device. The adb process runs until the
+        # corresponding device is stopped. Output is written directly to
+        # the blobber upload directory so that it is uploaded automatically
+        # at the end of the job.
+        logcat_filename = 'logcat-%s.log' % self.device_serial
+        logcat_path = os.path.join(self.abs_dirs['abs_blob_upload_dir'],
+                                   logcat_filename)
+        self.logcat_file = open(logcat_path, 'w')
+        logcat_cmd = [self.adb_path, '-s', self.device_serial, 'logcat', '-v',
+                      'threadtime', 'Trace:S', 'StrictMode:S',
+                      'ExchangeService:S']
+        self.info(' '.join(logcat_cmd))
+        self.logcat_proc = subprocess.Popen(logcat_cmd, stdout=self.logcat_file,
+                                            stdin=subprocess.PIPE)
+        # Get a post-boot device process list for diagnostics
+        ps_cmd = [self.adb_path, '-s', self.device_serial, 'shell', 'ps']
+        self._run_with_timeout(30, ps_cmd)
+
+    def download_and_extract(self):
+        """
+        Download and extract fennec APK, tests.zip, host utils, and robocop (if required).
+        """
+        super(AndroidHardwareTest, self).download_and_extract(
+            suite_categories=self._query_suite_categories())
+        dirs = self.query_abs_dirs()
+        if self.test_suite and self.test_suite.startswith('robocop'):
+            robocop_url = self.installer_url[:self.installer_url.rfind('/')] + '/robocop.apk'
+            self.info("Downloading robocop...")
+            self.download_file(robocop_url, 'robocop.apk', dirs['abs_work_dir'], error_level=FATAL)
+        self.rmtree(dirs['abs_xre_dir'])
+        self.mkdir_p(dirs['abs_xre_dir'])
+        if self.config["hostutils_manifest_path"]:
+            url = self._get_repo_url(self.config["hostutils_manifest_path"])
+            self._tooltool_fetch(url, dirs['abs_xre_dir'])
+            for p in glob.glob(os.path.join(dirs['abs_xre_dir'], 'host-utils-*')):
+                if os.path.isdir(p) and os.path.isfile(os.path.join(p, 'xpcshell')):
+                    self.xre_path = p
+            if not self.xre_path:
+                self.fatal("xre path not found in %s" % dirs['abs_xre_dir'])
+        else:
+            self.fatal("configure hostutils_manifest_path!")
+
+    def install(self):
+        """
+        Install APKs on the device.
+        """
+        install_needed = (not self.test_suite) or \
+            self.config["suite_definitions"][self.test_suite].get("install")
+        if install_needed is False:
+            self.info("Skipping apk installation for %s" % self.test_suite)
+            return
+
+        assert self.installer_path is not None, \
+            "Either add installer_path to the config or use --installer-path."
+
+        cmd = [self.adb_path, '-s', self.device_serial, 'shell',
+               'getprop', 'ro.build.version.sdk']
+        self.sdk_level, _ = self._run_with_timeout(30, cmd)
+
+        # Install Fennec
+        install_ok = self._retry(3, 30, self._install_fennec_apk, "Install app APK")
+        if not install_ok:
+            self.fatal('INFRA-ERROR: Failed to install %s on %s' %
+                       (self.installer_path, self.device_name),
+                       EXIT_STATUS_DICT[TBPL_RETRY])
+
+        # Install Robocop if required
+        if self.test_suite and self.test_suite.startswith('robocop'):
+            install_ok = self._retry(3, 30, self._install_robocop_apk, "Install Robocop APK")
+            if not install_ok:
+                self.fatal('INFRA-ERROR: Failed to install %s on %s' %
+                           (self.robocop_path, self.device_name),
+                           EXIT_STATUS_DICT[TBPL_RETRY])
+
+        self.info("Finished installing apps for %s" % self.device_name)
+
+    def _query_suites(self):
+        if self.test_suite:
+            return [(self.test_suite, self.test_suite)]
+        # per-test mode: determine test suites to run
+        all = [('mochitest', {'plain': 'mochitest',
+                              'chrome': 'mochitest-chrome',
+                              'plain-clipboard': 'mochitest-plain-clipboard',
+                              'plain-gpu': 'mochitest-plain-gpu'}),
+               ('reftest', {'reftest': 'reftest', 'crashtest': 'crashtest'}),
+               ('xpcshell', {'xpcshell': 'xpcshell'})]
+        suites = []
+        for (category, all_suites) in all:
+            cat_suites = self.query_per_test_category_suites(category, all_suites)
+            for k in cat_suites.keys():
+                suites.append((k, cat_suites[k]))
+        return suites
+
+    def _query_suite_categories(self):
+        if self.test_suite:
+            categories = [self.test_suite]
+        else:
+            # per-test mode
+            categories = ['mochitest', 'reftest', 'xpcshell']
+        return categories
+
+    def run_tests(self):
+        """
+        Run the tests
+        """
+        self.start_time = datetime.datetime.now()
+        max_per_test_time = datetime.timedelta(minutes=60)
+
+        per_test_args = []
+        suites = self._query_suites()
+        minidump = self.query_minidump_stackwalk()
+        for (per_test_suite, suite) in suites:
+            self.test_suite = suite
+
+            cmd = self._build_command()
+
+            try:
+                cwd = self._query_tests_dir()
+            except Exception:
+                self.fatal("Don't know how to run --test-suite '%s'!" % self.test_suite)
+            env = self.query_env()
+            if minidump:
+                env['MINIDUMP_STACKWALK'] = minidump
+            env['MOZ_UPLOAD_DIR'] = self.query_abs_dirs()['abs_blob_upload_dir']
+            env['MINIDUMP_SAVE_PATH'] = self.query_abs_dirs()['abs_blob_upload_dir']
+            env['RUST_BACKTRACE'] = 'full'
+
+            summary = None
+            for per_test_args in self.query_args(per_test_suite):
+                if (datetime.datetime.now() - self.start_time) > max_per_test_time:
+                    # Running tests has run out of time. That is okay! Stop running
+                    # them so that a task timeout is not triggered, and so that
+                    # (partial) results are made available in a timely manner.
+                    self.info("TinderboxPrint: Running tests took too long: "
+                              "Not all tests were executed.<br/>")
+                    # Signal per-test time exceeded, to break out of suites and
+                    # suite categories loops also.
+                    return False
+
+                final_cmd = copy.copy(cmd)
+                if len(per_test_args) > 0:
+                    # in per-test mode, remove any chunk arguments from command
+                    for arg in final_cmd:
+                        if 'total-chunk' in arg or 'this-chunk' in arg:
+                            final_cmd.remove(arg)
+                final_cmd.extend(per_test_args)
+
+                self.info("Running on %s the command %s" % (self.device_name,
+                          subprocess.list2cmdline(final_cmd)))
+                self.info("##### %s log begins" % self.test_suite)
+
+                suite_category = self.test_suite
+                parser = self.get_test_output_parser(
+                    suite_category,
+                    config=self.config,
+                    log_obj=self.log_obj,
+                    error_list=[])
+                self.run_command(final_cmd, cwd=cwd, env=env, output_parser=parser)
+                tbpl_status, log_level, summary = parser.evaluate_parser(0, summary)
+                parser.append_tinderboxprint_line(self.test_suite)
+
+                self.info("##### %s log ends" % self.test_suite)
+
+                if len(per_test_args) > 0:
+                    self.record_status(tbpl_status, level=log_level)
+                    self.log_per_test_status(per_test_args[-1], tbpl_status, log_level)
+                else:
+                    self.record_status(tbpl_status, level=log_level)
+                    self.log("The %s suite: %s ran with return status: %s" %
+                             (suite_category, suite, tbpl_status), level=log_level)
+
+    @PostScriptAction('run-tests')
+    def stop_device(self, action, success=None):
+        '''
+        Report device health.
+        '''
+        if self.logcat_proc:
+            self.info("Killing logcat pid %d." % self.logcat_proc.pid)
+            self.logcat_proc.kill()
+            self.logcat_file.close()
+
+
+if __name__ == '__main__':
+    hardwareTest = AndroidHardwareTest()
+    hardwareTest.run_and_exit()