Bug 1199377 - Fetch host-utils for Android tests when needed; r=jmaher
authorGeoff Brown <gbrown@mozilla.com>
Wed, 16 Sep 2015 16:58:52 -0600
changeset 295481 1dbf2804716827142f5ffb7a0b505b4921d07eb0
parent 295480 78c35eb90780bd9169d9c5417410fc5883970f76
child 295482 3ca0821a9b0569a1ceb00330dd1a528f65ce85bf
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1199377
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1199377 - Fetch host-utils for Android tests when needed; r=jmaher
testing/mochitest/mach_commands.py
testing/mozbase/mozrunner/mozrunner/devices/android_device.py
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -421,17 +421,17 @@ def setup_argument_parser():
         from mochitest_options import MochitestArgumentParser
 
     if conditions.is_android(build_obj):
         # On Android, check for a connected device (and offer to start an
         # emulator if appropriate) before running tests. This check must
         # be done in this admittedly awkward place because
         # MochitestArgumentParser initialization fails if no device is found.
         from mozrunner.devices.android_device import verify_android_device
-        verify_android_device(build_obj, install=True)
+        verify_android_device(build_obj, install=True, xre=True)
 
     return MochitestArgumentParser()
 
 
 # condition filters
 
 def is_buildapp_in(*apps):
     def is_buildapp_supported(cls):
--- a/testing/mozbase/mozrunner/mozrunner/devices/android_device.py
+++ b/testing/mozbase/mozrunner/mozrunner/devices/android_device.py
@@ -1,29 +1,38 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import glob
 import os
+import platform
 import psutil
 import re
 import shutil
 import signal
+import sys
 import telnetlib
 import time
 import urllib2
 from distutils.spawn import find_executable
 
 from mozdevice import DeviceManagerADB, DMError
 from mozprocess import ProcessHandler
 
 EMULATOR_HOME_DIR = os.path.join(os.path.expanduser('~'), '.mozbuild', 'android-device')
 
 TOOLTOOL_URL = 'https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py'
 
+TRY_URL = 'https://hg.mozilla.org/try/raw-file/default'
+
+MANIFEST_URL = '%s/testing/config/tooltool-manifests' % TRY_URL
+
+verbose_logging = False
+
 class AvdInfo(object):
     """
        Simple class to contain an AVD description.
     """
     def __init__(self, description, name, tooltool_manifest, extra_args,
                  port, uses_sut, sut_port, sut_port2):
         self.description = description
         self.name = name
@@ -65,39 +74,42 @@ AVD_DICT = {
                    ['-debug',
                     'init,console,gles,memcheck,adbserver,adbclient,adb,avd_config,socket',
                     '-qemu', '-m', '1024', '-enable-kvm'],
                    5554,
                    True,
                    20701, 20700)
 }
 
-def verify_android_device(build_obj, install=False):
+def verify_android_device(build_obj, install=False, xre=False):
     """
        Determine if any Android device is connected via adb.
        If no device is found, prompt to start an emulator.
        If a device is found or an emulator started and 'install' is
        specified, also check whether Firefox is installed on the
        device; if not, prompt to install Firefox.
+       If 'xre' is specified, also check with MOZ_HOST_BIN is set
+       to a valid xre/host-utils directory; if not, prompt to set
+       one up.
        Returns True if the emulator was started or another device was
        already connected.
     """
     device_verified = False
     emulator = AndroidEmulator('*', substs=build_obj.substs)
     devices = emulator.dm.devices()
     if len(devices) > 0:
         device_verified = True
     elif emulator.is_available():
         response = raw_input(
             "No Android devices connected. Start an emulator? (Y/n) ").strip()
         if response.lower().startswith('y') or response == '':
             if not emulator.check_avd():
-                print("Fetching AVD. This may take a while...")
+                _log_info("Fetching AVD. This may take a while...")
                 emulator.update_avd()
-            print("Starting emulator running %s..." %
+            _log_info("Starting emulator running %s..." %
                 emulator.get_avd_description())
             emulator.start()
             emulator.wait_for_start()
             device_verified = True
 
     if device_verified and install:
         # Determine if Firefox is installed on the device; if not,
         # prompt to install. This feature allows a test command to
@@ -113,19 +125,68 @@ def verify_android_device(build_obj, ins
         #  - installation may take a couple of minutes.
         installed = emulator.dm.shellCheckOutput(['pm', 'list',
             'packages', 'org.mozilla.'])
         if not 'fennec' in installed and not 'firefox' in installed:
             response = raw_input(
                 "It looks like Firefox is not installed on this device.\n"
                 "Install Firefox? (Y/n) ").strip()
             if response.lower().startswith('y') or response == '':
-                print("Installing Firefox. This may take a while...")
+                _log_info("Installing Firefox. This may take a while...")
                 build_obj._run_make(directory=".", target='install',
                     ensure_exit_code=False)
+
+    if device_verified and xre:
+        # Check whether MOZ_HOST_BIN has been set to a valid xre; if not,
+        # prompt to install one.
+        xre_path = os.environ.get('MOZ_HOST_BIN')
+        err = None
+        if not xre_path:
+            err = 'environment variable MOZ_HOST_BIN is not set to a directory containing host xpcshell'
+        elif not os.path.isdir(xre_path):
+            err = '$MOZ_HOST_BIN does not specify a directory'
+        elif not os.path.isfile(os.path.join(xre_path, 'xpcshell')):
+            err = '$MOZ_HOST_BIN/xpcshell does not exist'
+        if err:
+            xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR, 'host-utils*'))
+            for path in xre_path:
+                if os.path.isdir(path) and os.path.isfile(os.path.join(path, 'xpcshell')):
+                    os.environ['MOZ_HOST_BIN'] = path
+                    err = None
+                    break
+        if err:
+            _log_info("Host utilities not found: %s" % err)
+            response = raw_input(
+                "Download and setup your host utilities? (Y/n) ").strip()
+            if response.lower().startswith('y') or response == '':
+                _log_info("Installing host utilities. This may take a while...")
+                _download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
+                if 'darwin' in str(sys.platform).lower():
+                    plat = 'macosx64'
+                elif 'linux' in str(sys.platform).lower():
+                    if '64' in platform.architecture()[0]:
+                        plat = 'linux64'
+                    else:
+                        plat = 'linux32'
+                else:
+                    plat = None
+                    _log_warning("Unable to install host utilities -- your platform is not supported!")
+                if plat:
+                    url = '%s/%s/hostutils.manifest' % (MANIFEST_URL, plat)
+                    _download_file(url, 'releng.manifest', EMULATOR_HOME_DIR)
+                    _tooltool_fetch()
+                    xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR, 'host-utils*'))
+                    for path in xre_path:
+                        if os.path.isdir(path) and os.path.isfile(os.path.join(path, 'xpcshell')):
+                            os.environ['MOZ_HOST_BIN'] = path
+                            err = None
+                            break
+                    if err:
+                        _log_warning("Unable to install host utilities.")
+
     return device_verified
 
 
 class AndroidEmulator(object):
 
     """
         Support running the Android emulator with an AVD from Mozilla
         test automation.
@@ -137,28 +198,29 @@ class AndroidEmulator(object):
                     warn("this may take a while...")
                     emulator.update_avd()
                 emulator.start()
                 emulator.wait_for_start()
                 emulator.wait()
     """
 
     def __init__(self, avd_type='4.3', verbose=False, substs=None):
+        global verbose_logging
         self.emulator_log = None
         self.emulator_path = 'emulator'
-        self.verbose = verbose
+        verbose_logging = verbose
         self.substs = substs
         self.avd_type = self._get_avd_type(avd_type)
         self.avd_info = AVD_DICT[self.avd_type]
         adb_path = self._find_sdk_exe('adb', False)
         if not adb_path:
             adb_path = 'adb'
         self.dm = DeviceManagerADB(autoconnect=False, adbPath=adb_path, retryLimit=1)
         self.dm.default_timeout = 10
-        self._log_debug("Emulator created with type %s" % self.avd_type)
+        _log_debug("Emulator created with type %s" % self.avd_type)
 
     def __del__(self):
         if self.emulator_log:
             self.emulator_log.close()
 
     def is_running(self):
         """
            Returns True if the Android emulator is running.
@@ -191,17 +253,17 @@ class AndroidEmulator(object):
 
            Returns True if the AVD is installed.
         """
         avd = os.path.join(
             EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.avd')
         if force and os.path.exists(avd):
             shutil.rmtree(avd)
         if os.path.exists(avd):
-            self._log_debug("AVD found at %s" % avd)
+            _log_debug("AVD found at %s" % avd)
             return True
         return False
 
     def update_avd(self, force=False):
         """
            If required, update the AVD via tooltool.
 
            If the AVD directory is not found, or "force" is requested,
@@ -210,83 +272,84 @@ class AndroidEmulator(object):
            required archive (unless already present in the local tooltool
            cache) and install the AVD.
         """
         avd = os.path.join(
             EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.avd')
         if force and os.path.exists(avd):
             shutil.rmtree(avd)
         if not os.path.exists(avd):
-            self._fetch_tooltool()
-            self._fetch_tooltool_manifest()
-            self._tooltool_fetch()
+            _download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
+            url = '%s/%s' % (TRY_URL, self.avd_info.tooltool_manifest)
+            _download_file(url, 'releng.manifest', EMULATOR_HOME_DIR)
+            _tooltool_fetch()
             self._update_avd_paths()
 
     def start(self):
         """
            Launch the emulator.
         """
         def outputHandler(line):
             self.emulator_log.write("<%s>\n" % line)
         env = os.environ
         env['ANDROID_AVD_HOME'] = os.path.join(EMULATOR_HOME_DIR, "avd")
         command = [self.emulator_path, "-avd",
                    self.avd_info.name, "-port", "5554"]
         if self.avd_info.extra_args:
             command += self.avd_info.extra_args
         log_path = os.path.join(EMULATOR_HOME_DIR, 'emulator.log')
         self.emulator_log = open(log_path, 'w')
-        self._log_debug("Starting the emulator with this command: %s" %
+        _log_debug("Starting the emulator with this command: %s" %
                         ' '.join(command))
-        self._log_debug("Emulator output will be written to '%s'" %
+        _log_debug("Emulator output will be written to '%s'" %
                         log_path)
         self.proc = ProcessHandler(
             command, storeOutput=False, processOutputLine=outputHandler,
             env=env)
         self.proc.run()
-        self._log_debug("Emulator started with pid %d" %
+        _log_debug("Emulator started with pid %d" %
                         int(self.proc.proc.pid))
 
     def wait_for_start(self):
         """
            Verify that the emulator is running, the emulator device is visible
            to adb, and Android has booted.
         """
         if not self.proc:
-            self._log_warning("Emulator not started!")
+            _log_warning("Emulator not started!")
             return False
         if self.proc.proc.poll() is not None:
-            self._log_warning("Emulator has already completed!")
+            _log_warning("Emulator has already completed!")
             return False
-        self._log_debug("Waiting for device status...")
+        _log_debug("Waiting for device status...")
         while(('emulator-5554', 'device') not in self.dm.devices()):
             time.sleep(10)
             if self.proc.proc.poll() is not None:
-                self._log_warning("Emulator has already completed!")
+                _log_warning("Emulator has already completed!")
                 return False
-        self._log_debug("Device status verified.")
+        _log_debug("Device status verified.")
 
-        self._log_debug("Checking that Android has booted...")
+        _log_debug("Checking that Android has booted...")
         complete = False
         while(not complete):
             output = ''
             try:
                 output = self.dm.shellCheckOutput(
                     ['getprop', 'sys.boot_completed'], timeout=5)
             except DMError:
                 # adb not yet responding...keep trying
                 pass
             if output.strip() == '1':
                 complete = True
             else:
                 time.sleep(10)
                 if self.proc.proc.poll() is not None:
-                    self._log_warning("Emulator has already completed!")
+                    _log_warning("Emulator has already completed!")
                     return False
-        self._log_debug("Android boot status verified.")
+        _log_debug("Android boot status verified.")
 
         if not self._verify_emulator():
             return False
         if self.avd_info.uses_sut:
             if not self._verify_sut():
                 return False
         return True
 
@@ -308,88 +371,45 @@ class AndroidEmulator(object):
         self.proc.kill(signal.SIGTERM)
 
     def get_avd_description(self):
         """
            Return the human-friendly description of this AVD.
         """
         return self.avd_info.description
 
-    def _log_debug(self, text):
-        if self.verbose:
-            print "DEBUG: %s" % text
-
-    def _log_warning(self, text):
-        print "WARNING: %s" % text
-
-    def _fetch_tooltool(self):
-        self._download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
-
-    def _fetch_tooltool_manifest(self):
-        url = 'https://hg.mozilla.org/%s/raw-file/%s/%s' % (
-            "try", "default", self.avd_info.tooltool_manifest)
-        self._download_file(url, 'releng.manifest', EMULATOR_HOME_DIR)
-
-    def _tooltool_fetch(self):
-        def outputHandler(line):
-            self._log_debug(line)
-        command = ['python', 'tooltool.py', 'fetch', '-m', 'releng.manifest']
-        proc = ProcessHandler(
-            command, processOutputLine=outputHandler, storeOutput=False,
-            cwd=EMULATOR_HOME_DIR)
-        proc.run()
-        try:
-            proc.wait()
-        except:
-            if proc.poll() is None:
-                proc.kill(signal.SIGTERM)
-
     def _update_avd_paths(self):
         avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
         ini_file = os.path.join(avd_path, "test-1.ini")
         ini_file_new = os.path.join(avd_path, self.avd_info.name + ".ini")
         os.rename(ini_file, ini_file_new)
         avd_dir = os.path.join(avd_path, "test-1.avd")
         avd_dir_new = os.path.join(avd_path, self.avd_info.name + ".avd")
         os.rename(avd_dir, avd_dir_new)
         self._replace_ini_contents(ini_file_new)
 
-    def _download_file(self, url, filename, path):
-        f = urllib2.urlopen(url)
-        if not os.path.isdir(path):
-            try:
-                os.makedirs(path)
-            except Exception, e:
-                self._log_warning(str(e))
-                return False
-        local_file = open(os.path.join(path, filename), 'wb')
-        local_file.write(f.read())
-        local_file.close()
-        self._log_debug("Downloaded %s to %s/%s" % (url, path, filename))
-        return True
-
     def _replace_ini_contents(self, path):
         with open(path, "r") as f:
             lines = f.readlines()
         with open(path, "w") as f:
             for line in lines:
                 if line.startswith('path='):
                     avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
                     f.write('path=%s/%s.avd\n' %
                             (avd_path, self.avd_info.name))
                 elif line.startswith('path.rel='):
                     f.write('path.rel=avd/%s.avd\n' % self.avd_info.name)
                 else:
                     f.write(line)
 
     def _telnet_cmd(self, telnet, command):
-        self._log_debug(">>> " + command)
+        _log_debug(">>> " + command)
         telnet.write('%s\n' % command)
         result = telnet.read_until('OK', 10)
-        self._log_debug("<<< " + result)
+        _log_debug("<<< " + result)
         return result
 
     def _verify_emulator(self):
         telnet_ok = False
         tn = None
         while(not telnet_ok):
             try:
                 tn = telnetlib.Telnet('localhost', self.avd_info.port, 10)
@@ -406,54 +426,54 @@ class AndroidEmulator(object):
                              str(self.avd_info.sut_port2))
                         self._telnet_cmd(tn, cmd)
                     self._telnet_cmd(tn, 'redir list')
                     self._telnet_cmd(tn, 'network status')
                     tn.write('quit\n')
                     tn.read_all()
                     telnet_ok = True
                 else:
-                    self._log_warning("Unable to connect to port %d" % port)
+                    _log_warning("Unable to connect to port %d" % port)
             except:
-                self._log_warning("Trying again after unexpected exception")
+                _log_warning("Trying again after unexpected exception")
             finally:
                 if tn is not None:
                     tn.close()
             if not telnet_ok:
                 time.sleep(10)
                 if self.proc.proc.poll() is not None:
-                    self._log_warning("Emulator has already completed!")
+                    _log_warning("Emulator has already completed!")
                     return False
         return telnet_ok
 
     def _verify_sut(self):
         sut_ok = False
         while(not sut_ok):
             try:
                 tn = telnetlib.Telnet('localhost', self.avd_info.sut_port, 10)
                 if tn is not None:
-                    self._log_debug(
+                    _log_debug(
                         "Connected to port %d" % self.avd_info.sut_port)
                     res = tn.read_until('$>', 10)
                     if res.find('$>') == -1:
-                        self._log_debug("Unexpected SUT response: %s" % res)
+                        _log_debug("Unexpected SUT response: %s" % res)
                     else:
-                        self._log_debug("SUT response: %s" % res)
+                        _log_debug("SUT response: %s" % res)
                         sut_ok = True
                     tn.write('quit\n')
                     tn.read_all()
             except:
-                self._log_debug("Caught exception while verifying sutagent")
+                _log_debug("Caught exception while verifying sutagent")
             finally:
                 if tn is not None:
                     tn.close()
             if not sut_ok:
                 time.sleep(10)
                 if self.proc.proc.poll() is not None:
-                    self._log_warning("Emulator has already completed!")
+                    _log_warning("Emulator has already completed!")
                     return False
         return sut_ok
 
     def _get_avd_type(self, requested):
         if requested in AVD_DICT.keys():
             return requested
         if self.substs:
             if not self.substs['TARGET_CPU'].startswith('arm'):
@@ -475,51 +495,89 @@ class AndroidEmulator(object):
         # Can exe be found in the Android SDK?
         try:
             android_sdk_root = os.environ['ANDROID_SDK_ROOT']
             exe_path = os.path.join(
                 android_sdk_root, subdir, exe)
             if os.path.exists(exe_path):
                 found = True
             else:
-                self._log_debug(
+                _log_debug(
                     "Unable to find executable at %s" % exe_path)
         except KeyError:
-            self._log_debug("ANDROID_SDK_ROOT not set")
+            _log_debug("ANDROID_SDK_ROOT not set")
 
         if not found and self.substs:
             # Can exe be found in ANDROID_TOOLS/ANDROID_PLATFORM_TOOLS?
             try:
                 exe_path = os.path.join(
                     self.substs[var], exe)
                 if os.path.exists(exe_path):
                     found = True
                 else:
-                    self._log_debug(
+                    _log_debug(
                         "Unable to find executable at %s" % exe_path)
             except KeyError:
-                self._log_debug("%s not set" % var)
+                _log_debug("%s not set" % var)
 
         if not found:
             # Can exe be found in the default bootstrap location?
             mozbuild_path = os.environ.get('MOZBUILD_STATE_PATH',
                 os.path.expanduser(os.path.join('~', '.mozbuild')))
             exe_path = os.path.join(
                 mozbuild_path, 'android-sdk-linux', subdir, exe)
             if os.path.exists(exe_path):
                 found = True
             else:
-                self._log_debug(
+                _log_debug(
                     "Unable to find executable at %s" % exe_path)
 
         if not found:
             # Is exe on PATH?
             exe_path = find_executable(exe)
             if exe_path:
                 found = True
             else:
-                self._log_debug("Unable to find executable on PATH")
+                _log_debug("Unable to find executable on PATH")
 
         if found:
-            self._log_debug("%s found at %s" % (exe, exe_path))
+            _log_debug("%s found at %s" % (exe, exe_path))
         else:
             exe_path = None
         return exe_path
+
+def _log_debug(text):
+    if verbose_logging:
+        print "DEBUG: %s" % text
+
+def _log_warning(text):
+    print "WARNING: %s" % text
+
+def _log_info(text):
+    print "%s" % text
+
+def _download_file(url, filename, path):
+    f = urllib2.urlopen(url)
+    if not os.path.isdir(path):
+        try:
+            os.makedirs(path)
+        except Exception, e:
+            _log_warning(str(e))
+            return False
+    local_file = open(os.path.join(path, filename), 'wb')
+    local_file.write(f.read())
+    local_file.close()
+    _log_debug("Downloaded %s to %s/%s" % (url, path, filename))
+    return True
+
+def _tooltool_fetch():
+    def outputHandler(line):
+        _log_debug(line)
+    command = ['python', 'tooltool.py', 'fetch', '-o', '-m', 'releng.manifest']
+    proc = ProcessHandler(
+        command, processOutputLine=outputHandler, storeOutput=False,
+        cwd=EMULATOR_HOME_DIR)
+    proc.run()
+    try:
+        proc.wait()
+    except:
+        if proc.poll() is None:
+            proc.kill(signal.SIGTERM)