Bug 1199377 - Fetch host-utils for Android tests when needed; r=jmaher
authorGeoff Brown <gbrown@mozilla.com>
Wed, 16 Sep 2015 16:58:52 -0600
changeset 283555 1dbf2804716827142f5ffb7a0b505b4921d07eb0
parent 283554 78c35eb90780bd9169d9c5417410fc5883970f76
child 283556 3ca0821a9b0569a1ceb00330dd1a528f65ce85bf
push id8456
push userraliiev@mozilla.com
push dateMon, 21 Sep 2015 14:31:52 +0000
treeherdermozilla-aurora@7f2f0fb041b1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersjmaher
bugs1199377
milestone43.0a1
Bug 1199377 - Fetch host-utils for Android tests when needed; r=jmaher
testing/mochitest/mach_commands.py
testing/mozbase/mozrunner/mozrunner/devices/android_device.py
--- a/testing/mochitest/mach_commands.py
+++ b/testing/mochitest/mach_commands.py
@@ -421,17 +421,17 @@ def setup_argument_parser():
         from mochitest_options import MochitestArgumentParser
 
     if conditions.is_android(build_obj):
         # On Android, check for a connected device (and offer to start an
         # emulator if appropriate) before running tests. This check must
         # be done in this admittedly awkward place because
         # MochitestArgumentParser initialization fails if no device is found.
         from mozrunner.devices.android_device import verify_android_device
-        verify_android_device(build_obj, install=True)
+        verify_android_device(build_obj, install=True, xre=True)
 
     return MochitestArgumentParser()
 
 
 # condition filters
 
 def is_buildapp_in(*apps):
     def is_buildapp_supported(cls):
--- a/testing/mozbase/mozrunner/mozrunner/devices/android_device.py
+++ b/testing/mozbase/mozrunner/mozrunner/devices/android_device.py
@@ -1,29 +1,38 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import glob
 import os
+import platform
 import psutil
 import re
 import shutil
 import signal
+import sys
 import telnetlib
 import time
 import urllib2
 from distutils.spawn import find_executable
 
 from mozdevice import DeviceManagerADB, DMError
 from mozprocess import ProcessHandler
 
 EMULATOR_HOME_DIR = os.path.join(os.path.expanduser('~'), '.mozbuild', 'android-device')
 
 TOOLTOOL_URL = 'https://raw.githubusercontent.com/mozilla/build-tooltool/master/tooltool.py'
 
+TRY_URL = 'https://hg.mozilla.org/try/raw-file/default'
+
+MANIFEST_URL = '%s/testing/config/tooltool-manifests' % TRY_URL
+
+verbose_logging = False
+
 class AvdInfo(object):
     """
        Simple class to contain an AVD description.
     """
     def __init__(self, description, name, tooltool_manifest, extra_args,
                  port, uses_sut, sut_port, sut_port2):
         self.description = description
         self.name = name
@@ -65,39 +74,42 @@ AVD_DICT = {
                    ['-debug',
                     'init,console,gles,memcheck,adbserver,adbclient,adb,avd_config,socket',
                     '-qemu', '-m', '1024', '-enable-kvm'],
                    5554,
                    True,
                    20701, 20700)
 }
 
-def verify_android_device(build_obj, install=False):
+def verify_android_device(build_obj, install=False, xre=False):
     """
        Determine if any Android device is connected via adb.
        If no device is found, prompt to start an emulator.
        If a device is found or an emulator started and 'install' is
        specified, also check whether Firefox is installed on the
        device; if not, prompt to install Firefox.
+       If 'xre' is specified, also check with MOZ_HOST_BIN is set
+       to a valid xre/host-utils directory; if not, prompt to set
+       one up.
        Returns True if the emulator was started or another device was
        already connected.
     """
     device_verified = False
     emulator = AndroidEmulator('*', substs=build_obj.substs)
     devices = emulator.dm.devices()
     if len(devices) > 0:
         device_verified = True
     elif emulator.is_available():
         response = raw_input(
             "No Android devices connected. Start an emulator? (Y/n) ").strip()
         if response.lower().startswith('y') or response == '':
             if not emulator.check_avd():
-                print("Fetching AVD. This may take a while...")
+                _log_info("Fetching AVD. This may take a while...")
                 emulator.update_avd()
-            print("Starting emulator running %s..." %
+            _log_info("Starting emulator running %s..." %
                 emulator.get_avd_description())
             emulator.start()
             emulator.wait_for_start()
             device_verified = True
 
     if device_verified and install:
         # Determine if Firefox is installed on the device; if not,
         # prompt to install. This feature allows a test command to
@@ -113,19 +125,68 @@ def verify_android_device(build_obj, ins
         #  - installation may take a couple of minutes.
         installed = emulator.dm.shellCheckOutput(['pm', 'list',
             'packages', 'org.mozilla.'])
         if not 'fennec' in installed and not 'firefox' in installed:
             response = raw_input(
                 "It looks like Firefox is not installed on this device.\n"
                 "Install Firefox? (Y/n) ").strip()
             if response.lower().startswith('y') or response == '':
-                print("Installing Firefox. This may take a while...")
+                _log_info("Installing Firefox. This may take a while...")
                 build_obj._run_make(directory=".", target='install',
                     ensure_exit_code=False)
+
+    if device_verified and xre:
+        # Check whether MOZ_HOST_BIN has been set to a valid xre; if not,
+        # prompt to install one.
+        xre_path = os.environ.get('MOZ_HOST_BIN')
+        err = None
+        if not xre_path:
+            err = 'environment variable MOZ_HOST_BIN is not set to a directory containing host xpcshell'
+        elif not os.path.isdir(xre_path):
+            err = '$MOZ_HOST_BIN does not specify a directory'
+        elif not os.path.isfile(os.path.join(xre_path, 'xpcshell')):
+            err = '$MOZ_HOST_BIN/xpcshell does not exist'
+        if err:
+            xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR, 'host-utils*'))
+            for path in xre_path:
+                if os.path.isdir(path) and os.path.isfile(os.path.join(path, 'xpcshell')):
+                    os.environ['MOZ_HOST_BIN'] = path
+                    err = None
+                    break
+        if err:
+            _log_info("Host utilities not found: %s" % err)
+            response = raw_input(
+                "Download and setup your host utilities? (Y/n) ").strip()
+            if response.lower().startswith('y') or response == '':
+                _log_info("Installing host utilities. This may take a while...")
+                _download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
+                if 'darwin' in str(sys.platform).lower():
+                    plat = 'macosx64'
+                elif 'linux' in str(sys.platform).lower():
+                    if '64' in platform.architecture()[0]:
+                        plat = 'linux64'
+                    else:
+                        plat = 'linux32'
+                else:
+                    plat = None
+                    _log_warning("Unable to install host utilities -- your platform is not supported!")
+                if plat:
+                    url = '%s/%s/hostutils.manifest' % (MANIFEST_URL, plat)
+                    _download_file(url, 'releng.manifest', EMULATOR_HOME_DIR)
+                    _tooltool_fetch()
+                    xre_path = glob.glob(os.path.join(EMULATOR_HOME_DIR, 'host-utils*'))
+                    for path in xre_path:
+                        if os.path.isdir(path) and os.path.isfile(os.path.join(path, 'xpcshell')):
+                            os.environ['MOZ_HOST_BIN'] = path
+                            err = None
+                            break
+                    if err:
+                        _log_warning("Unable to install host utilities.")
+
     return device_verified
 
 
 class AndroidEmulator(object):
 
     """
         Support running the Android emulator with an AVD from Mozilla
         test automation.
@@ -137,28 +198,29 @@ class AndroidEmulator(object):
                     warn("this may take a while...")
                     emulator.update_avd()
                 emulator.start()
                 emulator.wait_for_start()
                 emulator.wait()
     """
 
     def __init__(self, avd_type='4.3', verbose=False, substs=None):
+        global verbose_logging
         self.emulator_log = None
         self.emulator_path = 'emulator'
-        self.verbose = verbose
+        verbose_logging = verbose
         self.substs = substs
         self.avd_type = self._get_avd_type(avd_type)
         self.avd_info = AVD_DICT[self.avd_type]
         adb_path = self._find_sdk_exe('adb', False)
         if not adb_path:
             adb_path = 'adb'
         self.dm = DeviceManagerADB(autoconnect=False, adbPath=adb_path, retryLimit=1)
         self.dm.default_timeout = 10
-        self._log_debug("Emulator created with type %s" % self.avd_type)
+        _log_debug("Emulator created with type %s" % self.avd_type)
 
     def __del__(self):
         if self.emulator_log:
             self.emulator_log.close()
 
     def is_running(self):
         """
            Returns True if the Android emulator is running.
@@ -191,17 +253,17 @@ class AndroidEmulator(object):
 
            Returns True if the AVD is installed.
         """
         avd = os.path.join(
             EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.avd')
         if force and os.path.exists(avd):
             shutil.rmtree(avd)
         if os.path.exists(avd):
-            self._log_debug("AVD found at %s" % avd)
+            _log_debug("AVD found at %s" % avd)
             return True
         return False
 
     def update_avd(self, force=False):
         """
            If required, update the AVD via tooltool.
 
            If the AVD directory is not found, or "force" is requested,
@@ -210,83 +272,84 @@ class AndroidEmulator(object):
            required archive (unless already present in the local tooltool
            cache) and install the AVD.
         """
         avd = os.path.join(
             EMULATOR_HOME_DIR, 'avd', self.avd_info.name + '.avd')
         if force and os.path.exists(avd):
             shutil.rmtree(avd)
         if not os.path.exists(avd):
-            self._fetch_tooltool()
-            self._fetch_tooltool_manifest()
-            self._tooltool_fetch()
+            _download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
+            url = '%s/%s' % (TRY_URL, self.avd_info.tooltool_manifest)
+            _download_file(url, 'releng.manifest', EMULATOR_HOME_DIR)
+            _tooltool_fetch()
             self._update_avd_paths()
 
     def start(self):
         """
            Launch the emulator.
         """
         def outputHandler(line):
             self.emulator_log.write("<%s>\n" % line)
         env = os.environ
         env['ANDROID_AVD_HOME'] = os.path.join(EMULATOR_HOME_DIR, "avd")
         command = [self.emulator_path, "-avd",
                    self.avd_info.name, "-port", "5554"]
         if self.avd_info.extra_args:
             command += self.avd_info.extra_args
         log_path = os.path.join(EMULATOR_HOME_DIR, 'emulator.log')
         self.emulator_log = open(log_path, 'w')
-        self._log_debug("Starting the emulator with this command: %s" %
+        _log_debug("Starting the emulator with this command: %s" %
                         ' '.join(command))
-        self._log_debug("Emulator output will be written to '%s'" %
+        _log_debug("Emulator output will be written to '%s'" %
                         log_path)
         self.proc = ProcessHandler(
             command, storeOutput=False, processOutputLine=outputHandler,
             env=env)
         self.proc.run()
-        self._log_debug("Emulator started with pid %d" %
+        _log_debug("Emulator started with pid %d" %
                         int(self.proc.proc.pid))
 
     def wait_for_start(self):
         """
            Verify that the emulator is running, the emulator device is visible
            to adb, and Android has booted.
         """
         if not self.proc:
-            self._log_warning("Emulator not started!")
+            _log_warning("Emulator not started!")
             return False
         if self.proc.proc.poll() is not None:
-            self._log_warning("Emulator has already completed!")
+            _log_warning("Emulator has already completed!")
             return False
-        self._log_debug("Waiting for device status...")
+        _log_debug("Waiting for device status...")
         while(('emulator-5554', 'device') not in self.dm.devices()):
             time.sleep(10)
             if self.proc.proc.poll() is not None:
-                self._log_warning("Emulator has already completed!")
+                _log_warning("Emulator has already completed!")
                 return False
-        self._log_debug("Device status verified.")
+        _log_debug("Device status verified.")
 
-        self._log_debug("Checking that Android has booted...")
+        _log_debug("Checking that Android has booted...")
         complete = False
         while(not complete):
             output = ''
             try:
                 output = self.dm.shellCheckOutput(
                     ['getprop', 'sys.boot_completed'], timeout=5)
             except DMError:
                 # adb not yet responding...keep trying
                 pass
             if output.strip() == '1':
                 complete = True
             else:
                 time.sleep(10)
                 if self.proc.proc.poll() is not None:
-                    self._log_warning("Emulator has already completed!")
+                    _log_warning("Emulator has already completed!")
                     return False
-        self._log_debug("Android boot status verified.")
+        _log_debug("Android boot status verified.")
 
         if not self._verify_emulator():
             return False
         if self.avd_info.uses_sut:
             if not self._verify_sut():
                 return False
         return True
 
@@ -308,88 +371,45 @@ class AndroidEmulator(object):
         self.proc.kill(signal.SIGTERM)
 
     def get_avd_description(self):
         """
            Return the human-friendly description of this AVD.
         """
         return self.avd_info.description
 
-    def _log_debug(self, text):
-        if self.verbose:
-            print "DEBUG: %s" % text
-
-    def _log_warning(self, text):
-        print "WARNING: %s" % text
-
-    def _fetch_tooltool(self):
-        self._download_file(TOOLTOOL_URL, 'tooltool.py', EMULATOR_HOME_DIR)
-
-    def _fetch_tooltool_manifest(self):
-        url = 'https://hg.mozilla.org/%s/raw-file/%s/%s' % (
-            "try", "default", self.avd_info.tooltool_manifest)
-        self._download_file(url, 'releng.manifest', EMULATOR_HOME_DIR)
-
-    def _tooltool_fetch(self):
-        def outputHandler(line):
-            self._log_debug(line)
-        command = ['python', 'tooltool.py', 'fetch', '-m', 'releng.manifest']
-        proc = ProcessHandler(
-            command, processOutputLine=outputHandler, storeOutput=False,
-            cwd=EMULATOR_HOME_DIR)
-        proc.run()
-        try:
-            proc.wait()
-        except:
-            if proc.poll() is None:
-                proc.kill(signal.SIGTERM)
-
     def _update_avd_paths(self):
         avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
         ini_file = os.path.join(avd_path, "test-1.ini")
         ini_file_new = os.path.join(avd_path, self.avd_info.name + ".ini")
         os.rename(ini_file, ini_file_new)
         avd_dir = os.path.join(avd_path, "test-1.avd")
         avd_dir_new = os.path.join(avd_path, self.avd_info.name + ".avd")
         os.rename(avd_dir, avd_dir_new)
         self._replace_ini_contents(ini_file_new)
 
-    def _download_file(self, url, filename, path):
-        f = urllib2.urlopen(url)
-        if not os.path.isdir(path):
-            try:
-                os.makedirs(path)
-            except Exception, e:
-                self._log_warning(str(e))
-                return False
-        local_file = open(os.path.join(path, filename), 'wb')
-        local_file.write(f.read())
-        local_file.close()
-        self._log_debug("Downloaded %s to %s/%s" % (url, path, filename))
-        return True
-
     def _replace_ini_contents(self, path):
         with open(path, "r") as f:
             lines = f.readlines()
         with open(path, "w") as f:
             for line in lines:
                 if line.startswith('path='):
                     avd_path = os.path.join(EMULATOR_HOME_DIR, "avd")
                     f.write('path=%s/%s.avd\n' %
                             (avd_path, self.avd_info.name))
                 elif line.startswith('path.rel='):
                     f.write('path.rel=avd/%s.avd\n' % self.avd_info.name)
                 else:
                     f.write(line)
 
     def _telnet_cmd(self, telnet, command):
-        self._log_debug(">>> " + command)
+        _log_debug(">>> " + command)
         telnet.write('%s\n' % command)
         result = telnet.read_until('OK', 10)
-        self._log_debug("<<< " + result)
+        _log_debug("<<< " + result)
         return result
 
     def _verify_emulator(self):
         telnet_ok = False
         tn = None
         while(not telnet_ok):
             try:
                 tn = telnetlib.Telnet('localhost', self.avd_info.port, 10)
@@ -406,54 +426,54 @@ class AndroidEmulator(object):
                              str(self.avd_info.sut_port2))
                         self._telnet_cmd(tn, cmd)
                     self._telnet_cmd(tn, 'redir list')
                     self._telnet_cmd(tn, 'network status')
                     tn.write('quit\n')
                     tn.read_all()
                     telnet_ok = True
                 else:
-                    self._log_warning("Unable to connect to port %d" % port)
+                    _log_warning("Unable to connect to port %d" % port)
             except:
-                self._log_warning("Trying again after unexpected exception")
+                _log_warning("Trying again after unexpected exception")
             finally:
                 if tn is not None:
                     tn.close()
             if not telnet_ok:
                 time.sleep(10)
                 if self.proc.proc.poll() is not None:
-                    self._log_warning("Emulator has already completed!")
+                    _log_warning("Emulator has already completed!")
                     return False
         return telnet_ok
 
     def _verify_sut(self):
         sut_ok = False
         while(not sut_ok):
             try:
                 tn = telnetlib.Telnet('localhost', self.avd_info.sut_port, 10)
                 if tn is not None:
-                    self._log_debug(
+                    _log_debug(
                         "Connected to port %d" % self.avd_info.sut_port)
                     res = tn.read_until('$>', 10)
                     if res.find('$>') == -1:
-                        self._log_debug("Unexpected SUT response: %s" % res)
+                        _log_debug("Unexpected SUT response: %s" % res)
                     else:
-                        self._log_debug("SUT response: %s" % res)
+                        _log_debug("SUT response: %s" % res)
                         sut_ok = True
                     tn.write('quit\n')
                     tn.read_all()
             except:
-                self._log_debug("Caught exception while verifying sutagent")
+                _log_debug("Caught exception while verifying sutagent")
             finally:
                 if tn is not None:
                     tn.close()
             if not sut_ok:
                 time.sleep(10)
                 if self.proc.proc.poll() is not None:
-                    self._log_warning("Emulator has already completed!")
+                    _log_warning("Emulator has already completed!")
                     return False
         return sut_ok
 
     def _get_avd_type(self, requested):
         if requested in AVD_DICT.keys():
             return requested
         if self.substs:
             if not self.substs['TARGET_CPU'].startswith('arm'):
@@ -475,51 +495,89 @@ class AndroidEmulator(object):
         # Can exe be found in the Android SDK?
         try:
             android_sdk_root = os.environ['ANDROID_SDK_ROOT']
             exe_path = os.path.join(
                 android_sdk_root, subdir, exe)
             if os.path.exists(exe_path):
                 found = True
             else:
-                self._log_debug(
+                _log_debug(
                     "Unable to find executable at %s" % exe_path)
         except KeyError:
-            self._log_debug("ANDROID_SDK_ROOT not set")
+            _log_debug("ANDROID_SDK_ROOT not set")
 
         if not found and self.substs:
             # Can exe be found in ANDROID_TOOLS/ANDROID_PLATFORM_TOOLS?
             try:
                 exe_path = os.path.join(
                     self.substs[var], exe)
                 if os.path.exists(exe_path):
                     found = True
                 else:
-                    self._log_debug(
+                    _log_debug(
                         "Unable to find executable at %s" % exe_path)
             except KeyError:
-                self._log_debug("%s not set" % var)
+                _log_debug("%s not set" % var)
 
         if not found:
             # Can exe be found in the default bootstrap location?
             mozbuild_path = os.environ.get('MOZBUILD_STATE_PATH',
                 os.path.expanduser(os.path.join('~', '.mozbuild')))
             exe_path = os.path.join(
                 mozbuild_path, 'android-sdk-linux', subdir, exe)
             if os.path.exists(exe_path):
                 found = True
             else:
-                self._log_debug(
+                _log_debug(
                     "Unable to find executable at %s" % exe_path)
 
         if not found:
             # Is exe on PATH?
             exe_path = find_executable(exe)
             if exe_path:
                 found = True
             else:
-                self._log_debug("Unable to find executable on PATH")
+                _log_debug("Unable to find executable on PATH")
 
         if found:
-            self._log_debug("%s found at %s" % (exe, exe_path))
+            _log_debug("%s found at %s" % (exe, exe_path))
         else:
             exe_path = None
         return exe_path
+
+def _log_debug(text):
+    if verbose_logging:
+        print "DEBUG: %s" % text
+
+def _log_warning(text):
+    print "WARNING: %s" % text
+
+def _log_info(text):
+    print "%s" % text
+
+def _download_file(url, filename, path):
+    f = urllib2.urlopen(url)
+    if not os.path.isdir(path):
+        try:
+            os.makedirs(path)
+        except Exception, e:
+            _log_warning(str(e))
+            return False
+    local_file = open(os.path.join(path, filename), 'wb')
+    local_file.write(f.read())
+    local_file.close()
+    _log_debug("Downloaded %s to %s/%s" % (url, path, filename))
+    return True
+
+def _tooltool_fetch():
+    def outputHandler(line):
+        _log_debug(line)
+    command = ['python', 'tooltool.py', 'fetch', '-o', '-m', 'releng.manifest']
+    proc = ProcessHandler(
+        command, processOutputLine=outputHandler, storeOutput=False,
+        cwd=EMULATOR_HOME_DIR)
+    proc.run()
+    try:
+        proc.wait()
+    except:
+        if proc.poll() is None:
+            proc.kill(signal.SIGTERM)