Bug 1108771 - Part 2: Implement |mach bootstrap| for mobile/android on Debian-like systems. r=gps
authorNick Alexander <nalexander@mozilla.com>
Sun, 21 Dec 2014 15:29:18 -0800
changeset 221186 3ecd0c6da57ba01c64ad467781c45d82510e3791
parent 221185 f5758bdccc0a052d52206fc9ff4897ab4a8d9f4f
child 221187 2b353a88f2b9f3927b53ea5283d1747ce395ea49
push id28013
push userphilringnalda@gmail.com
push dateWed, 24 Dec 2014 23:31:28 +0000
treeherdermozilla-central@38471b0310c9 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgps
bugs1108771
milestone37.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1108771 - Part 2: Implement |mach bootstrap| for mobile/android on Debian-like systems. r=gps This adds a generic android Python module that handles: * downloading and unpacking Google's Android SDK and NDK bundles; * using the |android| tool to install additional Android packages; * printing a mozconfig snippet suitable for mobile/android builds.
python/mozboot/bin/bootstrap.py
python/mozboot/mozboot/android.py
python/mozboot/mozboot/debian.py
--- a/python/mozboot/bin/bootstrap.py
+++ b/python/mozboot/bin/bootstrap.py
@@ -27,16 +27,17 @@ from optparse import OptionParser
 
 # The next two variables define where in the repository the Python files
 # reside. This is used to remotely download file content when it isn't
 # available locally.
 REPOSITORY_PATH_PREFIX = 'python/mozboot'
 
 REPOSITORY_PATHS = [
     'mozboot/__init__.py',
+    'mozboot/android.py',
     'mozboot/base.py',
     'mozboot/bootstrap.py',
     'mozboot/centos.py',
     'mozboot/debian.py',
     'mozboot/fedora.py',
     'mozboot/freebsd.py',
     'mozboot/gentoo.py',
     'mozboot/openbsd.py',
new file mode 100644
--- /dev/null
+++ b/python/mozboot/mozboot/android.py
@@ -0,0 +1,208 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this,
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# If we add unicode_literals, Python 2.6.1 (required for OS X 10.6) breaks.
+from __future__ import print_function
+
+import errno
+import os
+import subprocess
+
+# These are the platform and build-tools versions for building
+# mobile/android, respectively. Try to keep these in synch with the
+# build system and Mozilla's automation.
+ANDROID_PLATFORM = 'android-21'
+ANDROID_BUILD_TOOLS_VERSION = '21.1.2'
+
+# These are the "Android packages" needed for building Firefox for Android.
+# Use |android list sdk --extended| to see these identifiers.
+ANDROID_PACKAGES = [
+    'tools',
+    'platform-tools',
+    'build-tools-%s' % ANDROID_BUILD_TOOLS_VERSION,
+    ANDROID_PLATFORM,
+    'extra-android-support',
+    'extra-google-google_play_services',
+]
+
+ANDROID_NDK_EXISTS = '''
+Looks like you have the Android NDK installed at:
+%s
+'''
+
+ANDROID_SDK_EXISTS = '''
+Looks like you have the Android SDK installed at:
+%s
+We will install all required Android packages.
+'''
+
+NOT_INSTALLING_ANDROID_PACKAGES = '''
+It looks like you already have the following Android packages:
+%s
+No need to update!
+'''
+
+INSTALLING_ANDROID_PACKAGES = '''
+We are now installing the following Android packages:
+%s
+You may be prompted to agree to the Android license. You may see some of
+output as packages are downloaded and installed.
+'''
+
+MISSING_ANDROID_PACKAGES = '''
+We tried to install the following Android packages:
+%s
+But it looks like we couldn't install:
+%s
+Install these Android packages manually and run this bootstrapper again.
+'''
+
+MOBILE_ANDROID_MOZCONFIG_TEMPLATE = '''
+Paste the lines between the chevrons (>>> and <<<) into your mozconfig file:
+
+<<<
+# Build Firefox for Android:
+ac_add_options --enable-application=mobile/android
+ac_add_options --target=arm-linux-androideabi
+
+# With the following Android SDK and NDK:
+ac_add_options --with-android-sdk="%s"
+ac_add_options --with-android-ndk="%s"
+>>>
+'''
+
+
+def check_output(*args, **kwargs):
+    """Run subprocess.check_output even if Python doesn't provide it."""
+    from base import BaseBootstrapper
+    fn = getattr(subprocess, 'check_output', BaseBootstrapper._check_output)
+
+    return fn(*args, **kwargs)
+
+def list_missing_android_packages(android_tool, packages):
+    '''
+    Use the given |android| tool to return the sub-list of Android
+    |packages| given that are not installed.
+    '''
+    missing = []
+
+    # There's no obvious way to see what's been installed already,
+    # but packages that are installed don't appear in the list of
+    # available packages.
+    lines = check_output([android_tool,
+        'list', 'sdk', '--no-ui', '--extended']).splitlines()
+
+    # Lines look like: 'id: 59 or "extra-google-simulators"'
+    for line in lines:
+        is_id_line = False
+        try:
+            is_id_line = line.startswith("id:")
+        except:
+            # Some lines contain non-ASCII characters.  Ignore them.
+            pass
+        if not is_id_line:
+            continue
+
+        for package in packages:
+            if '"%s"' % package in line:
+                # Not installed!
+                missing.append(package)
+
+    return missing
+
+def install_mobile_android_sdk_or_ndk(url, path):
+    '''
+    Fetch an Android SDK or NDK from |url| and unpack it into
+    the given |path|.
+
+    We expect wget to be installed and found on the system path.
+
+    We use, and wget respects, https.  We could also include SHAs for a
+    small improvement in the integrity guarantee we give. But this script is
+    bootstrapped over https anyway, so it's a really minor improvement.
+
+    We use |wget --continue| as a cheap cache of the downloaded artifacts,
+    writing into |path|/mozboot.  We don't yet clean the cache; it's better
+    to waste disk and not require a long re-download than to wipe the cache
+    prematurely.
+    '''
+
+    old_path = os.getcwd()
+    try:
+        download_path = os.path.join(path, 'mozboot')
+        try:
+            os.makedirs(download_path)
+        except OSError as e:
+            if e.errno == errno.EEXIST and os.path.isdir(download_path):
+                pass
+            else:
+                raise
+
+        os.chdir(download_path)
+        subprocess.check_call(['wget', '--continue', url])
+        file = url.split('/')[-1]
+
+        os.chdir(path)
+        if file.endswith('.tar.gz') or file.endswith('.tgz'):
+            cmd = ['tar', 'zvxf']
+        elif file.endswith('.tar.bz2'):
+            cmd = ['tar', 'jvxf']
+        elif file.endswitch('.zip'):
+            cmd = ['unzip']
+        else:
+            raise NotImplementedError("Don't know how to unpack file: %s" % file)
+        subprocess.check_call(cmd + [os.path.join(download_path, file)])
+    finally:
+        os.chdir(old_path)
+
+def ensure_android_sdk_and_ndk(path, sdk_path, sdk_url, ndk_path, ndk_url):
+    '''
+    Ensure the Android SDK and NDK are found at the given paths.  If not, fetch
+    and unpack the SDK and/or NDK from the given URLs into |path|.
+    '''
+
+    # It's not particularyl bad to overwrite the NDK toolchain, but it does take
+    # a while to unpack, so let's avoid the disk activity if possible.  The SDK
+    # may prompt about licensing, so we do this first.
+    if os.path.isdir(ndk_path):
+        print(ANDROID_NDK_EXISTS % ndk_path)
+    else:
+        install_mobile_android_sdk_or_ndk(ndk_url, path)
+
+    # We don't want to blindly overwrite, since we use the |android| tool to
+    # install additional parts of the Android toolchain.  If we overwrite,
+    # we lose whatever Android packages the user may have already installed.
+    if os.path.isdir(sdk_path):
+        print(ANDROID_SDK_EXISTS % sdk_path)
+    else:
+        install_mobile_android_sdk_or_ndk(sdk_url, path)
+
+def ensure_android_packages(android_tool, packages=None):
+    '''
+    Use the given android tool (like 'android') to install required Android
+    packages.
+    '''
+
+    if not packages:
+        packages = ANDROID_PACKAGES
+
+    missing = list_missing_android_packages(android_tool, packages=packages)
+    if not missing:
+        print(NOT_INSTALLING_ANDROID_PACKAGES % ', '.join(packages))
+        return
+
+    # This tries to install all the required Android packages.  The user
+    # may be prompted to agree to the Android license.
+    print(INSTALLING_ANDROID_PACKAGES % ', '.join(missing))
+    subprocess.check_call([android_tool,
+        'update', 'sdk', '--no-ui',
+        '--filter', ','.join(missing)])
+
+    # Let's verify.
+    failing = list_missing_android_packages(android_tool, packages=packages)
+    if failing:
+        raise Exception(MISSING_ANDROID_PACKAGES % (', '.join(missing), ', '.join(failing)))
+
+def suggest_mozconfig(sdk_path=None, ndk_path=None):
+    print(MOBILE_ANDROID_MOZCONFIG_TEMPLATE % (sdk_path, ndk_path))
--- a/python/mozboot/mozboot/debian.py
+++ b/python/mozboot/mozboot/debian.py
@@ -1,12 +1,15 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import os
+import sys
+
 from mozboot.base import BaseBootstrapper
 
 class DebianBootstrapper(BaseBootstrapper):
     # These are common packages for all Debian-derived distros (such as
     # Ubuntu).
     COMMON_PACKAGES = [
         'autoconf2.13',
         'build-essential',
@@ -17,16 +20,18 @@ class DebianBootstrapper(BaseBootstrappe
         'unzip',
         'uuid',
         'zip',
     ]
 
     # Subclasses can add packages to this variable to have them installed.
     DISTRO_PACKAGES = []
 
+    # These are common packages for building Firefox for Desktop
+    # (browser) for all Debian-derived distros (such as Ubuntu).
     BROWSER_COMMON_PACKAGES = [
         'libasound2-dev',
         'libcurl4-openssl-dev',
         'libdbus-1-dev',
         'libdbus-glib-1-dev',
         'libgconf2-dev',
         'libgstreamer0.10-dev',
         'libgstreamer-plugins-base0.10-dev',
@@ -39,26 +44,86 @@ class DebianBootstrapper(BaseBootstrappe
         'python-dbus',
         'yasm',
         'xvfb',
     ]
 
     # Subclasses can add packages to this variable to have them installed.
     BROWSER_DISTRO_PACKAGES = []
 
+    # These are common packages for building Firefox for Android
+    # (mobile/android) for all Debian-derived distros (such as Ubuntu).
+    MOBILE_ANDROID_COMMON_PACKAGES = [
+        'zlib1g-dev', # mobile/android requires system zlib.
+        'openjdk-7-jdk',
+        'ant',
+        'wget', # For downloading the Android SDK and NDK.
+        'libncurses5:i386', # See comments about i386 below.
+        'libstdc++6:i386',
+        'zlib1g:i386',
+    ]
+
+    # Subclasses can add packages to this variable to have them installed.
+    MOBILE_ANDROID_DISTRO_PACKAGES = []
+
     def __init__(self, version, dist_id):
         BaseBootstrapper.__init__(self)
 
         self.version = version
         self.dist_id = dist_id
 
         self.packages = self.COMMON_PACKAGES + self.DISTRO_PACKAGES
         self.browser_packages = self.BROWSER_COMMON_PACKAGES + self.BROWSER_DISTRO_PACKAGES
+        self.mobile_android_packages = self.MOBILE_ANDROID_COMMON_PACKAGES + self.MOBILE_ANDROID_DISTRO_PACKAGES
 
     def install_system_packages(self):
         self.apt_install(*self.packages)
 
     def install_browser_packages(self):
         self.apt_install(*self.browser_packages)
 
+    def install_mobile_android_packages(self):
+        import android
+
+        # Multi-part process:
+        # 1. System packages.
+        # 2. Android SDK and NDK.
+        # 3. Android packages.
+
+        # 1. This is hard to believe, but the Android SDK binaries are 32-bit
+        # and that conflicts with 64-bit Debian and Ubuntu installations out of
+        # the box.  The solution is to add the i386 architecture.  See
+        # "Troubleshooting Ubuntu" at
+        # http://developer.android.com/sdk/installing/index.html?pkg=tools.
+        self.run_as_root(['dpkg', '--add-architecture', 'i386'])
+        # self.apt_update()
+        self.apt_install(*self.mobile_android_packages)
+
+        # 2. The user may have an external Android SDK (in which case we save
+        # them a lengthy download), or they may have already completed the
+        # download. We unpack to ~/.mozbuild/{android-sdk-linux, android-ndk-r8e}.
+        mozbuild_path = os.environ.get('MOZBUILD_STATE_PATH', os.path.expanduser(os.path.join('~', '.mozbuild')))
+        self.sdk_path = os.environ.get('ANDROID_SDK_HOME', os.path.join(mozbuild_path, 'android-sdk-linux'))
+        self.ndk_path = os.environ.get('ANDROID_NDK_HOME', os.path.join(mozbuild_path, 'android-ndk-r8e'))
+        self.sdk_url = 'https://dl.google.com/android/android-sdk_r24.0.1-linux.tgz'
+        is_64bits = sys.maxsize > 2**32
+        if is_64bits:
+            self.ndk_url = 'https://dl.google.com/android/ndk/android-ndk-r8e-linux-x86_64.tar.bz2'
+        else:
+            self.ndk_url = 'https://dl.google.com/android/ndk/android-ndk-r8e-linux-x86.tar.bz2'
+        android.ensure_android_sdk_and_ndk(path=mozbuild_path,
+            sdk_path=self.sdk_path, sdk_url=self.sdk_url,
+            ndk_path=self.ndk_path, ndk_url=self.ndk_url)
+
+        # 3. We expect the |android| tool to at
+        # ~/.mozbuild/android-sdk-linux/tools/android.
+        android_tool = os.path.join(self.sdk_path, 'tools', 'android')
+        android.ensure_android_packages(android_tool=android_tool)
+
+    def suggest_mobile_android_mozconfig(self):
+        import android
+
+        android.suggest_mozconfig(sdk_path=self.sdk_path,
+            ndk_path=self.ndk_path)
+
     def _update_package_manager(self):
         self.run_as_root(['apt-get', 'update'])