Bug 1344244 - Part 3: Use sdkmanager rather than android. r=glandium
authorNick Alexander <nalexander@mozilla.com>
Wed, 05 Jul 2017 16:49:09 -0700
changeset 369509 0bae2cd791690a4ed325dc651c5cdf2d113b7f80
parent 369508 01b0a01a38b1355ae9626308f1fb22be93b5acd9
child 369510 4f056c7650c754654c19fa6ae8409a4fa71f11f8
push id32198
push userryanvm@gmail.com
push dateWed, 19 Jul 2017 13:22:22 +0000
treeherdermozilla-central@a1ea69a3a0a7 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersglandium
bugs1344244
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1344244 - Part 3: Use sdkmanager rather than android. r=glandium This is the real fix. Google has replaced the |android --no-ui ...| tool with a simpler |sdkmanager| tool, which makes it easier to install packages with particular major versions. (Minor versions still can't be controlled; for example, the m2repository extras are constantly rolling forward.) |sdkmanager| fails if the required packages aren't installed and can't be installed, so there's no need to search for missing packages, etc, simplifying the code considerably. I don't see an easy way to upgrade outdated Android SDK installations -- it's not clear that unpacking over top of an existing SDK installation succeeds -- so I've included a message about moving or removing outdated installations. This will punish folks who have added additional Android platforms, or download emulator images using the Android toolchain (but not those downloaded using |mach emulator|). C'est la vie. MozReview-Commit-ID: GLhKyuq701k
python/mozboot/mozboot/android-packages.txt
python/mozboot/mozboot/android.py
new file mode 100644
--- /dev/null
+++ b/python/mozboot/mozboot/android-packages.txt
@@ -0,0 +1,5 @@
+platform-tools
+build-tools;23.0.3
+platforms;android-23
+extras;android;m2repository
+extras;google;m2repository
--- a/python/mozboot/mozboot/android.py
+++ b/python/mozboot/mozboot/android.py
@@ -34,37 +34,31 @@ Looks like you have the Android NDK inst
 '''
 
 ANDROID_SDK_EXISTS = '''
 Looks like you have the Android SDK installed at:
 %s
 We will install all required Android packages.
 '''
 
-NOT_INSTALLING_ANDROID_PACKAGES = '''
-It looks like you already have the following Android packages:
+ANDROID_SDK_TOO_OLD = '''
+Looks like you have an outdated Android SDK installed at:
 %s
-No need to update!
+I can't update outdated Android SDKs to have the required 'sdkmanager'
+tool.  Move it out of the way (or remove it entirely) and then run
+bootstrap again.
 '''
 
 INSTALLING_ANDROID_PACKAGES = '''
 We are now installing the following Android packages:
 %s
 You may be prompted to agree to the Android license. You may see some of
 output as packages are downloaded and installed.
 '''
 
-MISSING_ANDROID_PACKAGES = '''
-We tried to install the following Android packages:
-%s
-But it looks like we couldn't install:
-%s
-Install these Android packages manually and run this bootstrapper again.
-'''
-
 MOBILE_ANDROID_MOZCONFIG_TEMPLATE = '''
 Paste the lines between the chevrons (>>> and <<<) into your mozconfig file:
 
 <<<
 # Build Firefox for Android:
 ac_add_options --enable-application=mobile/android
 ac_add_options --target=arm-linux-androideabi
 
@@ -87,56 +81,16 @@ ac_add_options --enable-artifact-builds
 ac_add_options --with-android-sdk="%s"
 
 # Write build artifacts to:
 mk_add_options MOZ_OBJDIR=./objdir-frontend
 >>>
 '''
 
 
-def check_output(*args, **kwargs):
-    """Run subprocess.check_output even if Python doesn't provide it."""
-    from base import BaseBootstrapper
-    fn = getattr(subprocess, 'check_output', BaseBootstrapper._check_output)
-
-    return fn(*args, **kwargs)
-
-
-def list_missing_android_packages(android_tool, packages):
-    '''
-    Use the given |android| tool to return the sub-list of Android
-    |packages| given that are not installed.
-    '''
-    missing = []
-
-    # There's no obvious way to see what's been installed already,
-    # but packages that are installed don't appear in the list of
-    # available packages.
-    lines = check_output([android_tool,
-                          'list', 'sdk', '--no-ui', '--extended']).splitlines()
-
-    # Lines look like: 'id: 59 or "extra-google-simulators"'
-    for line in lines:
-        is_id_line = False
-        try:
-            is_id_line = line.startswith("id:")
-        except:
-            # Some lines contain non-ASCII characters.  Ignore them.
-            pass
-        if not is_id_line:
-            continue
-
-        for package in packages:
-            if '"%s"' % package in line:
-                # Not installed!
-                missing.append(package)
-
-    return missing
-
-
 def install_mobile_android_sdk_or_ndk(url, path):
     '''
     Fetch an Android SDK or NDK from |url| and unpack it into
     the given |path|.
 
     We expect wget to be installed and found on the system path.
 
     We use, and wget respects, https.  We could also include SHAs for a
@@ -213,29 +167,29 @@ def ensure_android(os_name, artifact_mod
 
     `os_name` can be 'linux' or 'macosx'.
     '''
     # The user may have an external Android SDK (in which case we
     # save them a lengthy download), or they may have already
     # completed the download. We unpack to
     # ~/.mozbuild/{android-sdk-$OS_NAME, android-ndk-r11c}.
     mozbuild_path, sdk_path, ndk_path = get_paths(os_name)
-    ext = 'zip' if os_name == 'macosx' else 'tgz'
-    sdk_url = 'https://dl.google.com/android/android-sdk_r24.0.1-{}.{}'.format(os_name, ext)
+    os_tag = 'darwin' if os_name == 'macosx' else os_name
+    sdk_url = 'https://dl.google.com/android/repository/sdk-tools-{}-3859397.zip'.format(os_tag)
     ndk_url = android_ndk_url(os_name)
 
-    ensure_android_sdk_and_ndk(path=mozbuild_path,
+    ensure_android_sdk_and_ndk(path=os.path.join(mozbuild_path, 'android-sdk-{}'.format(os_name)),
                                sdk_path=sdk_path, sdk_url=sdk_url,
                                ndk_path=ndk_path, ndk_url=ndk_url,
                                artifact_mode=artifact_mode)
 
-    # We expect the |android| tool to be at
-    # ~/.mozbuild/android-sdk-$OS_NAME/tools/android.
-    android_tool = os.path.join(sdk_path, 'tools', 'android')
-    ensure_android_packages(android_tool=android_tool)
+    # We expect the |sdkmanager| tool to be at
+    # ~/.mozbuild/android-sdk-$OS_NAME/tools/bin/sdkmanager.
+    sdkmanager_tool = os.path.join(sdk_path, 'tools', 'bin', 'sdkmanager')
+    ensure_android_packages(sdkmanager_tool=sdkmanager_tool)
 
 
 def ensure_android_sdk_and_ndk(path, sdk_path, sdk_url, ndk_path, ndk_url, artifact_mode):
     '''
     Ensure the Android SDK and NDK are found at the given paths.  If not, fetch
     and unpack the SDK and/or NDK from the given URLs into |path|.
     '''
 
@@ -244,55 +198,41 @@ def ensure_android_sdk_and_ndk(path, sdk
     # may prompt about licensing, so we do this first.
     # Check for Android NDK only if we are not in artifact mode.
     if not artifact_mode:
         if os.path.isdir(ndk_path):
             print(ANDROID_NDK_EXISTS % ndk_path)
         else:
             install_mobile_android_sdk_or_ndk(ndk_url, path)
 
-    # We don't want to blindly overwrite, since we use the |android| tool to
-    # install additional parts of the Android toolchain.  If we overwrite,
-    # we lose whatever Android packages the user may have already installed.
-    if os.path.isdir(sdk_path):
+    # We don't want to blindly overwrite, since we use the
+    # |sdkmanager| tool to install additional parts of the Android
+    # toolchain.  If we overwrite, we lose whatever Android packages
+    # the user may have already installed.
+    if os.path.isfile(os.path.join(sdk_path, 'tools', 'bin', 'sdkmanager')):
         print(ANDROID_SDK_EXISTS % sdk_path)
+    elif os.path.isdir(sdk_path):
+        raise NotImplementedError(ANDROID_SDK_TOO_OLD % sdk_path)
     else:
         install_mobile_android_sdk_or_ndk(sdk_url, path)
 
 
-def ensure_android_packages(android_tool, packages=None):
-    '''
-    Use the given android tool (like 'android') to install required Android
-    packages.
+def ensure_android_packages(sdkmanager_tool, packages=None):
     '''
-
-    if not packages:
-        packages = ANDROID_PACKAGES
-
-    # Bug 1171232: The |android| tool behaviour has changed; we no longer can
-    # see what packages are installed easily.  Force installing everything until
-    # we find a way to actually see the missing packages.
-    missing = packages
-    if not missing:
-        print(NOT_INSTALLING_ANDROID_PACKAGES % ', '.join(packages))
-        return
+    Use the given sdkmanager tool (like 'sdkmanager') to install required
+    Android packages.
+    '''
 
     # This tries to install all the required Android packages.  The user
     # may be prompted to agree to the Android license.
-    print(INSTALLING_ANDROID_PACKAGES % ', '.join(missing))
-    subprocess.check_call([android_tool,
-                           'update', 'sdk', '--no-ui', '--all',
-                           '--filter', ','.join(missing)])
-
-    # Bug 1171232: The |android| tool behaviour has changed; we no longer can
-    # see what packages are installed easily.  Don't check until we find a way
-    # to actually verify.
-    failing = []
-    if failing:
-        raise Exception(MISSING_ANDROID_PACKAGES % (', '.join(missing), ', '.join(failing)))
+    package_file_name = os.path.abspath(os.path.join(os.path.dirname(__file__), 'android-packages.txt'))
+    print(package_file_name)
+    print(INSTALLING_ANDROID_PACKAGES % open(package_file_name, 'rt').read())
+    subprocess.check_call([sdkmanager_tool,
+                           '--package_file={}'.format(package_file_name)])
 
 
 def suggest_mozconfig(os_name, artifact_mode=False):
     _mozbuild_path, sdk_path, ndk_path = get_paths(os_name)
     if artifact_mode:
         print(MOBILE_ANDROID_ARTIFACT_MODE_MOZCONFIG_TEMPLATE % (sdk_path))
     else:
         print(MOBILE_ANDROID_MOZCONFIG_TEMPLATE % (sdk_path, ndk_path))