Bug 1637379: Optimize android toolchains r=gbrown
authorChris AtLee <catlee@mozilla.com>
Thu, 21 May 2020 13:28:10 +0000
changeset 531427 5d1fbed42ab7dcd32dc2c9cbbfd89c25796b3480
parent 531426 497bed0b00e3223dad9ce2a72ec2beb93f07ef85
child 531428 4aae9dc67310a0d7cc8eae3c415205b9930c153d
push id37439
push userbtara@mozilla.com
push dateThu, 21 May 2020 21:49:34 +0000
treeherdermozilla-central@92c11f0bf14b [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgbrown
bugs1637379
milestone78.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1637379: Optimize android toolchains r=gbrown Create an emulator toolchain Stop including mozboot/sdk-*.zip in our tarball Differential Revision: https://phabricator.services.mozilla.com/D74931
python/mozboot/mozboot/android-emulator-packages.txt
python/mozboot/mozboot/android.py
taskcluster/ci/test/kind.yml
taskcluster/ci/test/xpcshell.yml
taskcluster/ci/toolchain/android.yml
taskcluster/scripts/misc/repack-android-emulator-linux.sh
new file mode 100644
--- /dev/null
+++ b/python/mozboot/mozboot/android-emulator-packages.txt
@@ -0,0 +1,2 @@
+platform-tools
+emulator
--- a/python/mozboot/mozboot/android.py
+++ b/python/mozboot/mozboot/android.py
@@ -131,19 +131,22 @@ def install_mobile_android_sdk_or_ndk(ur
 
         with open(os.devnull, "w") as stdout:
             # These unpack commands produce a ton of output; ignore it.  The
             # .bin files are 7z archives; there's no command line flag to quiet
             # output, so we use this hammer.
             subprocess.check_call(cmd, stdout=stdout)
 
         print('Unpacking %s... DONE' % abspath)
-
+        # Now delete the archive
+        os.unlink(abspath)
     finally:
         os.chdir(old_path)
+        # Remove the download directory
+        os.rmdir(download_path)
 
 
 def get_paths(os_name):
     mozbuild_path = os.environ.get('MOZBUILD_STATE_PATH',
                                    os.path.expanduser(os.path.join('~', '.mozbuild')))
     sdk_path = os.environ.get('ANDROID_SDK_HOME',
                               os.path.join(mozbuild_path, 'android-sdk-{0}'.format(os_name)))
     ndk_path = os.environ.get('ANDROID_NDK_HOME',
@@ -162,17 +165,18 @@ def ensure_dir(dir):
     if dir and not os.path.exists(dir):
         try:
             os.makedirs(dir)
         except OSError as error:
             if error.errno != errno.EEXIST:
                 raise
 
 
-def ensure_android(os_name, artifact_mode=False, ndk_only=False, no_interactive=False):
+def ensure_android(os_name, artifact_mode=False, ndk_only=False,
+                   emulator_only=False, no_interactive=False):
     '''
     Ensure the Android SDK (and NDK, if `artifact_mode` is falsy) are
     installed.  If not, fetch and unpack the SDK and/or NDK from the
     given URLs.  Ensure the required Android SDK packages are
     installed.
 
     `os_name` can be 'linux' or 'macosx'.
     '''
@@ -184,40 +188,42 @@ def ensure_android(os_name, artifact_mod
     os_tag = 'darwin' if os_name == 'macosx' else os_name
     sdk_url = 'https://dl.google.com/android/repository/sdk-tools-{0}-4333796.zip'.format(os_tag)
     ndk_url = android_ndk_url(os_name)
 
     ensure_android_sdk_and_ndk(mozbuild_path, os_name,
                                sdk_path=sdk_path, sdk_url=sdk_url,
                                ndk_path=ndk_path, ndk_url=ndk_url,
                                artifact_mode=artifact_mode,
-                               ndk_only=ndk_only)
+                               ndk_only=ndk_only,
+                               emulator_only=emulator_only)
 
     if ndk_only:
         return
 
     # We expect the |sdkmanager| tool to be at
     # ~/.mozbuild/android-sdk-$OS_NAME/tools/bin/sdkmanager.
     ensure_android_packages(sdkmanager_tool=sdkmanager_tool(sdk_path),
+                            emulator_only=emulator_only,
                             no_interactive=no_interactive)
 
 
 def ensure_android_sdk_and_ndk(mozbuild_path, os_name, sdk_path, sdk_url, ndk_path, ndk_url,
-                               artifact_mode, ndk_only):
+                               artifact_mode, ndk_only, emulator_only):
     '''
     Ensure the Android SDK and NDK are found at the given paths.  If not, fetch
     and unpack the SDK and/or NDK from the given URLs into
     |mozbuild_path/{android-sdk-$OS_NAME,android-ndk-$VER}|.
     '''
 
     # It's not particularly bad to overwrite the NDK toolchain, but it does take
     # a while to unpack, so let's avoid the disk activity if possible.  The SDK
     # may prompt about licensing, so we do this first.
     # Check for Android NDK only if we are not in artifact mode.
-    if not artifact_mode:
+    if not artifact_mode and not emulator_only:
         if os.path.isdir(ndk_path):
             print(ANDROID_NDK_EXISTS % ndk_path)
         else:
             # The NDK archive unpacks into a top-level android-ndk-$VER directory.
             install_mobile_android_sdk_or_ndk(ndk_url, mozbuild_path)
 
     if ndk_only:
         return
@@ -254,26 +260,30 @@ def get_packages_to_install(packages_fil
         sdkmanager --package_file=package_file [options]
     We're doing:
         sdkmanager "platform-tools" "platforms;android-26"
     """
     with open(packages_file_name) as package_file:
         return map(lambda package: package.strip(), package_file.readlines())
 
 
-def ensure_android_packages(sdkmanager_tool, packages=None, no_interactive=False):
+def ensure_android_packages(sdkmanager_tool, emulator_only=False, no_interactive=False):
     '''
     Use the given sdkmanager tool (like 'sdkmanager') to install required
     Android packages.
     '''
 
     # This tries to install all the required Android packages.  The user
     # may be prompted to agree to the Android license.
-    package_file_name = os.path.abspath(os.path.join(os.path.dirname(__file__),
-                                                     'android-packages.txt'))
+    if emulator_only:
+        package_file_name = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                         'android-emulator-packages.txt'))
+    else:
+        package_file_name = os.path.abspath(os.path.join(os.path.dirname(__file__),
+                                                         'android-packages.txt'))
     print(INSTALLING_ANDROID_PACKAGES % open(package_file_name, 'rt').read())
 
     args = [sdkmanager_tool]
     args.extend(get_packages_to_install(package_file_name))
 
     if not no_interactive:
         subprocess.check_call(args)
         return
@@ -338,35 +348,41 @@ def main(argv):
 
     parser = optparse.OptionParser()
     parser.add_option('-a', '--artifact-mode', dest='artifact_mode', action='store_true',
                       help='If true, install only the Android SDK (and not the Android NDK).')
     parser.add_option('--ndk-only', dest='ndk_only', action='store_true',
                       help='If true, install only the Android NDK (and not the Android SDK).')
     parser.add_option('--no-interactive', dest='no_interactive', action='store_true',
                       help='Accept the Android SDK licenses without user interaction.')
+    parser.add_option('--emulator-only', dest='emulator_only', action='store_true',
+                      help='If true, install only the Android emulator (and not the SDK or NDK).')
 
     options, _ = parser.parse_args(argv)
 
     if options.artifact_mode and options.ndk_only:
         raise NotImplementedError('Use no options to install the NDK and the SDK.')
 
+    if options.artifact_mode and options.emulator_only:
+        raise NotImplementedError('Use no options to install the SDK and emulators.')
+
     os_name = None
     if platform.system() == 'Darwin':
         os_name = 'macosx'
     elif platform.system() == 'Linux':
         os_name = 'linux'
     elif platform.system() == 'Windows':
         os_name = 'windows'
     else:
         raise NotImplementedError("We don't support bootstrapping the Android SDK (or Android "
                                   "NDK) on {0} yet!".format(platform.system()))
 
     ensure_android(os_name, artifact_mode=options.artifact_mode,
                    ndk_only=options.ndk_only,
+                   emulator_only=options.emulator_only,
                    no_interactive=options.no_interactive)
     suggest_mozconfig(os_name, options.artifact_mode)
 
     return 0
 
 
 if __name__ == '__main__':
     sys.exit(main(sys.argv))
--- a/taskcluster/ci/test/kind.yml
+++ b/taskcluster/ci/test/kind.yml
@@ -61,14 +61,14 @@ job-defaults:
             by-test-platform:
                 win.*:
                     - win32-minidump-stackwalk
                     - win32-fix-stacks
                 macosx.*:
                     - macosx64-minidump-stackwalk
                     - macosx64-fix-stacks
                 android-em-7.*:
-                    - android-sdk-linux
+                    - android-emulator-linux
                     - linux64-minidump-stackwalk
                     - linux64-fix-stacks
                 default:
                     - linux64-minidump-stackwalk
                     - linux64-fix-stacks
--- a/taskcluster/ci/test/xpcshell.yml
+++ b/taskcluster/ci/test/xpcshell.yml
@@ -87,11 +87,11 @@ xpcshell:
                     - win64-node
                     - win32-minidump-stackwalk
                     - win32-fix-stacks
                 win.*32.*:
                     - win32-node
                     - win32-minidump-stackwalk
                     - win32-fix-stacks
                 android-em-7.*:
-                    - android-sdk-linux
+                    - android-emulator-linux
                     - linux64-node
                     - linux64-minidump-stackwalk
--- a/taskcluster/ci/toolchain/android.yml
+++ b/taskcluster/ci/toolchain/android.yml
@@ -19,16 +19,32 @@ linux64-android-sdk-linux-repack:
               type: directory
     run:
         script: repack-android-sdk-linux.sh
         resources:
             - 'python/mozboot/**/*android*'
         toolchain-artifact: project/gecko/android-sdk/android-sdk-linux.tar.xz
         toolchain-alias: android-sdk-linux
 
+linux64-android-emulator-linux-repack:
+    description: "Android Emulator (Linux) repack toolchain build"
+    treeherder:
+        symbol: TL(android-emulator-linux)
+    worker:
+        artifacts:
+            - name: project/gecko/android-emulator
+              path: /builds/worker/project/gecko/android-emulator/
+              type: directory
+    run:
+        script: repack-android-emulator-linux.sh
+        resources:
+            - 'python/mozboot/**/*android*'
+        toolchain-artifact: project/gecko/android-emulator/android-emulator-linux.tar.xz
+        toolchain-alias: android-emulator-linux
+
 linux64-android-ndk-linux-repack:
     description: "Android NDK (Linux) repack toolchain build"
     treeherder:
         symbol: TL(android-ndk-linux)
     worker:
         artifacts:
             - name: project/gecko/android-ndk
               path: /builds/worker/project/gecko/android-ndk/
new file mode 100755
--- /dev/null
+++ b/taskcluster/scripts/misc/repack-android-emulator-linux.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -x -e -v
+
+# This script is for fetching and repacking the Android emulator (for
+# Linux), the tools required to produce Android packages.
+
+UPLOAD_DIR=$HOME/project/gecko/android-emulator
+
+mkdir -p $HOME/artifacts $UPLOAD_DIR
+
+# Populate /builds/worker/.mozbuild/android-emulator-linux.
+cd $GECKO_PATH
+./mach python python/mozboot/mozboot/android.py --emulator-only --no-interactive
+
+# Remove extra files we don't need
+rm -rfv /builds/worker/.mozbuild/android-sdk-linux/tools
+mkdir /builds/worker/.mozbuild/android-sdk-linux/system-images
+mkdir /builds/worker/.mozbuild/android-sdk-linux/platforms
+find /builds/worker/.mozbuild/android-sdk-linux/emulator/qemu -type f -not -name "*x86*" -print -delete
+
+tar cv -C /builds/worker/.mozbuild android-sdk-linux | xz > $UPLOAD_DIR/android-emulator-linux.tar.xz
+
+ls -al $UPLOAD_DIR