Bug 1062365 - Pull android sdk from tooltool instead of using /tools/android-sdk18; r=kmoir
authorGeoff Brown <gbrown@mozilla.com>
Tue, 03 Mar 2015 16:12:36 -0700 (2015-03-03)
changeset 3708 e170cecfdee2907e25f42e17a26500222085db12
parent 3707 4ec8ad84138323c31ef9565866a42914215c1bee
child 3709 051e3290c3fa57d5d6b44808af3944dccbbfb55f
child 3710 a8fbffe4c044d6fd1c09af1603ac0dda98c511c3
push id2912
push usergbrown@mozilla.com
push dateWed, 04 Mar 2015 18:01:07 +0000 (2015-03-04)
reviewerskmoir
bugs1062365
Bug 1062365 - Pull android sdk from tooltool instead of using /tools/android-sdk18; r=kmoir
configs/android/androidarm.py
configs/android/androidarm_4_3.py
configs/android/androidx86.py
mozharness/mozilla/testing/unittest.py
scripts/android_emulator_unittest.py
--- a/configs/android/androidarm.py
+++ b/configs/android/androidarm.py
@@ -5,28 +5,38 @@ config = {
     "host_utils_url": "http://talos-remote.pvt.build.mozilla.org/tegra/tegra-host-utils.Linux.1109310.2.zip",
     "robocop_package_name": "org.mozilla.roboexample.test",
     "device_ip": "127.0.0.1",
     "default_sut_port1": "20701",
     "default_sut_port2": "20700", # does not prompt for commands
     "tooltool_manifest_path": "testing/config/tooltool-manifests/androidarm/releng.manifest",
     "tooltool_cache": "/builds/tooltool_cache",
     "tooltool_servers": ["http://tooltool.pvt.build.mozilla.org/build/"],
+    "emulator_manifest": """
+        [
+        {
+        "size": 193383673,
+        "digest": "6609e8b95db59c6a3ad60fc3dcfc358b2c8ec8b4dda4c2780eb439e1c5dcc5d550f2e47ce56ba14309363070078d09b5287e372f6e95686110ff8a2ef1838221",
+        "algorithm": "sha512",
+        "filename": "android-sdk18_0.r18moz1.orig.tar.gz",
+        "unpack": "True"
+        }
+        ] """,
     "emulator_process_name": "emulator64-arm",
     "emulator_extra_args": "-debug init,console,gles,memcheck,adbserver,adbclient,adb,avd_config,socket -qemu -m 1024 -cpu cortex-a9",
     "device_manager": "sut",
     "exes": {
-        'adb': '/tools/android-sdk18/platform-tools/adb',
+        'adb': '%(abs_work_dir)s/android-sdk18/platform-tools/adb',
         'python': '/tools/buildbot/bin/python',
         'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
         'tooltool.py': "/tools/tooltool.py",
     },
     "env": {
         "DISPLAY": ":0.0",
-        "PATH": "%(PATH)s:/tools/android-sdk18/tools:/tools/android-sdk18/platform-tools",
+        "PATH": "%(PATH)s:%(abs_work_dir)s/android-sdk18/tools:%(abs_work_dir)s/android-sdk18/platform-tools",
         "MINIDUMP_SAVEPATH": "%(abs_work_dir)s/../minidumps"
     },
     "default_actions": [
         'clobber',
         'read-buildbot-config',
         'setup-avds',
         'start-emulators',
         'download-and-extract',
--- a/configs/android/androidarm_4_3.py
+++ b/configs/android/androidarm_4_3.py
@@ -3,29 +3,48 @@ import os
 config = {
     "buildbot_json_path": "buildprops.json",
     "host_utils_url": "http://talos-remote.pvt.build.mozilla.org/tegra/tegra-host-utils.Linux.1109310.2.zip",
     "robocop_package_name": "org.mozilla.roboexample.test",
     "tooltool_manifest_path": "testing/config/tooltool-manifests/androidarm/releng.manifest",
     "tooltool_cache": "/builds/tooltool_cache",
     "tooltool_servers": ["http://tooltool.pvt.build.mozilla.org/build/"],
     ".avds_dir": "/home/cltbld/.android",
-    "emulator_url": "http://people.mozilla.org/~gbrown/android-sdk_r24.0.2-linux.zip",
+    "emulator_manifest": """
+        [
+        {
+        "size": 140097024,
+        "digest": "51781032335c09103e8509b1a558bf22a7119392cf1ea301c49c01bdf21ff0ceb37d260bc1c322cd9b903252429fb01830fc27e4632be30cd345c95bf4b1a39b",
+        "algorithm": "sha512",
+        "filename": "android-sdk_r24.0.2-linux.tgz",
+        "unpack": "True"
+        }
+        ] """,
+    "tools_manifest": """
+        [
+        {
+        "size": 193383673,
+        "digest": "6609e8b95db59c6a3ad60fc3dcfc358b2c8ec8b4dda4c2780eb439e1c5dcc5d550f2e47ce56ba14309363070078d09b5287e372f6e95686110ff8a2ef1838221",
+        "algorithm": "sha512",
+        "filename": "android-sdk18_0.r18moz1.orig.tar.gz",
+        "unpack": "True"
+        }
+        ] """,
     "emulator_process_name": "emulator64-arm",
     "emulator_extra_args": "-debug init,console,gles,memcheck,adbserver,adbclient,adb,avd_config,socket",
     "device_manager": "adb",
     "exes": {
-        'adb': '/tools/android-sdk18/platform-tools/adb',
+        'adb': '%(abs_work_dir)s/android-sdk18/platform-tools/adb',
         'python': '/tools/buildbot/bin/python',
         'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
         'tooltool.py': "/tools/tooltool.py",
     },
     "env": {
         "DISPLAY": ":0.0",
-        "PATH": "%(PATH)s:%(abs_work_dir)s/emulator/android-sdk-linux/tools:/tools/android-sdk18/platform-tools",
+        "PATH": "%(PATH)s:%(abs_work_dir)s/android-sdk-linux/tools:%(abs_work_dir)s/android-sdk18/platform-tools",
         "MINIDUMP_SAVEPATH": "%(abs_work_dir)s/../minidumps"
     },
     "default_actions": [
         'clobber',
         'read-buildbot-config',
         'setup-avds',
         'start-emulators',
         'download-and-extract',
--- a/configs/android/androidx86.py
+++ b/configs/android/androidx86.py
@@ -5,28 +5,38 @@ config = {
     "host_utils_url": "http://talos-remote.pvt.build.mozilla.org/tegra/tegra-host-utils.Linux.1109310.2.zip",
     "robocop_package_name": "org.mozilla.roboexample.test",
     "device_ip": "127.0.0.1",
     "default_sut_port1": "20701",
     "default_sut_port2": "20700", # does not prompt for commands
     "tooltool_manifest_path": "testing/config/tooltool-manifests/androidx86/releng.manifest",
     "tooltool_cache": "/builds/tooltool_cache",
     "tooltool_servers": ["http://tooltool.pvt.build.mozilla.org/build/"],
+    "emulator_manifest": """
+        [
+        {
+        "size": 193383673,
+        "digest": "6609e8b95db59c6a3ad60fc3dcfc358b2c8ec8b4dda4c2780eb439e1c5dcc5d550f2e47ce56ba14309363070078d09b5287e372f6e95686110ff8a2ef1838221",
+        "algorithm": "sha512",
+        "filename": "android-sdk18_0.r18moz1.orig.tar.gz",
+        "unpack": "True"
+        }
+        ] """,
     "emulator_process_name": "emulator64-x86",
     "emulator_extra_args": "-debug init,console,gles,memcheck,adbserver,adbclient,adb,avd_config,socket -qemu -m 1024 -enable-kvm",
     "device_manager": "sut",
     "exes": {
-        'adb': '/tools/android-sdk18/platform-tools/adb',
+        'adb': '%(abs_work_dir)s/android-sdk18/platform-tools/adb',
         'python': '/tools/buildbot/bin/python',
         'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
         'tooltool.py': "/tools/tooltool.py",
     },
     "env": {
         "DISPLAY": ":0.0",
-        "PATH": "%(PATH)s:/tools/android-sdk18/tools:/tools/android-sdk18/platform-tools",
+        "PATH": "%(PATH)s:%(abs_work_dir)s/android-sdk18/tools:%(abs_work_dir)s/android-sdk18/platform-tools",
     },
     "default_actions": [
         'clobber',
         'read-buildbot-config',
         'setup-avds',
         'start-emulators',
         'download-and-extract',
         'create-virtualenv',
--- a/mozharness/mozilla/testing/unittest.py
+++ b/mozharness/mozilla/testing/unittest.py
@@ -200,37 +200,47 @@ class DesktopUnittestOutputParser(Output
                                      self.crashed,
                                      self.leaked)
         self.info("TinderboxPrint: %s<br/>%s\n" % (suite_name, summary))
 
 
 class EmulatorMixin(object):
     """ Currently dependent on both TooltoolMixin and TestingMixin)"""
 
-    def install_emulator_from_tooltool(self, manifest_path):
+    def install_emulator_from_tooltool(self, manifest_path, do_unzip=True):
         dirs = self.query_abs_dirs()
         if self.tooltool_fetch(manifest_path, output_dir=dirs['abs_work_dir']):
             self.fatal("Unable to download emulator via tooltool!")
-        unzip = self.query_exe("unzip")
-        unzip_cmd = [unzip, '-q', os.path.join(dirs['abs_work_dir'], "emulator.zip")]
-        self.run_command(unzip_cmd, cwd=dirs['abs_emulator_dir'], halt_on_failure=True,
-                         fatal_exit_code=3)
+        if do_unzip:
+            unzip = self.query_exe("unzip")
+            unzip_cmd = [unzip, '-q', os.path.join(dirs['abs_work_dir'], "emulator.zip")]
+            self.run_command(unzip_cmd, cwd=dirs['abs_emulator_dir'], halt_on_failure=True,
+                             fatal_exit_code=3)
 
     def install_emulator(self):
         dirs = self.query_abs_dirs()
         self.mkdir_p(dirs['abs_emulator_dir'])
         if self.config.get('emulator_url'):
             self._download_unzip(self.config['emulator_url'], dirs['abs_emulator_dir'])
         elif self.config.get('emulator_manifest'):
             manifest_path = self.create_tooltool_manifest(self.config['emulator_manifest'])
-            self.install_emulator_from_tooltool(manifest_path)
+            do_unzip = True
+            if 'unpack' in self.config['emulator_manifest']:
+                do_unzip = False
+            self.install_emulator_from_tooltool(manifest_path, do_unzip)
         elif self.buildbot_config:
             props = self.buildbot_config.get('properties')
             url = 'https://hg.mozilla.org/%s/raw-file/%s/b2g/test/emulator.manifest' % (
                 props['repo_path'], props['revision'])
             manifest_path = self.download_file(url,
                                                file_name='tooltool.tt',
                                                parent_dir=dirs['abs_work_dir'])
             if not manifest_path:
                 self.fatal("Can't download emulator manifest from %s" % url)
             self.install_emulator_from_tooltool(manifest_path)
         else:
             self.fatal("Can't get emulator; set emulator_url or emulator_manifest in the config!")
+        if self.config.get('tools_manifest'):
+            manifest_path = self.create_tooltool_manifest(self.config['tools_manifest'])
+            do_unzip = True
+            if 'unpack' in self.config['tools_manifest']:
+                do_unzip = False
+            self.install_emulator_from_tooltool(manifest_path, do_unzip)
--- a/scripts/android_emulator_unittest.py
+++ b/scripts/android_emulator_unittest.py
@@ -101,17 +101,17 @@ class AndroidEmulatorTest(BlobUploadMixi
                 # IP address of the host as seen from the emulator
                 'remote_webserver': '10.0.2.2',
             }
         )
 
         # these are necessary since self.config is read only
         c = self.config
         abs_dirs = self.query_abs_dirs()
-        self.adb_path = c.get('adb_path', self._query_adb())
+        self.adb_path = self.query_exe('adb')
         self.installer_url = c.get('installer_url')
         self.installer_path = c.get('installer_path')
         self.test_url = c.get('test_url')
         self.test_manifest = c.get('test_manifest')
         self.robocop_url = c.get('robocop_url')
         self.robocop_path = os.path.join(abs_dirs['abs_work_dir'], "robocop.apk")
         self.host_utils_url = c.get('host_utils_url')
         self.minidump_stackwalk_path = c.get("minidump_stackwalk_path")
@@ -448,19 +448,16 @@ class AndroidEmulatorTest(BlobUploadMixi
             argname = arg.split('=')[0]
             # only add the extra arg if it wasn't already defined by in-tree configs
             if any(a.split('=')[0] == argname for a in cmd):
                 continue
             cmd.append(arg)
 
         return cmd
 
-    def _query_adb(self):
-        return self.which('adb') or os.getenv('ADB_PATH')
-
     def preflight_run_tests(self):
         super(AndroidEmulatorTest, self).preflight_run_tests()
 
         if not os.path.isfile(self.adb_path):
             self.fatal("The adb binary '%s' is not a valid file!" % self.adb_path)
 
     def _trigger_test(self, suite_name, emulator_index):
         """
@@ -564,17 +561,17 @@ class AndroidEmulatorTest(BlobUploadMixi
 
     def start_emulators(self):
         '''
         This action starts the emulators and redirects the two SUT ports for each one of them
         '''
         assert len(self.test_suites) <= len(self.emulators), \
             "We can't run more tests that the number of emulators we start"
 
-        if 'emulator_url' in self.config or 'emulator_manifest' in self.config:
+        if 'emulator_url' in self.config or 'emulator_manifest' in self.config or 'tools_manifest' in self.config:
             self.install_emulator()
 
         if not self.config.get("developer_mode"):
             # We kill compiz because it sometimes prevents us from starting the emulators
             self._kill_processes("compiz")
             self._kill_processes("xpcshell")
 
         # We add a symlink for libGL.so because the emulator dlopen()s it by that name
@@ -691,17 +688,17 @@ class AndroidEmulatorTest(BlobUploadMixi
             config = {
                 'device-id': emulator["device_id"],
                 'enable_automation': True,
                 'device_package_name': self._query_package_name()
             }
             config = dict(config.items() + self.config.items())
 
             self.info("Creating ADBDevicHandler for %s with config %s" % (emulator["name"], config))
-            dh = ADBDeviceHandler(config=config, log_obj=self.log_obj)
+            dh = ADBDeviceHandler(config=config, log_obj=self.log_obj, script_obj=self)
             dh.device_id = emulator['device_id']
 
             # Install Fennec
             self.info("Installing Fennec for %s" % emulator["name"])
             dh.install_app(self.installer_path)
 
             # Install the robocop apk if required
             if suite_name.startswith('robocop'):