Bug 910092 - misc residual changes to android emulator test config, AVD build script and unit test script, r=gbrown
authorGraydon Hoare <graydon@mozilla.com>
Thu, 30 Jan 2014 15:59:56 -0800
changeset 2357 e7fa438cddd9d9a7089609fc2d1b048a0a532c73
parent 2355 98c4d0a88fe0dd2747046b6bd189bd0d0433c381
child 2358 e7dc863f3d76eedbeb0aac9b4c964fc0ddf973d7
push id1694
push usergraydon@mozilla.com
push dateFri, 31 Jan 2014 00:03:11 +0000
reviewersgbrown
bugs910092
Bug 910092 - misc residual changes to android emulator test config, AVD build script and unit test script, r=gbrown
configs/android/androidarm.py
scripts/android_emulator_build.py
scripts/android_emulator_unittest.py
new file mode 100644
--- /dev/null
+++ b/configs/android/androidarm.py
@@ -0,0 +1,168 @@
+import os
+
+config = {
+    "buildbot_json_path": "buildprops.json",
+    "host_utils_url": "http://bm-remote.build.mozilla.org/tegra/tegra-host-utils.Linux.742597.zip",
+    "robocop_package_name": "org.mozilla.roboexample.test",
+    "device_ip": "127.0.0.1",
+    "default_sut_port1": "20701",
+    "default_sut_port2": "20700", # does not prompt for commands
+    "tooltool_url": "http://tooltool.pvt.build.mozilla.org/build/sha512",
+    "tooltool_cache_path": "/builds/slave/talos-slave/cached",
+    "tooltool_cacheable_artifacts": {
+        "avd_tar_ball": ("AVDs-armv7a-gingerbread-build-2014-01-23-ubuntu.tar.gz",
+            "7140e026b7b747236545dc30e377a959b0bdf91bb4d70efd7f97f92fce12a9196042503124b8df8d30c2d97b7eb5f9df9556afdffa0b5d9625008aead305c32b"),
+    },
+    ".avds_dir": "/home/cltbld/.android",
+    "emulator_process_name": "emulator64-arm",
+    "emulator_cpu": "cortex-a9",
+    "exes": {
+        'adb': '/tools/android-sdk18/platform-tools/adb',
+        'python': '/tools/buildbot/bin/python',
+        'virtualenv': ['/tools/buildbot/bin/python', '/tools/misc-python/virtualenv.py'],
+    },
+    "env": {
+        "DISPLAY": ":0.0",
+        "PATH": "%(PATH)s:/tools/android-sdk18/tools:/tools/android-sdk18/platform-tools",
+        "MINIDUMP_STACKWALK": "/home/cltbld/talos-slave/test/build/venv/lib/python2.7/site-packages/talos/breakpad/linux64/minidump_stackwalk",
+        "MINIDUMP_SAVEPATH": "%(abs_work_dir)s/../minidumps"
+    },
+    "default_actions": [
+        'clobber',
+        'read-buildbot-config',
+        'download-cacheable-artifacts',
+        'setup-avds',
+        'start-emulators',
+        'download-and-extract',
+        'create-virtualenv',
+        'install',
+        'run-tests',
+        'stop-emulators',
+    ],
+    "emulators": [
+        {
+            "name": "test-1",
+            "device_id": "emulator-5554",
+            "http_port": "8854", # starting http port to use for the mochitest server
+            "ssl_port": "4454", # starting ssl port to use for the server
+            "emulator_port": 5554,
+            "sut_port1": 20701,
+            "sut_port2": 20700
+        },
+        {
+            "name": "test-2",
+            "device_id": "emulator-5556",
+            "http_port": "8856", # starting http port to use for the mochitest server
+            "ssl_port": "4456", # starting ssl port to use for the server
+            "emulator_port": 5556,
+            "sut_port1": 20703,
+            "sut_port2": 20702
+        },
+        {
+            "name": "test-3",
+            "device_id": "emulator-5558",
+            "http_port": "8858", # starting http port to use for the mochitest server
+            "ssl_port": "4458", # starting ssl port to use for the server
+            "emulator_port": 5558,
+            "sut_port1": 20705,
+            "sut_port2": 20704
+        },
+        {
+            "name": "test-4",
+            "device_id": "emulator-5560",
+            "http_port": "8860", # starting http port to use for the mochitest server
+            "ssl_port": "4460", # starting ssl port to use for the server
+            "emulator_port": 5560,
+            "sut_port1": 20707,
+            "sut_port2": 20706
+        }
+    ],
+    "test_suite_definitions": {
+        "jsreftest": {
+            "category": "reftest",
+            "extra_args": ["../jsreftest/tests/jstests.list",
+                "--extra-profile-file=jsreftest/tests/user.js"]
+        },
+        "mochitest-1": {
+            "category": "mochitest",
+            "extra_args": ["--total-chunks", "2", "--this-chunk", "1", "--run-only-tests", "androidx86.json"],
+        },
+        "mochitest-2": {
+            "category": "mochitest",
+            "extra_args": ["--total-chunks", "2", "--this-chunk", "2", "--run-only-tests", "androidx86.json"],
+        },
+        "mochitest-gl": {
+            "category": "mochitest",
+            "extra_args": ["--test-path", "content/canvas/test/webgl"],
+        },
+        "reftest-1": {
+            "category": "reftest",
+            "extra_args": ["--total-chunks", "3", "--this-chunk", "1",
+                "tests/layout/reftests/reftest.list"]
+        },
+        "reftest-2": {
+            "category": "reftest",
+            "extra_args": ["--total-chunks", "3", "--this-chunk", "2",
+                "tests/layout/reftests/reftest.list"]
+        },
+        "reftest-3": {
+            "category": "reftest",
+            "extra_args": ["--total-chunks", "3", "--this-chunk", "3",
+                "tests/layout/reftests/reftest.list"]
+        },
+        "crashtest": {
+            "category": "reftest",
+            "extra_args": ["tests/testing/crashtest/crashtests.list"]
+        },
+        "xpcshell": {
+            "category": "xpcshell",
+            "extra_args": ["--manifest", "tests/xpcshell_android.ini"]
+        },
+        "robocop-1": {
+            "category": "mochitest",
+            "extra_args": ["--total-chunks", "3", "--this-chunk", "1", "--robocop-path=../..",
+                "--robocop-ids=fennec_ids.txt", "--robocop=robocop.ini"],
+        },
+        "robocop-2": {
+            "category": "mochitest",
+            "extra_args": ["--total-chunks", "3", "--this-chunk", "2", "--robocop-path=../..",
+                "--robocop-ids=fennec_ids.txt", "--robocop=robocop.ini"],
+        },
+        "robocop-3": {
+            "category": "mochitest",
+            "extra_args": ["--total-chunks", "3", "--this-chunk", "3", "--robocop-path=../..",
+                "--robocop-ids=fennec_ids.txt", "--robocop=robocop.ini"],
+        },
+    }, # end of "test_definitions"
+    "suite_definitions": {
+        "mochitest": {
+            "run_filename": "runtestsremote.py",
+            "options": ["--autorun", "--close-when-done", "--dm_trans=sut",
+                "--console-level=INFO", "--app=%(app)s", "--remote-webserver=%(remote_webserver)s",
+                "--xre-path=%(xre_path)s", "--utility-path=%(utility_path)s",
+                "--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s",
+                "--http-port=%(http_port)s", "--ssl-port=%(ssl_port)s",
+                "--certificate-path=%(certs_path)s", "--symbols-path=%(symbols_path)s"
+            ],
+        },
+        "reftest": {
+            "run_filename": "remotereftest.py",
+            "options": [ "--app=%(app)s", "--ignore-window-size",
+                "--bootstrap", "--enable-privilege",
+                "--remote-webserver=%(remote_webserver)s", "--xre-path=%(xre_path)s",
+                "--utility-path=%(utility_path)s", "--deviceIP=%(device_ip)s",
+                "--devicePort=%(device_port)s", "--http-port=%(http_port)s",
+                "--ssl-port=%(ssl_port)s", "--httpd-path", "reftest/components",
+                "--symbols-path=%(symbols_path)s",
+            ],
+        },
+        "xpcshell": {
+            "run_filename": "remotexpcshelltests.py",
+            "options": ["--deviceIP=%(device_ip)s", "--devicePort=%(device_port)s",
+                "--xre-path=%(xre_path)s", "--testing-modules-dir=%(modules_dir)s",
+                "--apk=%(installer_path)s", "--no-logfiles",
+                "--symbols-path=%(symbols_path)s",
+            ],
+        },
+    }, # end of "suite_definitions"
+}
--- a/scripts/android_emulator_build.py
+++ b/scripts/android_emulator_build.py
@@ -164,17 +164,17 @@ class EmulatorBuild(BaseScript, PurgeMix
             "help": "android version to build (eg. 2.3.7, 4.0, 4.3.1, gingerbread, ics, jb; default gingerbread)",
         }],
         [["--android-tag"], {
             "dest": "android_tag",
             "help": "android tag to check out (eg. android-2.3.7_r1; default inferred from --android-version)",
         }],
         [["--patch"], {
             "dest": "patch",
-            "help": "'dir=url' of patch to apply to AOSP before building (eg. development=http://foo.com/bar.patch; default inferred)",
+            "help": "'dir=url' comma-separated list of patches to apply to AOSP before building (eg. development=http://foo.com/bar.patch; default inferred)",
         }],
         [["--android-apilevel"], {
             "dest": "android_apilevel",
             "help": "android API-level to build AVD for (eg. 10, 14, 18; default inferred from --android-version)",
         }],
         [["--android-url"], {
             "dest": "android_url",
             "help": "where to fetch AOSP from, default https://android.googlesource.com/platform/manifest",
@@ -259,21 +259,21 @@ class EmulatorBuild(BaseScript, PurgeMix
 
         if not (platform.machine() in ['i386', 'i486', 'i586', 'i686', 'x86_64']):
             self.fatal("this script only works on x86 and x86_64")
 
         self.tag = self.config['android_tag']
         if self.tag == 'inferred':
             self.tag = self.select_android_tag(self.config['android_version'])
 
-        self.patch = self.config['patch']
-        if self.patch == 'inferred':
-            self.patch = self.select_patch(self.tag)
+        self.patches = self.config['patch']
+        if self.patches == 'inferred':
+            self.patches = self.select_patches(self.tag)
         else:
-            self.patch = self.patch.split('=')
+            self.patches = [x.split('=') for x in self.patches.split(',')]
 
         self.apilevel = self.config['android_apilevel']
         if self.apilevel == 'inferred':
             self.apilevel = self.android_apilevel(self.tag)
 
         self.workdir = os.path.abspath(self.config['work_dir'])
         self.bindir = os.path.join(self.workdir, "bin")
         self.aospdir = os.path.join(self.workdir, "aosp")
@@ -419,28 +419,29 @@ class EmulatorBuild(BaseScript, PurgeMix
         self.mkdir_p(self.workdir)
         self.run_command(["git", "clone",
                           "https://github.com/wlach/orangutan.git"],
                          cwd=self.workdir,
                          halt_on_failure=True)
 
 
     def patch_aosp(self):
-        if self.patch != None:
-            projectdir = self.patch[0]
-            url = self.patch[1]
-            patchdir = os.path.join(self.aospdir, projectdir)
-            self.info("downloading and applying AOSP patch %s to %s" % (url, patchdir))
-            self.download_file(url,
-                               file_name='aosp.patch',
-                               parent_dir=self.workdir)
-            self.run_command(['patch', '-p1',
-                              '-i', os.path.join(self.workdir, 'aosp.patch')],
-                             cwd=patchdir,
-                             halt_on_failure=True)
+        if self.patches != None:
+            for patch in self.patches:
+                projectdir = patch[0]
+                url = patch[1]
+                patchdir = os.path.join(self.aospdir, projectdir)
+                self.info("downloading and applying AOSP patch %s to %s" % (url, patchdir))
+                self.download_file(url,
+                                   file_name='aosp.patch',
+                                   parent_dir=self.workdir)
+                self.run_command(['patch', '-p1',
+                                  '-i', os.path.join(self.workdir, 'aosp.patch')],
+                                 cwd=patchdir,
+                                 halt_on_failure=True)
 
     def build_aosp(self):
 
         self.patch_aosp()
 
         arch = None
         variant = None
         abi = None
@@ -462,18 +463,19 @@ class EmulatorBuild(BaseScript, PurgeMix
         if abi2 != "":
             abi2 = " TARGET_CPU_ABI2=" + abi2
 
         env = { "BUILD_EMULATOR_OPENGL": "true",
                 "BUILD_EMULATOR_OPENGL_DRIVER": "true" }
 
         self.run_command(["/bin/bash", "-c",
                           ". build/envsetup.sh "
-                          "&& lunch full-eng "
+                          "&& lunch sdk-eng "
                           "&& make -j " + str(self.ncores) +
+                          " sdk"
                           " TARGET_ARCH=" + arch +
                           " TARGET_ARCH_VARIANT=" + variant +
                           " TARGET_CPU_ABI=" + abi +
                           abi2 +
                           " CC=gcc-4.4 CXX=g++-4.4"],
                          cwd=self.aospdir,
                          halt_on_failure=True,
                          partial_env=env)
@@ -921,21 +923,23 @@ class EmulatorBuild(BaseScript, PurgeMix
             vers = codenames[vers]
 
         for tag in reversed(tags):
             if tag.startswith(vers) or tag.startswith("android-" + vers):
                 self.info("selecting tag '%s' for version '%s'" % (tag, vers))
                 return tag
         self.fatal("requested android version '%s' doesn't match any tag" % vers)
 
-    def select_patch(self, tag):
+    def select_patches(self, tag):
         if tag == 'gingerbread':
             # FIXME: perhaps put this patch someplace more stable than bugzilla?
-            return ('development',
-                    'https://bug910092.bugzilla.mozilla.org/attachment.cgi?id=8361456')
+            return [('development',
+                     'https://bug910092.bugzilla.mozilla.org/attachment.cgi?id=8361456'),
+                    ('external/sqlite',
+                     'https://bug910092.bugzilla.mozilla.org/attachment.cgi?id=8364687')]
         return None
 
     def _post_fatal(self, message=None, exit_code=None):
         for i in ['emulator', 'emulator-arm', 'emulator-x86',
                   'emulator64', 'emulator64-arm', 'emulator64-x86']:
             self._kill_processes(i)
 
 
--- a/scripts/android_emulator_unittest.py
+++ b/scripts/android_emulator_unittest.py
@@ -192,25 +192,33 @@ class AndroidEmulatorTest(BlobUploadMixi
                 redirect_completed = True
         else:
             self.warning('failed to establish a telnet connection with the emulator')
         return redirect_completed
 
     def _launch_emulator(self, emulator_index):
         emulator = self.emulators[emulator_index]
         env = self.query_env()
+
+        # Set $LD_LIBRARY_PATH to self.dirs['abs_work_dir'] so that
+        # the emulator picks up the symlink to libGL.so.1 that we
+        # constructed in start_emulators.
+        env['LD_LIBRARY_PATH'] = self.abs_dirs['abs_work_dir']
+
         command = [
             "emulator", "-avd", emulator["name"],
             "-debug", "init,console,gles,memcheck,adbserver,adbclient,adb,avd_config,socket",
             "-port", str(emulator["emulator_port"]),
             # Enable kvm; -qemu arguments must be at the end of the command
-            "-qemu", "-m", "1024", "-enable-kvm"
+            "-qemu", "-m", "1024"
         ]
         if "emulator_cpu" in self.config:
-            command += ["-qemu", "-cpu", self.config["emulator_cpu"] ]
+            command += ["-cpu", self.config["emulator_cpu"] ]
+        else:
+            command += ["-enable-kvm"]
         tmp_file = tempfile.NamedTemporaryFile(mode='w')
         tmp_stdout = open(tmp_file.name, 'w')
         self.info("Created temp file %s." % tmp_file.name)
         self.info("Trying to start the emulator with this command: %s" % ' '.join(command))
         proc = subprocess.Popen(command, stdout=tmp_stdout, stderr=tmp_stdout, env=env)
         return {
             "process": proc,
             "tmp_file": tmp_file,
@@ -473,16 +481,35 @@ class AndroidEmulatorTest(BlobUploadMixi
         '''
         This action starts the emulators and redirects the two SUT ports for each one of them
         '''
         assert len(self.test_suites) <= len(self.emulators), \
             "We can't run more tests that the number of emulators we start"
         # We kill compiz because it sometimes prevents us from starting the emulators
         self._kill_processes("compiz")
 
+        # We add a symlink for libGL.so because the emulator dlopen()s it by that name
+        # even though the installed library on most systems without dev packages is
+        # libGL.so.1
+        linkfile = os.path.join(self.abs_dirs['abs_work_dir'], "libGL.so")
+        self.info("Attempting to establish symlink for %s" % linkfile)
+        try:
+            os.unlink(linkfile)
+        except OSError:
+            pass
+        for libdir in [ "/usr/lib/x86_64-linux-gnu/mesa",
+                        "/usr/lib/i386-linux-gnu/mesa",
+                        "/usr/lib/mesa" ]:
+            libfile = os.path.join(libdir, "libGL.so.1");
+            if os.path.exists(libfile):
+                self.info("Symlinking %s -> %s" % (linkfile, libfile))
+                self.mkdir_p(self.abs_dirs['abs_work_dir'])
+                os.symlink(libfile, linkfile)
+                break
+
         attempts = 0
         redirect_failed = True
         # Launch the required emulators and redirect the SUT ports for each. If unable
         # to redirect the SUT ports, kill the emulators and try starting them again.
         # The wait-and-retry logic is necessary because the emulators intermittently fail
         # to respond to telnet connections immediately after startup: bug 949740. In this
         # case, the emulator log shows "ioctl(KVM_CREATE_VM) failed: Interrupted system call".
         # We do not know how to avoid this error and the only way we have found to