Bug 1495430 - Use AndroidMixin in web_platform_tests.py mozharness script; r=bc,automatedtester
authorGeoff Brown <gbrown@mozilla.com>
Thu, 11 Oct 2018 09:47:56 -0600
changeset 499162 5095c0fdd000de42f501b7fea62a313871eca327
parent 499161 912784581287a7b736c4ec25d4b85fe562e868fa
child 499163 e68617c25cbeaa0db57264bab8359f2073d07924
push id1864
push userffxbld-merge
push dateMon, 03 Dec 2018 15:51:40 +0000
treeherdermozilla-release@f040763d99ad [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbc, automatedtester
bugs1495430
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1495430 - Use AndroidMixin in web_platform_tests.py mozharness script; r=bc,automatedtester
testing/mozharness/configs/web_platform_tests/prod_config_android.py
testing/mozharness/scripts/web_platform_tests.py
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/web_platform_tests/prod_config_android.py
@@ -0,0 +1,50 @@
+# ***** BEGIN LICENSE BLOCK *****
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+# ***** END LICENSE BLOCK *****
+import os
+
+config = {
+    "options": [
+        "--prefs-root=%(test_path)s/prefs",
+        "--processes=1",
+        "--config=%(test_path)s/wptrunner.ini",
+        "--ca-cert-path=%(test_path)s/tests/tools/certs/cacert.pem",
+        "--host-key-path=%(test_path)s/tests/tools/certs/web-platform.test.key",
+        "--host-cert-path=%(test_path)s/tests/tools/certs/web-platform.test.pem",
+        "--certutil-binary=%(xre_path)s/certutil",
+        "--product=fennec",
+    ],
+    "avds_dir": "/builds/worker/workspace/build/.android",
+    "binary_path": "/tmp",
+    "download_minidump_stackwalk": False,
+    "emulator_avd_name": "test-1",
+    "emulator_extra_args": "-gpu swiftshader_indirect -skip-adb-auth -verbose -show-kernel -use-system-libs -ranchu -selinux permissive -memory 3072 -cores 4",
+    "emulator_manifest": """
+        [
+        {
+        "size": 135064025,
+        "digest": "125678c5b0d93ead8bbf01ba94253e532909417b40637460624cfca34e92f431534fc77a0225e9c4728dcbcf2884a8f7fa1ee059efdfa82d827ca20477d41705",
+        "algorithm": "sha512",
+        "filename": "android-sdk_r27.1.12-linux-x86emu.tar.gz",
+        "unpack": "True"
+        }
+        ] """,
+    "emulator_process_name": "emulator64-x86",
+    "env": {
+        "PATH": "%(PATH)s:%(abs_work_dir)s/android-sdk-linux/emulator:%(abs_work_dir)s/android-sdk-linux/tools:%(abs_work_dir)s/android-sdk-linux/platform-tools",
+    },
+    "exes": {
+        'adb': '%(abs_work_dir)s/android-sdk-linux/platform-tools/adb',
+    },
+    "geckodriver": "%(abs_test_bin_dir)s/geckodriver",
+    "hostutils_manifest_path": "testing/config/tooltool-manifests/linux64/hostutils.manifest",
+    "log_tbpl_level": "info",
+    "log_raw_level": "info",
+    "minidump_stackwalk_path": "/usr/local/bin/linux64-minidump_stackwalk",
+    "per_test_category": "web-platform",
+    "tooltool_cache": os.environ.get("TOOLTOOL_CACHE"),
+    "tooltool_manifest_path": "testing/config/tooltool-manifests/androidx86_7_0/releng.manifest",
+    "tooltool_servers": ['http://relengapi/tooltool/'],
+}
--- a/testing/mozharness/scripts/web_platform_tests.py
+++ b/testing/mozharness/scripts/web_platform_tests.py
@@ -13,28 +13,29 @@ from datetime import datetime, timedelta
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 import mozinfo
 
 from mozharness.base.errors import BaseErrorList
 from mozharness.base.script import PreScriptAction
 from mozharness.base.vcs.vcsbase import MercurialScript
+from mozharness.mozilla.testing.android import AndroidMixin
 from mozharness.mozilla.testing.testbase import TestingMixin, testing_config_options
 from mozharness.mozilla.testing.codecoverage import (
     CodeCoverageMixin,
     code_coverage_config_options
 )
 from mozharness.mozilla.testing.errors import HarnessErrorList
 
 from mozharness.mozilla.structuredlog import StructuredOutputParser
 from mozharness.base.log import INFO
 
 
-class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin):
+class WebPlatformTest(TestingMixin, MercurialScript, CodeCoverageMixin, AndroidMixin):
     config_options = [
         [['--test-type'], {
             "action": "extend",
             "dest": "test_type",
             "help": "Specify the test types to run."}
          ],
         [['--e10s'], {
             "action": "store_true",
@@ -99,33 +100,39 @@ class WebPlatformTest(TestingMixin, Merc
     ] + copy.deepcopy(testing_config_options) + \
         copy.deepcopy(code_coverage_config_options)
 
     def __init__(self, require_config_file=True):
         super(WebPlatformTest, self).__init__(
             config_options=self.config_options,
             all_actions=[
                 'clobber',
+                'setup-avds',
+                'start-emulator',
                 'download-and-extract',
                 'create-virtualenv',
                 'pull',
+                'verify-device',
                 'install',
                 'run-tests',
             ],
             require_config_file=require_config_file,
             config={'require_test_zip': True})
 
         # Surely this should be in the superclass
         c = self.config
         self.installer_url = c.get('installer_url')
         self.test_url = c.get('test_url')
         self.test_packages_url = c.get('test_packages_url')
         self.installer_path = c.get('installer_path')
         self.binary_path = c.get('binary_path')
         self.abs_app_dir = None
+        self.xre_path = None
+        if self.is_emulator:
+            self.device_serial = 'emulator-5554'
 
     def query_abs_app_dir(self):
         """We can't set this in advance, because OSX install directories
         change depending on branding and opt/debug.
         """
         if self.abs_app_dir:
             return self.abs_app_dir
         if not self.binary_path:
@@ -139,16 +146,20 @@ class WebPlatformTest(TestingMixin, Merc
         abs_dirs = super(WebPlatformTest, self).query_abs_dirs()
 
         dirs = {}
         dirs['abs_app_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'application')
         dirs['abs_test_install_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'tests')
         dirs['abs_test_bin_dir'] = os.path.join(dirs['abs_test_install_dir'], 'bin')
         dirs["abs_wpttest_dir"] = os.path.join(dirs['abs_test_install_dir'], "web-platform")
         dirs['abs_blob_upload_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'blobber_upload_dir')
+        if self.is_android:
+            dirs['abs_xre_dir'] = os.path.join(abs_dirs['abs_work_dir'], 'hostutils')
+        if self.is_emulator:
+            dirs['abs_avds_dir'] = self.config.get('avds_dir')
 
         abs_dirs.update(dirs)
         self.abs_dirs = abs_dirs
 
         return self.abs_dirs
 
     @PreScriptAction('create-virtualenv')
     def _pre_create_virtualenv(self, action):
@@ -182,17 +193,18 @@ class WebPlatformTest(TestingMixin, Merc
 
         dirs = self.query_abs_dirs()
         abs_app_dir = self.query_abs_app_dir()
         str_format_values = {
             'binary_path': self.binary_path,
             'test_path': dirs["abs_wpttest_dir"],
             'test_install_path': dirs["abs_test_install_dir"],
             'abs_app_dir': abs_app_dir,
-            'abs_work_dir': dirs["abs_work_dir"]
+            'abs_work_dir': dirs["abs_work_dir"],
+            'xre_path': self.xre_path,
         }
 
         cmd = [self.query_python_path('python'), '-u']
         cmd.append(os.path.join(dirs["abs_wpttest_dir"], run_file_name))
 
         # Make sure that the logging directory exists
         if self.mkdir_p(dirs["abs_blob_upload_dir"]) == -1:
             self.fatal("Could not create blobber upload directory")
@@ -209,16 +221,20 @@ class WebPlatformTest(TestingMixin, Merc
                                                        "wpt_errorsummary.log"),
                 "--binary=%s" % self.binary_path,
                 "--symbols-path=%s" % self.query_symbols_url(),
                 "--stackwalk-binary=%s" % self.query_minidump_stackwalk(),
                 "--stackfix-dir=%s" % os.path.join(dirs["abs_test_install_dir"], "bin"),
                 "--run-by-dir=%i" % (3 if not mozinfo.info["asan"] else 0),
                 "--no-pause-after-test"]
 
+        if self.is_android:
+            cmd += ["--device-serial=%s" % self.device_serial]
+            cmd += ["--package-name=%s" % self.query_package_name()]
+
         if not sys.platform.startswith("linux"):
             cmd += ["--exclude=css"]
 
         for test_type in test_types:
             cmd.append("--test-type=%s" % test_type)
 
         if c['extra_prefs']:
             cmd.extend(['--setpref={}'.format(p) for p in c['extra_prefs']])
@@ -277,18 +293,29 @@ class WebPlatformTest(TestingMixin, Merc
                           "config/*",
                           "mozbase/*",
                           "marionette/*",
                           "tools/*",
                           "web-platform/*",
                           "mozpack/*",
                           "mozbuild/*"],
             suite_categories=["web-platform"])
+        if self.is_android:
+            dirs = self.query_abs_dirs()
+            self.xre_path = self.download_hostutils(dirs['abs_xre_dir'])
+
+    def install(self):
+        if self.is_android:
+            self.install_apk(self.installer_path)
+        else:
+            super(WebPlatformTest, self).install()
 
     def _install_fonts(self):
+        if self.is_android:
+            return
         # Ensure the Ahem font is available
         dirs = self.query_abs_dirs()
 
         if not sys.platform.startswith("darwin"):
             font_path = os.path.join(os.path.dirname(self.binary_path), "fonts")
         else:
             font_path = os.path.join(os.path.dirname(self.binary_path), os.pardir,
                                      "Resources", "res", "fonts")
@@ -322,16 +349,19 @@ class WebPlatformTest(TestingMixin, Merc
             env['MOZ_HEADLESS_WIDTH'] = self.config['headless_width']
             env['MOZ_HEADLESS_HEIGHT'] = self.config['headless_height']
 
         if self.config['single_stylo_traversal']:
             env['STYLO_THREADS'] = '1'
         else:
             env['STYLO_THREADS'] = '4'
 
+        if self.is_android:
+            env['ADB_PATH'] = self.adb_path
+
         env = self.query_env(partial_env=env, log_level=INFO)
 
         start_time = datetime.now()
         max_per_test_time = timedelta(minutes=60)
         max_per_test_tests = 10
         if self.per_test_coverage:
             max_per_test_tests = 30
         executed_tests = 0