Bug 1466648 - add unity3d benchmark to raptor/android; r=bc,jmaher
authorRob Wood <rwood@mozilla.com>
Mon, 01 Oct 2018 19:01:57 +0000
changeset 494775 3cc5501b986679f776c335747079610ed06ad6ee
parent 494774 9d5e7b73af8f2576a76a1014b392fc4258c2683e
child 494776 05df95ba3a989632898ca6d379cf7cce580f9a76
push id9984
push userffxbld-merge
push dateMon, 15 Oct 2018 21:07:35 +0000
treeherdermozilla-beta@183d27ea8570 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersbc, jmaher
bugs1466648
milestone64.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1466648 - add unity3d benchmark to raptor/android; r=bc,jmaher Differential Revision: https://phabricator.services.mozilla.com/D7114
taskcluster/ci/fetch/benchmarks.yml
taskcluster/ci/test/raptor.yml
taskcluster/ci/test/test-platforms.yml
taskcluster/ci/test/test-sets.yml
taskcluster/taskgraph/transforms/job/mozharness_test.py
testing/raptor/mach_commands.py
testing/raptor/raptor/tests/raptor-speedometer.ini
testing/raptor/raptor/tests/raptor-unity-webgl.ini
--- a/taskcluster/ci/fetch/benchmarks.yml
+++ b/taskcluster/ci/fetch/benchmarks.yml
@@ -13,19 +13,19 @@ octane:
     url: https://github.com/mozilla/perf-automation/releases/download/octane-v1/octane-0f5b8d48d4a9.zip
     sha256: 38425ee1abfc5feca178b2f60fbd82b5873897c345112a85359be00024402f9f
     size: 1816138
 
 unity-webgl:
   description: unity-webgl benchmark
   fetch:
     type: static-url
-    url: https://github.com/mozilla/perf-automation/releases/download/unity-webgl-v1/unity-webgl-6beb3d3e22ab.zip
-    sha256: f71ee3a3f5b9513f041e1dd01c032d51f2071e1ad130e8ac2cf0c553c468b9ea
-    size: 27062962
+    url: https://github.com/mozilla/perf-automation/releases/download/unity-webgl-v3/unity-webgl-2720cdc79082.zip
+    sha256: 2d5a741cb3bfbd191d0aa2b4c4f71d615db84a0760e20c4e4cf701b3682df0d0
+    size: 27062969
 
 assorted-dom:
   description: assorted-dom benchmark
   fetch:
     type: static-url
     url: https://github.com/mozilla/perf-automation/releases/download/assorted-dom-v1/assorted-dom-4befd28725c6.zip
     sha256: e4eafe4a8e70c7ae6d42d668d3b1640b9fd9b696c486ff35aab754c368f78c2c
     size: 402665
--- a/taskcluster/ci/test/raptor.yml
+++ b/taskcluster/ci/test/raptor.yml
@@ -223,16 +223,34 @@ raptor-unity-webgl-firefox:
     max-run-time: 3600
     mozharness:
         extra-options:
             - --test=raptor-unity-webgl
     fetches:
         fetch:
             - unity-webgl
 
+raptor-unity-webgl-geckoview:
+    description: "Raptor Unity WebGL on Geckoview"
+    try-name: raptor-unity-webgl-geckoview
+    treeherder-symbol: Rap(ugl)
+    target: geckoview_example.apk
+    run-on-projects:
+        by-test-platform:
+            android-hw-.*arm7.*: ['try', 'mozilla-central']
+    max-run-time: 900
+    mozharness:
+        extra-options:
+            - --test=raptor-unity-webgl
+            - --app=geckoview
+            - --binary=org.mozilla.geckoview_example
+    fetches:
+        fetch:
+            - unity-webgl
+
 raptor-unity-webgl-chrome:
     description: "Raptor Unity WebGL on Chrome"
     try-name: raptor-unity-webgl-chrome
     treeherder-symbol: Rap-C(ugl)
     run-on-projects: ['try', 'mozilla-central']
     max-run-time: 3600
     mozharness:
         extra-options:
--- a/taskcluster/ci/test/test-platforms.yml
+++ b/taskcluster/ci/test/test-platforms.yml
@@ -351,24 +351,26 @@ android-em-7.0-x86/opt:
 # android-hw test platforms execute on real devices attached to Autophone hosts.
 
 # android-hw-g5-7-0 Motorola Moto G5 Android 7.0
 
 android-hw-g5-7-0-arm7-api-16/opt:
     build-platform: android-api-16/opt
     test-sets:
         - android-hw-arm7-raptor
+        - raptor-fetch-geckoview
 
 # android-hw-p2-8-0 Google Pixel 2 Android 8.0
 
 android-hw-p2-8-0-arm7-api-16/opt:
     build-platform: android-api-16/opt
     test-sets:
         - android-hw-arm7-opt-unittests
         - android-hw-arm7-raptor
+        - raptor-fetch-geckoview
 
 android-hw-p2-8-0-arm7-api-16/debug:
     build-platform: android-api-16/debug
     test-sets:
         - android-hw-arm7-debug-unittests
 
 android-hw-p2-8-0-android-aarch64/opt:
     build-platform: android-aarch64/opt
--- a/taskcluster/ci/test/test-sets.yml
+++ b/taskcluster/ci/test/test-sets.yml
@@ -103,16 +103,19 @@ raptor-chrome:
 # so these need to be separate sets.
 raptor-fetch-firefox:
     - raptor-unity-webgl-firefox
     - raptor-wasm-misc-firefox
     - raptor-wasm-misc-baseline-firefox
     - raptor-wasm-misc-ion-firefox
     - raptor-assorted-dom-firefox
 
+raptor-fetch-geckoview:
+    - raptor-unity-webgl-geckoview
+
 raptor-fetch-chrome:
     - raptor-unity-webgl-chrome
     - raptor-wasm-misc-chrome
     - raptor-assorted-dom-chrome
 
 awsy:
     - awsy
     - awsy-base
--- a/taskcluster/taskgraph/transforms/job/mozharness_test.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness_test.py
@@ -208,21 +208,21 @@ def mozharness_test_on_generic_worker(co
     taskdesc['scopes'].extend(
         ['generic-worker:os-group:{}/{}'.format(
             job['worker-type'],
             group
         ) for group in test['os-groups']])
 
     worker['os-groups'] = test['os-groups']
 
-    # run-as-administrator is a feature for workers with UAC enabled and as such should not be
-    # included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
-    # gecko Windows 10 workers, however this may be subject to change. Worker type
-    # environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
-    # See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
+    # run-as-administrator is a feature for workers with UAC enabled and as such should not be
+    # included in tasks on workers that have UAC disabled. Currently UAC is only enabled on
+    # gecko Windows 10 workers, however this may be subject to change. Worker type
+    # environment definitions can be found in https://github.com/mozilla-releng/OpenCloudConfig
+    # See https://docs.microsoft.com/en-us/windows/desktop/secauthz/user-account-control
     # for more information about UAC.
     if test.get('run-as-administrator', False):
         if job['worker-type'].startswith('aws-provisioner-v1/gecko-t-win10-64'):
             taskdesc['scopes'].extend(
                 ['generic-worker:run-as-administrator:{}'.format(job['worker-type'])])
             worker['run-as-administrator'] = True
         else:
             raise Exception('run-as-administrator not supported on {}'.format(job['worker-type']))
@@ -443,18 +443,23 @@ def mozharness_test_on_script_engine_aut
         "MOZ_NO_REMOTE": '1',
         "XPCOM_DEBUG_BREAK": 'warn',
         "NO_FAIL_ON_TEST_ERRORS": '1',
         "MOZ_HIDE_RESULTS_TABLE": '1',
         "MOZ_NODE_PATH": "/usr/local/bin/node",
         'MOZ_AUTOMATION': '1',
         'WORKSPACE': '/builds/worker/workspace',
         'TASKCLUSTER_WORKER_TYPE': job['worker-type'],
+    }
 
-    }
+    # for fetch tasks on mobile
+    if 'env' in job['worker'] and 'MOZ_FETCHES' in job['worker']['env']:
+        env['MOZ_FETCHES'] = job['worker']['env']['MOZ_FETCHES']
+        env['MOZ_FETCHES_DIR'] = job['worker']['env']['MOZ_FETCHES_DIR']
+
     # talos tests don't need Xvfb
     if is_talos:
         env['NEED_XVFB'] = 'false'
 
     script = 'test-linux.sh'
     worker['context'] = '{}/raw-file/{}/taskcluster/scripts/tester/{}'.format(
         config.params['head_repository'], config.params['head_rev'], script
     )
--- a/testing/raptor/mach_commands.py
+++ b/testing/raptor/mach_commands.py
@@ -18,17 +18,17 @@ import subprocess
 import mozfile
 from mach.decorators import CommandProvider, Command
 from mozboot.util import get_state_dir
 from mozbuild.base import MozbuildObject, MachCommandBase
 from mozbuild.base import MachCommandConditions as conditions
 
 HERE = os.path.dirname(os.path.realpath(__file__))
 BENCHMARK_REPOSITORY = 'https://github.com/mozilla/perf-automation'
-BENCHMARK_REVISION = '4befd28725c687b91ce749420eab29352ecbcab4'
+BENCHMARK_REVISION = '2720cdc790828952964524bb44ce8b4c14670e90'
 
 
 class RaptorRunner(MozbuildObject):
     def run_test(self, raptor_args, app=None):
         """
         We want to do couple of things before running raptor
         1. Clone mozharness
         2. Make config for raptor mozharness
--- a/testing/raptor/raptor/tests/raptor-speedometer.ini
+++ b/testing/raptor/raptor/tests/raptor-speedometer.ini
@@ -4,24 +4,24 @@
 
 # speedometer benchmark for firefox and chrome
 
 [DEFAULT]
 type =  benchmark
 test_url = http://localhost:<port>/Speedometer/index.html?raptor
 page_cycles = 5
 page_timeout = 180000
-unit = score
-subtest_unit = ms
-lower_is_better = false
-subtest_lower_is_better = true
+unit = score
+subtest_unit = ms
+lower_is_better = false
+subtest_lower_is_better = true
 alert_threshold = 2.0
 
 [raptor-speedometer-firefox]
 apps = firefox
 
 [raptor-speedometer-chrome]
 apps = chrome
 
 [raptor-speedometer-geckoview]
-page_timeout = 1800000 # temp 30 min for debugging
+page_timeout = 900000  # 15 min
 page_cycles = 1
 apps = geckoview
--- a/testing/raptor/raptor/tests/raptor-unity-webgl.ini
+++ b/testing/raptor/raptor/tests/raptor-unity-webgl.ini
@@ -7,8 +7,13 @@ unit = score
 lower_is_better = false
 alert_threshold = 2.0
 
 [raptor-unity-webgl-firefox]
 apps = firefox
 
 [raptor-unity-webgl-chrome]
 apps = chrome
+
+[raptor-unity-webgl-geckoview]
+page_timeout = 900000  # 15 min
+page_cycles = 1
+apps = geckoview