Bug 1188337 - Enable Mulet TaskCluster Gu and prepare for Gu-oop r=garndt,aus
authorAlexandre Lissy <lissyx@lissyx.dyndns.org>
Tue, 15 Sep 2015 14:08:01 -0700
changeset 295405 8cf7b6d58a2259d0e32e59e62651a3225eb63453
parent 295404 b40d8597756443334a92f56988b86317a7dcc7e9
child 295406 9cf189d8500382ff0d60e236e64cc1f454910176
push id5245
push userraliiev@mozilla.com
push dateThu, 29 Oct 2015 11:30:51 +0000
treeherdermozilla-beta@dac831dc1bd0 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersgarndt, aus
bugs1188337
milestone43.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1188337 - Enable Mulet TaskCluster Gu and prepare for Gu-oop r=garndt,aus
testing/mozharness/mozharness.json
testing/mozharness/scripts/gaia_unit.py
testing/taskcluster/tasks/branches/base_jobs.yml
testing/taskcluster/tasks/branches/try/job_flags.yml
testing/taskcluster/tasks/tests/mulet_gaia_unit.yml
testing/taskcluster/tasks/tests/mulet_gaia_unit_oop.yml
--- a/testing/mozharness/mozharness.json
+++ b/testing/mozharness/mozharness.json
@@ -1,4 +1,4 @@
 {
     "repo": "https://hg.mozilla.org/build/mozharness",
-    "revision": "9e3b69e89620"
+    "revision": "e218c0e82735"
 }
--- a/testing/mozharness/scripts/gaia_unit.py
+++ b/testing/mozharness/scripts/gaia_unit.py
@@ -2,16 +2,19 @@
 # ***** BEGIN LICENSE BLOCK *****
 # This Source Code Form is subject to the terms of the Mozilla Public
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 # ***** END LICENSE BLOCK *****
 
 import os
 import sys
+import glob
+import subprocess
+import json
 
 # load modules from parent dir
 sys.path.insert(1, os.path.dirname(sys.path[0]))
 
 from mozharness.mozilla.testing.gaia_test import GaiaTest
 from mozharness.mozilla.testing.unittest import TestSummaryOutputParserHelper
 
 
@@ -36,23 +39,58 @@ class GaiaUnitTest(GaiaTest):
                        xre_url=self.config.get('xre_url'),
                        debug=True)
 
         # build the testrunner command arguments
         python = self.query_python_path('python')
         cmd = [python, '-u', os.path.join(dirs['abs_runner_dir'],
                                           'gaia_unit_test',
                                           'main.py')]
-        binary = os.path.join(os.path.dirname(self.binary_path), 'b2g-bin')
+        executable = 'firefox'
+        if 'b2g' in self.binary_path:
+                executable = 'b2g-bin'
+
+        profile = os.path.join(dirs['abs_gaia_dir'], 'profile-debug')
+        binary = os.path.join(os.path.dirname(self.binary_path), executable)
         cmd.extend(self._build_arg('--binary', binary))
-        cmd.extend(self._build_arg('--profile', os.path.join(dirs['abs_gaia_dir'],
-                                                             'profile-debug')))
+        cmd.extend(self._build_arg('--profile', profile))
         cmd.extend(self._build_arg('--symbols-path', self.symbols_path))
         cmd.extend(self._build_arg('--browser-arg', self.config.get('browser_arg')))
 
+        # Add support for chunking
+        if self.config.get('total_chunks') and self.config.get('this_chunk'):
+                chunker = [ os.path.join(dirs['abs_gaia_dir'], 'bin', 'chunk'),
+                            self.config.get('total_chunks'), self.config.get('this_chunk') ]
+
+                disabled_tests = []
+                disabled_manifest = os.path.join(dirs['abs_runner_dir'],
+                                                 'gaia_unit_test',
+                                                 'disabled.json')
+                with open(disabled_manifest, 'r') as m:
+                    try:
+                        disabled_tests = json.loads(m.read())
+                    except:
+                        print "Error while decoding disabled.json; please make sure this file has valid JSON syntax."
+                        sys.exit(1)
+
+                # Construct a list of all tests
+                unit_tests  = []
+                for path in ('apps', 'tv_apps'):
+                    test_root    = os.path.join(dirs['abs_gaia_dir'], path)
+                    full_paths   = glob.glob(os.path.join(test_root, '*/test/unit/*_test.js'))
+                    unit_tests  += map(lambda x: os.path.relpath(x, test_root), full_paths)
+
+                # Remove the tests that are disabled
+                active_unit_tests = filter(lambda x: x not in disabled_tests, unit_tests)
+
+                # Chunk the list as requested
+                tests_to_run = subprocess.check_output(chunker + active_unit_tests).strip().split(' ')
+
+                cmd.extend(tests_to_run)
+
         output_parser = TestSummaryOutputParserHelper(config=self.config,
                                                       log_obj=self.log_obj,
                                                       error_list=self.error_list)
 
         upload_dir = self.query_abs_dirs()['abs_blob_upload_dir']
         if not os.path.isdir(upload_dir):
             self.mkdir_p(upload_dir)
 
--- a/testing/taskcluster/tasks/branches/base_jobs.yml
+++ b/testing/taskcluster/tasks/branches/base_jobs.yml
@@ -137,16 +137,20 @@ tests:
   gaia-ui-test-unit:
     allowed_build_tasks:
       tasks/builds/b2g_flame_kk_eng.yml:
         task: tasks/tests/flame_kk_gaia_ui_test_unit.yml
   gaia-unit:
     allowed_build_tasks:
       tasks/builds/b2g_desktop_opt.yml:
         task: tasks/tests/b2g_gaia_unit.yml
+  gaia-unit-oop:
+    allowed_build_tasks:
+      tasks/builds/mulet_linux.yml:
+        task: tasks/tests/mulet_gaia_unit_oop.yml
   marionette:
     allowed_build_tasks:
       tasks/builds/b2g_emulator_ics_opt.yml:
         task: tasks/tests/b2g_emulator_marionette.yml
   marionette-webapi:
     allowed_build_tasks:
       tasks/builds/b2g_emulator_ics_opt.yml:
         task: tasks/tests/b2g_emulator_marionette_webapi.yml
--- a/testing/taskcluster/tasks/branches/try/job_flags.yml
+++ b/testing/taskcluster/tasks/branches/try/job_flags.yml
@@ -236,16 +236,22 @@ tests:
       tasks/builds/b2g_desktop_opt.yml:
         task: tasks/tests/b2g_linter.yml
       tasks/builds/mulet_linux.yml:
         task: tasks/tests/mulet_linter.yml
   gaia-unit:
     allowed_build_tasks:
       tasks/builds/b2g_desktop_opt.yml:
         task: tasks/tests/b2g_gaia_unit.yml
+      tasks/builds/mulet_linux.yml:
+        task: tasks/tests/mulet_gaia_unit.yml
+  gaia-unit-oop:
+    allowed_build_tasks:
+      tasks/builds/mulet_linux.yml:
+        task: tasks/tests/mulet_gaia_unit_oop.yml
   marionette:
     allowed_build_tasks:
       tasks/builds/b2g_emulator_x86_kk_opt.yml:
         task: tasks/tests/b2g_emulator_marionette.yml
       tasks/builds/b2g_emulator_ics_opt.yml:
         task: tasks/tests/b2g_emulator_marionette.yml
   marionette-webapi:
     allowed_build_tasks:
new file mode 100644
--- /dev/null
+++ b/testing/taskcluster/tasks/tests/mulet_gaia_unit.yml
@@ -0,0 +1,44 @@
+---
+$inherits:
+  from: 'tasks/test.yml'
+task:
+  metadata:
+    name: '[TC] Mulet Gaia Unit Test'
+    description: Mulet Gaia Unit Test
+
+  payload:
+    command:
+      - entrypoint
+      - ./bin/pull_gaia.sh &&
+      - >
+        python ./mozharness/scripts/gaia_unit.py
+        --application firefox
+        --no-read-buildbot-config
+        --config-file b2g/gaia_unit_production_config.py
+        --config-file ./mozharness_configs/gaia_integration_override.py
+        --config-file ./mozharness_configs/remove_executables.py
+        --no-pull
+        --installer-url {{build_url}}
+        --test-packages-url {{test_packages_url}}
+        --download-symbols ondemand
+        --gaia-repo {{gaia_head_repository}}
+        --gaia-dir /home/worker
+        --xre-url https://queue.taskcluster.net/v1/task/wXAHAaxDQpqxoWF1iljJjg/runs/0/artifacts/public/cache/xulrunner-sdk-40.zip
+        --total-chunk={{total_chunks}}
+        --this-chunk={{chunk}}
+    artifacts:
+      'public/build':
+        type: directory
+        path: '/home/worker/artifacts/'
+        expires: '{{#from_now}}1 year{{/from_now}}'
+
+  extra:
+    treeherderEnv:
+      - production
+      - staging
+    chunks:
+      total: 10
+    treeherder:
+      groupSymbol: Gu
+      groupName: Mulet gaia unit tests
+      symbol: {{chunk}}
new file mode 100644
--- /dev/null
+++ b/testing/taskcluster/tasks/tests/mulet_gaia_unit_oop.yml
@@ -0,0 +1,44 @@
+---
+$inherits:
+  from: 'tasks/test.yml'
+task:
+  metadata:
+    name: '[TC] Mulet Gaia Unit Test OOP'
+    description: Mulet Gaia Unit Test OOP
+
+  payload:
+    command:
+      - entrypoint
+      - ./bin/pull_gaia.sh &&
+      - >
+        python ./mozharness/scripts/gaia_unit.py
+        --application firefox
+        --no-read-buildbot-config
+        --config-file b2g/gaia_unit_production_config.py
+        --config-file ./mozharness_configs/gaia_integration_override.py
+        --config-file ./mozharness_configs/remove_executables.py
+        --no-pull
+        --browser-arg -oop
+        --installer-url {{build_url}}
+        --test-packages-url {{test_packages_url}}
+        --download-symbols ondemand
+        --gaia-repo {{gaia_head_repository}}
+        --gaia-dir /home/worker
+        --xre-url https://queue.taskcluster.net/v1/task/wXAHAaxDQpqxoWF1iljJjg/runs/0/artifacts/public/cache/xulrunner-sdk-40.zip
+        --total-chunk={{total_chunks}}
+        --this-chunk={{chunk}}
+    artifacts:
+      'public/build':
+        type: directory
+        path: '/home/worker/artifacts/'
+        expires: '{{#from_now}}1 year{{/from_now}}'
+
+  extra:
+    treeherderEnv:
+      - staging
+    chunks:
+      total: 10
+    treeherder:
+      groupSymbol: Gu-oop
+      groupName: Mulet Gaia unit tests OOP
+      symbol: {{chunk}}