Bug 1371038: Add configs for doing DMD builds r=Callek
authorChris AtLee <catlee@mozilla.com>
Thu, 22 Jun 2017 12:49:03 -0400
changeset 368164 a16d92beb70daa33c4e29bf25300ca259ed61b2a
parent 368163 e3de568a6fe49bec206e05db2d79f083d52f4665
child 368165 f323fe1f0d48ada9e36dba4cbd7a4b00ad61a73a
push id32158
push usercbook@mozilla.com
push dateTue, 11 Jul 2017 10:48:59 +0000
treeherdermozilla-central@5e2692f8a367 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersCallek
bugs1371038
milestone56.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
Bug 1371038: Add configs for doing DMD builds r=Callek This adds the mozconfigs, mozharness configs and taskcluster changes required to create optimized DMD builds for linux64, win32, win64 and macosx64. These builds will happen nightly on mozilla-central We also add support for custom build variants on Windows (or other generic worker environments). MozReview-Commit-ID: HrVT9PLSWVx
.cron.yml
browser/config/mozconfigs/linux32/opt-dmd
browser/config/mozconfigs/linux64/opt-dmd
browser/config/mozconfigs/macosx64/opt-dmd
browser/config/mozconfigs/win32/opt-dmd
browser/config/mozconfigs/win64/opt-dmd
browser/installer/package-manifest.in
taskcluster/ci/build/linux.yml
taskcluster/ci/build/macosx.yml
taskcluster/ci/build/windows.yml
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
taskcluster/taskgraph/transforms/job/mozharness.py
testing/mozharness/configs/builds/releng_sub_linux_configs/32_dmd.py
testing/mozharness/configs/builds/releng_sub_linux_configs/64_dmd.py
testing/mozharness/configs/builds/releng_sub_mac_configs/64_dmd.py
testing/mozharness/configs/builds/releng_sub_windows_configs/32_dmd.py
testing/mozharness/configs/builds/releng_sub_windows_configs/64_dmd.py
testing/mozharness/mozharness/mozilla/building/buildbase.py
--- a/.cron.yml
+++ b/.cron.yml
@@ -62,8 +62,20 @@ jobs:
           type: decision-task
           treeherder-symbol: Vg
           target-tasks-method: mochitest_valgrind
       run-on-projects:
           - mozilla-central
       when:
           - {hour: 16, minute: 0}
           - {hour: 4, minute: 0}
+
+    - name: nightly-dmd
+      job:
+          type: decision-task
+          treeherder-symbol: Ndmd
+          target-tasks-method: nightly_dmd
+      run-on-projects:
+          - mozilla-central
+      when:
+          by-project:
+            mozilla-central: [{hour: 10, minute: 0}]
+            # No default
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/linux32/opt-dmd
@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/linux32/nightly"
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/linux64/opt-dmd
@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/linux64/nightly"
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/macosx64/opt-dmd
@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/macosx64/nightly"
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/win32/opt-dmd
@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/win32/nightly"
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/win64/opt-dmd
@@ -0,0 +1,3 @@
+ac_add_options --enable-dmd
+
+. "$topsrcdir/browser/config/mozconfigs/win64/nightly"
--- a/browser/installer/package-manifest.in
+++ b/browser/installer/package-manifest.in
@@ -820,11 +820,19 @@ bin/libfreebl_32int64_3.so
 @RESPATH@/gmp-clearkey/0.1/manifest.json
 
 ; gfx
 #ifdef XP_WIN
 @RESPATH@/components/GfxSanityTest.manifest
 @RESPATH@/components/SanityTest.js
 #endif
 
-#ifdef MOZ_MULET
-#include ../../b2g/installer/package-manifest.in
+#ifdef MOZ_DMD
+; DMD
+@RESPATH@/dmd.py
+@RESPATH@/fix_stack_using_bpsyms.py
+#ifdef XP_MACOSX
+@RESPATH@/fix_macosx_stack.py
 #endif
+#ifdef XP_LINUX
+@RESPATH@/fix_linux_stack.py
+#endif
+#endif
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -17,16 +17,42 @@ linux64/opt:
         config:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
 
+linux64-dmd/opt:
+    description: "Linux64 DMD Opt"
+    index:
+        product: firefox
+        job-name: linux64-dmd-opt
+    treeherder:
+        platform: linux64-dmd/opt
+        symbol: tc(Bdmd)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-linux
+    worker:
+        max-run-time: 36000
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/linux64/releng.manifest"
+    run:
+        using: mozharness
+        actions: [get-secrets build check-test update]
+        config:
+            - builds/releng_base_linux_64_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: dmd
+        secrets: true
+        tooltool-downloads: public
+        need-xvfb: true
+    run-on-projects: [ ]
+
 linux64/pgo:
     description: "Linux64 PGO"
     index:
         product: firefox
         job-name: linux64-pgo
     treeherder:
         platform: linux64/pgo
         symbol: tc(B)
--- a/taskcluster/ci/build/macosx.yml
+++ b/taskcluster/ci/build/macosx.yml
@@ -40,16 +40,41 @@ macosx64/opt:
         using: mozharness
         actions: [get-secrets build update]
         config:
             - builds/releng_base_mac_64_cross_builds.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: internal
 
+macosx64-dmd/opt:
+    description: "MacOS X x64 DMD Cross-compile"
+    index:
+        product: firefox
+        job-name: macosx64-dmd-opt
+    treeherder:
+        platform: osx-10-10-dmd/opt
+        symbol: tc(Bdmd)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-macosx64
+    worker:
+        max-run-time: 36000
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/macosx64/cross-releng.manifest"
+    run:
+        using: mozharness
+        actions: [get-secrets build update]
+        config:
+            - builds/releng_base_mac_64_cross_builds.py
+        script: "mozharness/scripts/fx_desktop_build.py"
+        custom-build-variant-cfg: dmd
+        secrets: true
+        tooltool-downloads: internal
+    run-on-projects: [ ]
+
 macosx64-devedition/opt:
     description: "MacOS X Dev Edition x64"
     index:
         product: firefox
         job-name: macosx64-devedition-opt
     treeherder:
         platform: macosx64-devedition/opt
         symbol: tc(B)
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -33,16 +33,38 @@ win32/opt:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
     run:
         using: mozharness
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/taskcluster_firefox_windows_32_opt.py
 
+win32-dmd/opt:
+    description: "Win32 DMD Opt"
+    index:
+        product: firefox
+        job-name: win32-dmd-opt
+    treeherder:
+        platform: windows2012-32-dmd/opt
+        symbol: tc(Bdmd)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win32/releng.manifest"
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_windows_32_opt.py
+        custom-build-variant-cfg: dmd
+    run-on-projects: [ ]
+
 win32/pgo:
     description: "Win32 Opt PGO"
     index:
         product: firefox
         job-name: win32-pgo
     treeherder:
         platform: windows2012-32/pgo
         symbol: tc(B)
@@ -94,16 +116,38 @@ win64/opt:
         env:
             TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
     run:
         using: mozharness
         script: mozharness/scripts/fx_desktop_build.py
         config:
             - builds/taskcluster_firefox_windows_64_opt.py
 
+win64-dmd/opt:
+    description: "Win64 DMD Opt"
+    index:
+        product: firefox
+        job-name: win64-dmd-opt
+    treeherder:
+        platform: windows2012-64-dmd/opt
+        symbol: tc(Bdmd)
+        tier: 2
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
+    run:
+        using: mozharness
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/taskcluster_firefox_windows_64_opt.py
+        custom-build-variant-cfg: dmd
+    run-on-projects: [ ]
+
 win64-nightly/opt:
     description: "Win64 Nightly"
     index:
         product: firefox
         job-name: win64-opt
         type: nightly
     attributes:
         nightly: true
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -314,8 +314,18 @@ def target_tasks_nightly_win64(full_task
     """Select the set of tasks required for a nightly build of win64. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     def filter(task):
         platform = task.attributes.get('build_platform')
         if platform in ('win64-nightly', ):
             return task.attributes.get('nightly', False)
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
+
+
+# Opt DMD builds should only run nightly
+@_target_task('nightly_dmd')
+def target_tasks_dmd(full_task_graph, parameters):
+    """Target DMD that run nightly on the m-c branch."""
+    def filter(task):
+        platform = task.attributes.get('build_platform', '')
+        return platform.endswith('-dmd')
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t)]
--- a/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
+++ b/taskcluster/taskgraph/transforms/gecko_v2_whitelist.py
@@ -44,26 +44,28 @@ JOB_NAME_WHITELIST = set([
     'linux64-noopt-debug',
     'linux64-opt',
     'linux64-pgo',
     'linux64-st-an-debug',
     'linux64-st-an-opt',
     'linux64-stylo-debug',
     'linux64-stylo-opt',
     'linux64-valgrind-opt',
+    'linux64-dmd-opt',
     'macosx64-add-on-devel',
     'macosx64-clang-tidy',
     'macosx64-debug',
     'macosx64-nightly-repackage',
     'macosx64-nightly-repackage-signing',
     'macosx64-noopt-debug',
     'macosx64-opt',
     'macosx64-devedition-opt',
     'macosx64-st-an-debug',
     'macosx64-st-an-opt',
+    'macosx64-dmd-opt',
     'shell-haz-debug',
     'sm-arm-sim-debug',
     'sm-arm64-sim-debug',
     'sm-asan-opt',
     'sm-compacting-debug',
     'sm-fuzzing',
     'sm-mozjs-sys-debug',
     'sm-msan-opt',
@@ -76,28 +78,30 @@ JOB_NAME_WHITELIST = set([
     'win32-add-on-devel',
     'win32-clang-tidy',
     'win32-debug',
     'win32-noopt-debug',
     'win32-opt',
     'win32-pgo',
     'win32-st-an-debug',
     'win32-st-an-opt',
+    'win32-dmd-opt',
     'win64-add-on-devel',
     'win64-clang-tidy',
     'win64-debug',
     'win64-nightly-repackage',
     'win64-nightly-repackage-signing',
     'win64-noopt-debug',
     'win64-opt',
     'win64-pgo',
     'win64-st-an-debug',
     'win64-st-an-opt',
     'win64-asan-debug',
     'win64-asan-opt',
+    'win64-dmd-opt',
 ])
 
 JOB_NAME_WHITELIST_ERROR = """\
 The gecko-v2 job name {} is not in the whitelist in gecko_v2_whitelist.py.
 If this job runs on Buildbot, please ensure that the job names match between
 Buildbot and TaskCluster, then add the job name to the whitelist.  If this is a
 new job, there is nothing to check -- just add the job to the whitelist.
 """
--- a/taskcluster/taskgraph/transforms/job/mozharness.py
+++ b/taskcluster/taskgraph/transforms/job/mozharness.py
@@ -180,17 +180,17 @@ def mozharness_on_docker_worker_setup(co
 @run_job_using("generic-worker", "mozharness", schema=mozharness_run_schema)
 def mozharness_on_generic_worker(config, job, taskdesc):
     assert job['worker']['os'] == 'windows', 'only supports windows right now'
 
     run = job['run']
 
     # fail if invalid run options are included
     invalid = []
-    for prop in ['actions', 'custom-build-variant-cfg',
+    for prop in ['actions',
                  'tooltool-downloads', 'secrets', 'taskcluster-proxy',
                  'need-xvfb']:
         if prop in run and run[prop]:
             invalid.append(prop)
     if not run.get('keep-artifacts', True):
         invalid.append('keep-artifacts')
     if invalid:
         raise Exception("Jobs run using mozharness on Windows do not support properties " +
@@ -221,16 +221,19 @@ def mozharness_on_generic_worker(config,
     mh_command = [r'c:\mozilla-build\python\python.exe']
     mh_command.append('\\'.join([r'.\build\src\testing', run['script'].replace('/', '\\')]))
     for cfg in run['config']:
         mh_command.append('--config ' + cfg.replace('/', '\\'))
     mh_command.append('--branch ' + config.params['project'])
     mh_command.append(r'--skip-buildbot-actions --work-dir %cd:Z:=z:%\build')
     for option in run.get('options', []):
         mh_command.append('--' + option)
+    if run.get('custom-build-variant-cfg'):
+        mh_command.append('--custom-build-variant')
+        mh_command.append(run['custom-build-variant-cfg'])
 
     hg_command = ['"c:\\Program Files\\Mercurial\\hg.exe"']
     hg_command.append('robustcheckout')
     hg_command.extend(['--sharebase', 'y:\\hg-shared'])
     hg_command.append('--purge')
     hg_command.extend(['--upstream', 'https://hg.mozilla.org/mozilla-unified'])
     hg_command.extend(['--revision', env['GECKO_HEAD_REV']])
     hg_command.append(env['GECKO_HEAD_REPOSITORY'])
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/32_dmd.py
@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/linux32/opt-dmd',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_linux_configs/64_dmd.py
@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/linux64/opt-dmd',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_mac_configs/64_dmd.py
@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/macosx64/opt-dmd',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_windows_configs/32_dmd.py
@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/win32/opt-dmd',
+}
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/releng_sub_windows_configs/64_dmd.py
@@ -0,0 +1,3 @@
+config = {
+    'src_mozconfig': 'browser/config/mozconfigs/win64/opt-dmd',
+}
--- a/testing/mozharness/mozharness/mozilla/building/buildbase.py
+++ b/testing/mozharness/mozharness/mozilla/building/buildbase.py
@@ -379,16 +379,17 @@ class BuildOptionParser(object):
         'android-test': 'builds/releng_sub_%s_configs/%s_test.py',
         'android-checkstyle': 'builds/releng_sub_%s_configs/%s_checkstyle.py',
         'android-lint': 'builds/releng_sub_%s_configs/%s_lint.py',
         'android-findbugs': 'builds/releng_sub_%s_configs/%s_findbugs.py',
         'valgrind' : 'builds/releng_sub_%s_configs/%s_valgrind.py',
         'artifact': 'builds/releng_sub_%s_configs/%s_artifact.py',
         'debug-artifact': 'builds/releng_sub_%s_configs/%s_debug_artifact.py',
         'devedition': 'builds/releng_sub_%s_configs/%s_devedition.py',
+        'dmd': 'builds/releng_sub_%s_configs/%s_dmd.py',
     }
     build_pool_cfg_file = 'builds/build_pool_specifics.py'
     branch_cfg_file = 'builds/branch_specifics.py'
 
     @classmethod
     def _query_pltfrm_and_bits(cls, target_option, options):
         """ determine platform and bits