Bug 1473259 - Windows updates for ASan reporter nightly build. r?catlee draft
authorChristian Holler <choller@mozilla.com>
Tue, 03 Jul 2018 23:41:31 +0200
changeset 818497 023250ed1cc387facd39a0e261a8ab363d8c03b0
parent 817669 e951f4ad123aa87d1d392c286db14cabb41a8560
push id116275
push usercholler@mozilla.com
push dateSat, 14 Jul 2018 08:55:08 +0000
reviewerscatlee
bugs1473259
milestone63.0a1
Bug 1473259 - Windows updates for ASan reporter nightly build. r?catlee MozReview-Commit-ID: 6DQD5DnzyqB
browser/config/mozconfigs/win64/nightly-asan-reporter
taskcluster/ci/beetmover-repackage/kind.yml
taskcluster/ci/beetmover/kind.yml
taskcluster/ci/build/windows.yml
taskcluster/ci/partials/kind.yml
taskcluster/ci/repackage-signing-l10n/kind.yml
taskcluster/ci/repackage-signing/kind.yml
taskcluster/ci/repackage/kind.yml
taskcluster/ci/upload-symbols/kind.yml
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/util/partials.py
testing/mozharness/configs/builds/taskcluster_sub_win64/asan_reporter_opt.py
new file mode 100644
--- /dev/null
+++ b/browser/config/mozconfigs/win64/nightly-asan-reporter
@@ -0,0 +1,22 @@
+MOZ_AUTOMATION_L10N_CHECK=0
+
+ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}
+MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-1}
+
+. "$topsrcdir/build/mozconfig.win-common"
+. "$topsrcdir/browser/config/mozconfigs/common"
+
+ac_add_options --disable-debug
+ac_add_options --enable-optimize="-O2 -gline-tables-only"
+ac_add_options --enable-address-sanitizer-reporter
+
+. "$topsrcdir/build/win64/mozconfig.vs-latest"
+
+. "$topsrcdir/build/win64/mozconfig.asan"
+
+export MOZ_PKG_SPECIAL=asan-reporter
+
+# Need this to add source information into platform.ini
+export MOZILLA_OFFICIAL=1
+
+. "$topsrcdir/build/mozconfig.common.override"
--- a/taskcluster/ci/beetmover-repackage/kind.yml
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@@ -22,11 +22,12 @@ only-for-build-platforms:
    - win32-nightly/opt
    - win64-nightly/opt
    - linux-devedition-nightly/opt
    - linux64-devedition-nightly/opt
    - macosx64-devedition-nightly/opt
    - win32-devedition-nightly/opt
    - win64-devedition-nightly/opt
    - linux64-asan-reporter-nightly/opt
+   - win64-asan-reporter-nightly/opt
 
 job-template:
    shipping-phase: promote
--- a/taskcluster/ci/beetmover/kind.yml
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -22,8 +22,9 @@ not-for-build-platforms:
    - win32-nightly/opt
    - win64-nightly/opt
    - linux-devedition-nightly/opt
    - linux64-devedition-nightly/opt
    - macosx64-devedition-nightly/opt
    - win32-devedition-nightly/opt
    - win64-devedition-nightly/opt
    - linux64-asan-reporter-nightly/opt
+   - win64-asan-reporter-nightly/opt
--- a/taskcluster/ci/build/windows.yml
+++ b/taskcluster/ci/build/windows.yml
@@ -646,16 +646,50 @@ win64-asan/opt:
             - builds/taskcluster_sub_win64/asan_opt.py
     run-on-projects: ['trunk', 'try']
     toolchains:
         - win64-clang-cl
         - win64-rust
         - win64-rust-size
         - win64-sccache
 
+win64-asan-reporter-nightly/opt:
+    description: "Win64 Opt ASAN Reporter Nightly"
+    attributes:
+        nightly: true
+    index:
+        product: firefox
+        job-name: win64-asan-reporter-opt
+        type: nightly
+    treeherder:
+        platform: win64-asan-reporter/opt
+        symbol: BoR
+    worker-type: aws-provisioner-v1/gecko-{level}-b-win2012
+    worker:
+        max-run-time: 7200
+        env:
+            TOOLTOOL_MANIFEST: "browser/config/tooltool-manifests/win64/releng.manifest"
+            PERFHERDER_EXTRA_OPTIONS: "asan-reporter"
+    run:
+        using: mozharness
+        options: [append-env-variables-from-configs]
+        script: mozharness/scripts/fx_desktop_build.py
+        config:
+            - builds/releng_base_firefox.py
+            - builds/taskcluster_base_windows.py
+            - builds/taskcluster_base_win64.py
+            - builds/taskcluster_sub_win64/asan_reporter_opt.py
+            - taskcluster_nightly.py
+    run-on-projects: ['mozilla-central']
+    toolchains:
+        - win64-clang-cl
+        - win64-rust
+        - win64-rust-size
+        - win64-sccache
+
 win32-devedition-nightly/opt:
     description: "Win32 Dev Edition Nightly"
     index:
         product: devedition
         job-name: win32-opt
         type: nightly
     attributes:
         nightly: true
--- a/taskcluster/ci/partials/kind.yml
+++ b/taskcluster/ci/partials/kind.yml
@@ -23,11 +23,12 @@ only-for-build-platforms:
    - win32-devedition-nightly/opt
    - win64-nightly/opt
    - win64-devedition-nightly/opt
    - linux-nightly/opt
    - linux-devedition-nightly/opt
    - linux64-nightly/opt
    - linux64-devedition-nightly/opt
    - linux64-asan-reporter-nightly/opt
+   - win64-asan-reporter-nightly/opt
 
 job-template:
    shipping-phase: promote
--- a/taskcluster/ci/repackage-signing-l10n/kind.yml
+++ b/taskcluster/ci/repackage-signing-l10n/kind.yml
@@ -22,8 +22,9 @@ only-for-build-platforms:
    - macosx64-nightly/opt
    - macosx64-devedition-nightly/opt
    - win32-nightly/opt
    - win32-devedition-nightly/opt
    - win32/opt
    - win64-nightly/opt
    - win64-devedition-nightly/opt
    - win64/opt
+   - win64-asan-reporter-nightly/opt
--- a/taskcluster/ci/repackage-signing/kind.yml
+++ b/taskcluster/ci/repackage-signing/kind.yml
@@ -22,8 +22,9 @@ only-for-build-platforms:
    - macosx64-nightly/opt
    - macosx64-devedition-nightly/opt
    - win32-nightly/opt
    - win32-devedition-nightly/opt
    - win32/opt
    - win64-nightly/opt
    - win64-devedition-nightly/opt
    - win64/opt
+   - win64-asan-reporter-nightly/opt
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@@ -28,16 +28,17 @@ only-for-build-platforms:
     - macosx64-devedition-nightly/opt
     - macosx64/opt
     - win32-nightly/opt
     - win32-devedition-nightly/opt
     - win32/opt
     - win64-nightly/opt
     - win64-devedition-nightly/opt
     - win64/opt
+    - win64-asan-reporter-nightly/opt
 
 job-template:
     mozharness:
         config:
             by-build-platform:
                 linux\b.*:
                     - repackage/linux32_signed.py
                 linux64\b.*:
--- a/taskcluster/ci/upload-symbols/kind.yml
+++ b/taskcluster/ci/upload-symbols/kind.yml
@@ -10,16 +10,17 @@ transforms:
     - taskgraph.transforms.task:transforms
 
 kind-dependencies:
     - build
 
 not-for-build-platforms:
     # Address Sanitizer builds don't have symbols to upload.
     - linux64-asan-reporter-nightly/opt
+    - win64-asan-reporter-nightly/opt
 
 job-template:
     description: Upload Symbols
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         docker-image: {in-tree: "lint"}
         max-run-time: 900
         env:
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -54,16 +54,17 @@ def filter_beta_release_tasks(task, para
         ]
     platform = task.attributes.get('build_platform')
     if platform in (
             # On beta, Nightly builds are already PGOs
             'linux-pgo', 'linux64-pgo',
             'win32-pgo', 'win64-pgo',
             # ASAN is central-only
             'linux64-asan-reporter-nightly',
+            'win64-asan-reporter-nightly',
             ):
         return False
     if str(platform).startswith('android') and 'nightly' in str(platform):
         return False
 
     if platform in (
             'linux', 'linux64',
             'macosx64',
@@ -528,17 +529,17 @@ def target_tasks_nightly_win64(full_task
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
 
 
 @_target_task('nightly_asan')
 def target_tasks_nightly_asan(full_task_graph, parameters, graph_config):
     """Select the set of tasks required for a nightly build of asan. The
     nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
-    filter = make_nightly_filter({'linux64-asan-reporter-nightly'})
+    filter = make_nightly_filter({'linux64-asan-reporter-nightly', 'win64-asan-reporter-nightly'})
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
 
 
 @_target_task('nightly_desktop')
 def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
     """Select the set of tasks required for a nightly build of linux, mac,
     windows."""
     # Avoid duplicate tasks.
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -142,16 +142,22 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     'linux64-asan-reporter-nightly':
         # ASan reporter builds don't generate the regular crashreporter symbol
         # packages, so we shouldn't try to beetmove them
         filter(lambda a: a != 'target.crashreporter-symbols.zip',
                _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
                     "host/bin/mar",
                     "host/bin/mbsdiff",
                 ]),
+    'win64-asan-reporter-nightly':
+        filter(lambda a: a != 'target.crashreporter-symbols.zip',
+               _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+                    "host/bin/mar.exe",
+                    "host/bin/mbsdiff.exe",
+                ]),
     'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-aarch64-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-16-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
     ],
     'macosx64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
@@ -209,16 +215,19 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux64-asan-reporter-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
+    'win64-asan-reporter-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.zip",
+    ],
     'android-x86-nightly': ["en-US/target.apk"],
     'android-aarch64-nightly': ["en-US/target.apk"],
     'android-api-16-nightly': ["en-US/target.apk"],
     'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.dmg",
         "target.dmg.asc",
     ],
     'macosx64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -70,49 +70,56 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
             'host/bin/mbsdiff',
         ],
     r'^linux64-asan-reporter-nightly$':
         filter(lambda a: a not in ('target.crashreporter-symbols.zip', 'target.jsshell.zip'),
                _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
                     "host/bin/mar",
                     "host/bin/mbsdiff",
                 ]),
+    r'^win64-asan-reporter-nightly$':
+        filter(lambda a: a not in ('target.crashreporter-symbols.zip', 'target.jsshell.zip'),
+               _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+                    "host/bin/mar.exe",
+                    "host/bin/mbsdiff.exe",
+                ]),
     r'^win(32|64)(|-devedition)-nightly$':
         _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
             'host/bin/mar.exe',
             'host/bin/mbsdiff.exe',
         ],
     r'^(linux(|64)|macosx64|win(32|64))(|-devedition)-nightly-l10n$': [],
 }
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     r'^linux(|64)(|-devedition|-asan-reporter)-nightly(|-l10n)$':
         ['target.tar.bz2', 'target.tar.bz2.asc'],
-    r'^win(32|64)(|-devedition)-nightly(|-l10n)$': ['target.zip'],
+    r'^win(32|64)(|-devedition|-asan-reporter)-nightly(|-l10n)$': ['target.zip'],
 }
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
     r'^macosx64(|-devedition)-nightly(|-l10n)$': ['target.dmg'],
 }
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = {
     r'^(linux(|64)|macosx64)(|-devedition|-asan-reporter)-nightly(|-l10n)$':
         ['target.complete.mar'],
-    r'^win64(|-devedition)-nightly(|-l10n)$': ['target.complete.mar', 'target.installer.exe'],
+    r'^win64(|-devedition|-asan-reporter)-nightly(|-l10n)$':
+        ['target.complete.mar', 'target.installer.exe'],
     r'^win32(|-devedition)-nightly(|-l10n)$': [
         'target.complete.mar',
         'target.installer.exe',
         'target.stub-installer.exe'
         ],
 }
 
 # Compile every regex once at import time
--- a/taskcluster/taskgraph/util/partials.py
+++ b/taskcluster/taskgraph/util/partials.py
@@ -38,27 +38,31 @@ BALROG_PLATFORM_MAP = {
     "win32": [
         "WINNT_x86-msvc",
         "WINNT_x86-msvc-x86",
         "WINNT_x86-msvc-x64"
     ],
     "win64": [
         "WINNT_x86_64-msvc",
         "WINNT_x86_64-msvc-x64"
+    ],
+    "win64-asan-reporter": [
+        "WINNT_x86_64-clangcl-asan"
     ]
 }
 
 FTP_PLATFORM_MAP = {
     "Darwin_x86-gcc3": "mac",
     "Darwin_x86-gcc3-u-i386-x86_64": "mac",
     "Darwin_x86_64-gcc3": "mac",
     "Darwin_x86_64-gcc3-u-i386-x86_64": "mac",
     "Linux_x86-gcc3": "linux-i686",
     "Linux_x86_64-gcc3": "linux-x86_64",
     "Linux_x86_64-gcc3-asan": "linux-x86_64-asan-reporter",
+    "WINNT_x86_64-clangcl-asan": "win64-asan-reporter",
     "WINNT_x86-msvc": "win32",
     "WINNT_x86-msvc-x64": "win32",
     "WINNT_x86-msvc-x86": "win32",
     "WINNT_x86_64-msvc": "win64",
     "WINNT_x86_64-msvc-x64": "win64",
 }
 
 
new file mode 100644
--- /dev/null
+++ b/testing/mozharness/configs/builds/taskcluster_sub_win64/asan_reporter_opt.py
@@ -0,0 +1,4 @@
+config = {
+    'stage_platform': 'win64-asan-reporter',
+    'mozconfig_variant': 'nightly-asan-reporter',
+}