Bug 1403548: Enable nightly asan reportor builds and updates r=Callek draft
authorChris AtLee <catlee@mozilla.com>
Thu, 19 Apr 2018 09:28:59 -0400
changeset 788063 70d25b081ed874d4f88944e05f40764f948a45a7
parent 787860 7f6a582f00bfb5d0acb8d8bf7f8c79ca37c99b65
push id107886
push usercatlee@mozilla.com
push dateWed, 25 Apr 2018 21:52:34 +0000
reviewersCallek
bugs1403548
milestone61.0a1
Bug 1403548: Enable nightly asan reportor builds and updates r=Callek MozReview-Commit-ID: 2X03vU5bO32
browser/config/mozconfigs/linux64/nightly-asan-reporter
browser/confvars.sh
taskcluster/ci/beetmover-repackage/kind.yml
taskcluster/ci/beetmover/kind.yml
taskcluster/ci/build/linux.yml
taskcluster/ci/partials/kind.yml
taskcluster/ci/repackage-signing/kind.yml
taskcluster/ci/repackage/kind.yml
taskcluster/taskgraph/target_tasks.py
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/util/partials.py
toolkit/moz.configure
--- a/browser/config/mozconfigs/linux64/nightly-asan-reporter
+++ b/browser/config/mozconfigs/linux64/nightly-asan-reporter
@@ -1,18 +1,23 @@
 # We still need to build with debug symbols
 ac_add_options --disable-debug
 ac_add_options --enable-optimize="-O2 -gline-tables-only"
+ac_add_options --enable-update-channel=${MOZ_UPDATE_CHANNEL}
+MOZ_AUTOMATION_UPDATE_PACKAGING=${MOZ_AUTOMATION_UPDATE_PACKAGING-1}
 
 . $topsrcdir/build/mozconfig.stylo
 
 # ASan specific options on Linux
 ac_add_options --enable-valgrind
 
 . $topsrcdir/build/unix/mozconfig.asan
 ac_add_options --disable-elf-hack
 
 ac_add_options --enable-address-sanitizer-reporter
 
 # Need this to prevent name conflicts with the normal nightly build packages
 export MOZ_PKG_SPECIAL=asan-reporter
 
+# Need this to add source information into platform.ini
+export MOZILLA_OFFICIAL=1
+
 . "$topsrcdir/build/mozconfig.common.override"
--- a/browser/confvars.sh
+++ b/browser/confvars.sh
@@ -48,15 +48,21 @@ MOZ_APP_ID={ec8030f7-c20a-464f-9b0e-13a3
 # The MAR_CHANNEL_ID must not contain the following 3 characters: ",\t "
 if test "$MOZ_UPDATE_CHANNEL" = "aurora"; then
   ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-aurora
   MAR_CHANNEL_ID=firefox-mozilla-aurora
 else
   ACCEPTED_MAR_CHANNEL_IDS=firefox-mozilla-central
   MAR_CHANNEL_ID=firefox-mozilla-central
 fi
+# ASan reporter builds should have different channel ids
+if [ "${MOZ_ASAN_REPORTER}" = "1" ]; then
+    ACCEPTED_MAR_CHANNEL_IDS="${ACCEPTED_MAR_CHANNEL_IDS}-asan"
+    MAR_CHANNEL_ID="${MAR_CHANNEL_ID}-asan"
+fi
+
 MOZ_PROFILE_MIGRATOR=1
 
 # Enable checking that add-ons are signed by the trusted root
 MOZ_ADDON_SIGNING=1
 
 # Include the DevTools client, not just the server (which is the default)
 MOZ_DEVTOOLS=all
--- a/taskcluster/ci/beetmover-repackage/kind.yml
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@@ -21,11 +21,12 @@ only-for-build-platforms:
    - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
    - linux-devedition-nightly/opt
    - linux64-devedition-nightly/opt
    - macosx64-devedition-nightly/opt
    - win32-devedition-nightly/opt
    - win64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt
 
 job-template:
    shipping-phase: promote
--- a/taskcluster/ci/beetmover/kind.yml
+++ b/taskcluster/ci/beetmover/kind.yml
@@ -22,8 +22,9 @@ not-for-build-platforms:
    - macosx64-nightly/opt
    - win32-nightly/opt
    - win64-nightly/opt
    - linux-devedition-nightly/opt
    - linux64-devedition-nightly/opt
    - macosx64-devedition-nightly/opt
    - win32-devedition-nightly/opt
    - win64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -521,35 +521,39 @@ linux64-asan-fuzzing/opt:
         tooltool-downloads: public
         need-xvfb: true
     toolchains:
         - linux64-clang-6-pre
         - linux64-gcc
         - linux64-rust
         - linux64-sccache
 
-linux64-asan-reporter/opt:
+linux64-asan-reporter-nightly/opt:
     description: "Linux64 Opt ASAN Reporter Nightly"
+    attributes:
+        nightly: true
     index:
         product: firefox
         job-name: linux64-asan-reporter-opt
+        type: nightly
     treeherder:
-        platform: linux64/asan
+        platform: linux64-asan-reporter/opt
         symbol: BoR
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         env:
             PERFHERDER_EXTRA_OPTIONS: asan-reporter
         max-run-time: 36000
     run:
         using: mozharness
         actions: [get-secrets build check-test update]
         config:
             - builds/releng_base_firefox.py
             - builds/releng_base_linux_64_builds.py
+            - taskcluster_nightly.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: asan-reporter-tc
         tooltool-downloads: public
         need-xvfb: true
     toolchains:
         - linux64-clang-6-pre
         - linux64-gcc
--- a/taskcluster/ci/partials/kind.yml
+++ b/taskcluster/ci/partials/kind.yml
@@ -21,11 +21,12 @@ only-for-build-platforms:
    - win32-nightly/opt
    - win32-devedition-nightly/opt
    - win64-nightly/opt
    - win64-devedition-nightly/opt
    - linux-nightly/opt
    - linux-devedition-nightly/opt
    - linux64-nightly/opt
    - linux64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt
 
 job-template:
    shipping-phase: promote
--- a/taskcluster/ci/repackage-signing/kind.yml
+++ b/taskcluster/ci/repackage-signing/kind.yml
@@ -15,14 +15,15 @@ kind-dependencies:
    - repackage
    - repackage-l10n
 
 only-for-build-platforms:
    - linux-nightly/opt
    - linux-devedition-nightly/opt
    - linux64-nightly/opt
    - linux64-devedition-nightly/opt
+   - linux64-asan-reporter-nightly/opt
    - macosx64-nightly/opt
    - macosx64-devedition-nightly/opt
    - win32-nightly/opt
    - win32-devedition-nightly/opt
    - win64-nightly/opt
    - win64-devedition-nightly/opt
--- a/taskcluster/ci/repackage/kind.yml
+++ b/taskcluster/ci/repackage/kind.yml
@@ -16,16 +16,17 @@ kind-dependencies:
     - build-signing
     - toolchain
 
 only-for-build-platforms:
     - linux-nightly/opt
     - linux-devedition-nightly/opt
     - linux64-nightly/opt
     - linux64-devedition-nightly/opt
+    - linux64-asan-reporter-nightly/opt
     - macosx64-nightly/opt
     - macosx64-devedition-nightly/opt
     - win32-nightly/opt
     - win32-devedition-nightly/opt
     - win64-nightly/opt
     - win64-devedition-nightly/opt
 
 job-template:
--- a/taskcluster/taskgraph/target_tasks.py
+++ b/taskcluster/taskgraph/target_tasks.py
@@ -598,26 +598,36 @@ def target_tasks_nightly_win32(full_task
 def target_tasks_nightly_win64(full_task_graph, parameters, graph_config):
     """Select the set of tasks required for a nightly build of win32 and win64.
     The nightly build process involves a pipeline of builds, signing,
     and, eventually, uploading the tasks to balrog."""
     filter = make_nightly_filter({'win64-nightly'})
     return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
 
 
+@_target_task('nightly_asan')
+def target_tasks_nightly_asan(full_task_graph, parameters, graph_config):
+    """Select the set of tasks required for a nightly build of asan. The
+    nightly build process involves a pipeline of builds, signing,
+    and, eventually, uploading the tasks to balrog."""
+    filter = make_nightly_filter({'linux64-asan-reporter-nightly'})
+    return [l for l, t in full_task_graph.tasks.iteritems() if filter(t, parameters)]
+
+
 @_target_task('nightly_desktop')
 def target_tasks_nightly_desktop(full_task_graph, parameters, graph_config):
     """Select the set of tasks required for a nightly build of linux, mac,
     windows."""
     # Avoid duplicate tasks.
     return list(
         set(target_tasks_nightly_win32(full_task_graph, parameters, graph_config))
         | set(target_tasks_nightly_win64(full_task_graph, parameters, graph_config))
         | set(target_tasks_nightly_macosx(full_task_graph, parameters, graph_config))
         | set(target_tasks_nightly_linux(full_task_graph, parameters, graph_config))
+        | set(target_tasks_nightly_asan(full_task_graph, parameters, graph_config))
     )
 
 
 # Opt DMD builds should only run nightly
 @_target_task('nightly_dmd')
 def target_tasks_dmd(full_task_graph, parameters, graph_config):
     """Target DMD that run nightly on the m-c branch."""
     def filter(task):
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -131,16 +131,24 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
     ],
     'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
         "host/bin/mar",
         "host/bin/mbsdiff",
     ],
+    'linux64-asan-reporter-nightly':
+        # ASan reporter builds don't generate the regular crashreporter symbol
+        # packages, so we shouldn't try to beetmove them
+        filter(lambda a: a != 'target.crashreporter-symbols.zip',
+               _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+                    "host/bin/mar",
+                    "host/bin/mbsdiff",
+                ]),
     'linux64-source': [
     ],
     'linux64-devedition-source': [
     ],
     'linux64-fennec-source': [
     ],
     'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-aarch64-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
@@ -200,16 +208,20 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
+    'linux64-asan-reporter-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
     'linux64-source': [
         "source.tar.xz",
         "source.tar.xz.asc",
     ],
     'linux64-devedition-source': [
         "source.tar.xz",
         "source.tar.xz.asc",
     ],
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -70,16 +70,22 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
         ],
     r'^(linux(|64)|macosx64)-devedition-nightly$':
         _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
             'host/bin/mar',
             'host/bin/mbsdiff',
             # TODO Bug 1453033: Sign devedition langpacks
             'target.langpack.xpi',
         ],
+    r'^linux64-asan-reporter-nightly$':
+        filter(lambda a: a not in ('target.crashreporter-symbols.zip', 'target.jsshell.zip'),
+               _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
+                    "host/bin/mar",
+                    "host/bin/mbsdiff",
+                ]),
     r'^win(32|64)-nightly$':
         _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
             'host/bin/mar.exe',
             'host/bin/mbsdiff.exe',
         ],
     r'^win(32|64)-devedition-nightly$':
         _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US + [
             'host/bin/mar.exe',
@@ -95,39 +101,41 @@ UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
         ],
 }
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_SIGNED_PATHS = {
-    r'^linux(|64)(|-devedition)-nightly(|-l10n)$': ['target.tar.bz2', 'target.tar.bz2.asc'],
+    r'^linux(|64)(|-devedition|-asan-reporter)-nightly(|-l10n)$':
+        ['target.tar.bz2', 'target.tar.bz2.asc'],
     r'^win(32|64)(|-devedition)-nightly(|-l10n)$': ['target.zip'],
 }
 
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_REPACKAGE_PATHS = {
     r'^macosx64(|-devedition)-nightly(|-l10n)$': ['target.dmg'],
 }
 # Until bug 1331141 is fixed, if you are adding any new artifacts here that
 # need to be transfered to S3, please be aware you also need to follow-up
 # with a beetmover patch in https://github.com/mozilla-releng/beetmoverscript/.
 # See example in bug 1348286
 UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS = {
-    r'^(linux(|64)|macosx64)(|-devedition)-nightly(|-l10n)$': ['target.complete.mar'],
+    r'^(linux(|64)|macosx64)(|-devedition|-asan-reporter)-nightly(|-l10n)$':
+        ['target.complete.mar'],
     r'^win64(|-devedition)-nightly(|-l10n)$': ['target.complete.mar', 'target.installer.exe'],
     r'^win32(|-devedition)-nightly(|-l10n)$': [
         'target.complete.mar',
         'target.installer.exe',
         'target.stub-installer.exe'
-    ],
+        ],
 }
 
 # Compile every regex once at import time
 for dict_ in (
     UPSTREAM_ARTIFACT_UNSIGNED_PATHS, UPSTREAM_ARTIFACT_SIGNED_PATHS,
     UPSTREAM_ARTIFACT_REPACKAGE_PATHS, UPSTREAM_ARTIFACT_SIGNED_REPACKAGE_PATHS,
 ):
     for uncompiled_regex, value in dict_.iteritems():
@@ -286,30 +294,30 @@ def generate_upstream_artifacts(job, bui
     mapping = [
         build_mapping,
         build_signing_mapping,
         repackage_mapping,
         repackage_signing_mapping
     ]
 
     for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping):
-        plarform_was_previously_matched_by_regex = None
+        platform_was_previously_matched_by_regex = None
         for platform_regex, paths in mapping.iteritems():
             if platform_regex.match(platform) is not None:
                 _check_platform_matched_only_one_regex(
-                    tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
+                    tasktype, platform, platform_was_previously_matched_by_regex, platform_regex
                 )
                 if paths:
                     upstream_artifacts.append({
                         "taskId": {"task-reference": ref},
                         "taskType": tasktype,
                         "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
                         "locale": locale or "en-US",
                     })
-                plarform_was_previously_matched_by_regex = platform_regex
+                platform_was_previously_matched_by_regex = platform_regex
 
     return upstream_artifacts
 
 
 def generate_partials_upstream_artifacts(job, artifacts, platform, locale=None):
     artifact_prefix = get_artifact_prefix(job)
     if locale and locale != 'en-US':
         artifact_prefix = '{}/{}'.format(artifact_prefix, locale)
@@ -321,24 +329,24 @@ def generate_partials_upstream_artifacts
                   for p in artifacts],
         'locale': locale or 'en-US',
     }]
 
     return upstream_artifacts
 
 
 def _check_platform_matched_only_one_regex(
-    task_type, platform, plarform_was_previously_matched_by_regex, platform_regex
+    task_type, platform, platform_was_previously_matched_by_regex, platform_regex
 ):
-    if plarform_was_previously_matched_by_regex is not None:
+    if platform_was_previously_matched_by_regex is not None:
         raise Exception('In task type "{task_type}", platform "{platform}" matches at \
 least 2 regular expressions. First matched: "{first_matched}". Second matched: \
 "{second_matched}"'.format(
             task_type=task_type, platform=platform,
-            first_matched=plarform_was_previously_matched_by_regex.pattern,
+            first_matched=platform_was_previously_matched_by_regex.pattern,
             second_matched=platform_regex.pattern
         ))
 
 
 def is_valid_beetmover_job(job):
     # beetmover after partials-signing should have six dependencies.
     # windows builds w/o partials don't have docker-image, so fewer
     # dependencies
--- a/taskcluster/taskgraph/util/partials.py
+++ b/taskcluster/taskgraph/util/partials.py
@@ -23,16 +23,19 @@ BALROG_PLATFORM_MAP = {
         "Linux_x86-gcc3"
     ],
     "linux32": [
         "Linux_x86-gcc3"
     ],
     "linux64": [
         "Linux_x86_64-gcc3"
     ],
+    "linux64-asan-reporter": [
+        "Linux_x86_64-gcc3-asan"
+    ],
     "macosx64": [
         "Darwin_x86_64-gcc3-u-i386-x86_64",
         "Darwin_x86-gcc3-u-i386-x86_64",
         "Darwin_x86-gcc3",
         "Darwin_x86_64-gcc3"
     ],
     "win32": [
         "WINNT_x86-msvc",
@@ -47,16 +50,17 @@ BALROG_PLATFORM_MAP = {
 
 FTP_PLATFORM_MAP = {
     "Darwin_x86-gcc3": "mac",
     "Darwin_x86-gcc3-u-i386-x86_64": "mac",
     "Darwin_x86_64-gcc3": "mac",
     "Darwin_x86_64-gcc3-u-i386-x86_64": "mac",
     "Linux_x86-gcc3": "linux-i686",
     "Linux_x86_64-gcc3": "linux-x86_64",
+    "Linux_x86_64-gcc3-asan": "linux-x86_64-asan-reporter",
     "WINNT_x86-msvc": "win32",
     "WINNT_x86-msvc-x64": "win32",
     "WINNT_x86-msvc-x86": "win32",
     "WINNT_x86_64-msvc": "win64",
     "WINNT_x86_64-msvc-x64": "win64",
 }
 
 
--- a/toolkit/moz.configure
+++ b/toolkit/moz.configure
@@ -1154,16 +1154,17 @@ option('--enable-address-sanitizer-repor
 
 @depends('--enable-address-sanitizer-reporter')
 def enable_asan_reporter(value):
     if value:
         return True
 
 set_config('MOZ_ASAN_REPORTER', enable_asan_reporter)
 set_define('MOZ_ASAN_REPORTER', enable_asan_reporter)
+add_old_configure_assignment('MOZ_ASAN_REPORTER', enable_asan_reporter)
 
 # Elfhack
 # ==============================================================
 @depends(host, target)
 def has_elfhack(host, target):
     return target.kernel == 'Linux' and host.kernel == 'Linux' and \
            target.cpu in ('arm', 'x86', 'x86_64')