bug 1358601: set up CI builds and tests for DevEdition on mozilla-beta - fix build configuration and signing of DevEdition-on-beta builds. r=aki,rail
authorBen Hearsum <bhearsum@mozilla.com>
Fri, 05 May 2017 16:56:57 -0400
changeset 407386 28f449764ca0b551dd74f8ae2bbaf0ac577771ac
parent 407385 1c03cb9ed456e08aa9451a71bd77964095474650
child 407387 eee394a4575b6caf23b820c930059bf994ea228d
push id1490
push usermtabara@mozilla.com
push dateMon, 31 Jul 2017 14:08:16 +0000
treeherdermozilla-release@70e32e6bf15e [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki, rail
bugs1358601
milestone55.0a1
first release with
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
last release without
nightly linux32
nightly linux64
nightly mac
nightly win32
nightly win64
bug 1358601: set up CI builds and tests for DevEdition on mozilla-beta - fix build configuration and signing of DevEdition-on-beta builds. r=aki,rail MozReview-Commit-ID: 2oaCCHJWU8W
browser/config/mozconfigs/linux32/devedition
browser/config/mozconfigs/linux64/devedition
browser/config/mozconfigs/macosx64/devedition
browser/config/mozconfigs/win32/devedition
browser/config/mozconfigs/win64/devedition
browser/confvars.sh
taskcluster/ci/build/linux.yml
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/signing.py
taskcluster/taskgraph/util/scriptworker.py
testing/mozharness/configs/builds/branch_specifics.py
--- a/browser/config/mozconfigs/linux32/devedition
+++ b/browser/config/mozconfigs/linux32/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-1}
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/browser/config/mozconfigs/linux32/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --enable-verify-mar
 
 # This will overwrite the default of stripping everything and keep the symbol table.
--- a/browser/config/mozconfigs/linux64/devedition
+++ b/browser/config/mozconfigs/linux64/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-1}
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/browser/config/mozconfigs/linux64/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --enable-verify-mar
 
 # This will overwrite the default of stripping everything and keep the symbol table.
--- a/browser/config/mozconfigs/macosx64/devedition
+++ b/browser/config/mozconfigs/macosx64/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=1
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/browser/config/mozconfigs/macosx64/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --disable-install-strip
 ac_add_options --enable-verify-mar
 
--- a/browser/config/mozconfigs/win32/devedition
+++ b/browser/config/mozconfigs/win32/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=1
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/build/mozconfig.win-common"
 . "$topsrcdir/browser/config/mozconfigs/win32/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --enable-verify-mar
 
--- a/browser/config/mozconfigs/win64/devedition
+++ b/browser/config/mozconfigs/win64/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=1
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/build/mozconfig.win-common"
 . "$topsrcdir/browser/config/mozconfigs/win64/common-win64"
 . "$topsrcdir/browser/config/mozconfigs/win64/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --enable-verify-mar
--- a/browser/confvars.sh
+++ b/browser/confvars.sh
@@ -13,16 +13,17 @@ if test "$OS_ARCH" = "WINNT" -o \
   MOZ_BUNDLED_FONTS=1
 fi
 
 if test "$OS_ARCH" = "WINNT"; then
   MOZ_MAINTENANCE_SERVICE=1
   if ! test "$HAVE_64BIT_BUILD"; then
     if test "$MOZ_UPDATE_CHANNEL" = "nightly" -o \
             "$MOZ_UPDATE_CHANNEL" = "aurora" -o \
+            "$MOZ_UPDATE_CHANNEL" = "aurora-dev" -o \
             "$MOZ_UPDATE_CHANNEL" = "beta" -o \
             "$MOZ_UPDATE_CHANNEL" = "beta-dev" -o \
             "$MOZ_UPDATE_CHANNEL" = "release" -o \
             "$MOZ_UPDATE_CHANNEL" = "release-dev"; then
       if ! test "$MOZ_DEBUG"; then
         MOZ_STUB_INSTALLER=1
       fi
     fi
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -65,33 +65,37 @@ linux64/debug:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: debug
         tooltool-downloads: public
         need-xvfb: true
 
-linux64-devedition/opt:
-    description: "Linux64 devedition Opt"
+linux64-devedition-nightly/opt:
+    description: "Linux64 devedition Nightly"
+    attributes:
+        nightly: true
     index:
         product: devedition
         job-name: linux64-opt
+        type: nightly
     treeherder:
         platform: linux64-devedition/opt
-        symbol: tc(B)
+        symbol: tc(N)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
     run:
         using: mozharness
         actions: [get-secrets build generate-build-stats check-test update]
         config:
             - builds/releng_base_linux_64_builds.py
+            - disable_signing.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
         custom-build-variant-cfg: devedition
     run-on-projects: [ 'mozilla-beta', ]
 
@@ -164,33 +168,37 @@ linux/pgo:
         config:
             - builds/releng_base_linux_32_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
 
-linux-devedition/opt:
-    description: "Linux32 devedition Opt"
+linux-devedition-nightly/opt:
+    description: "Linux32 devedition Nightly"
+    attributes:
+        nightly: true
     index:
         product: devedition
         job-name: linux-opt
+        type: nightly
     treeherder:
         platform: linux32-devedition/opt
-        symbol: tc(B)
+        symbol: tc(N)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
     run:
         using: mozharness
         actions: [get-secrets build generate-build-stats check-test update]
         config:
             - builds/releng_base_linux_32_builds.py
+            - disable_signing.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
         custom-build-variant-cfg: devedition
     run-on-projects: [ 'mozilla-beta', ]
 
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -98,16 +98,18 @@ from voluptuous import Any, Required, Op
 _MOBILE_UPSTREAM_ARTIFACTS_SIGNED_MULTI = [
     "target.apk",
 ]
 
 
 UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-15-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
 
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
     'android-api-15-nightly-l10n': ["balrog_props.json"],
@@ -118,16 +120,24 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
+    'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
     'android-x86-nightly': ["en-US/target.apk"],
     'android-api-15-nightly': ["en-US/target.apk"],
     'macosx64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.dmg",
         "target.dmg.asc",
     ],
 
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -4,17 +4,17 @@
 """
 Transform the signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema, Schema
-from taskgraph.util.scriptworker import get_signing_cert_scope
+from taskgraph.util.scriptworker import get_signing_cert_scope, get_devedition_signing_cert_scope
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
@@ -101,17 +101,24 @@ def make_task_description(config, jobs):
             'nightly': dep_job.attributes.get('nightly', False),
             'build_platform': dep_job.attributes.get('build_platform'),
             'build_type': dep_job.attributes.get('build_type'),
         }
         if dep_job.attributes.get('chunk_locales'):
             # Used for l10n attribute passthrough
             attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
 
-        signing_cert_scope = get_signing_cert_scope(config)
+        # This code wasn't originally written with the possibility of using different
+        # signing cert scopes for different platforms on the same branch. This isn't
+        # ideal, but it's what we currently have to make this possible.
+        if dep_job.attributes.get('build_platform') in set(['linux64-devedition-nightly', 'linux-devedition-nightly']):
+            signing_cert_scope = get_devedition_signing_cert_scope(config)
+        else:
+            signing_cert_scope = get_signing_cert_scope(config)
+
         task = {
             'label': label,
             'description': "{} Signing".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': "scriptworker-prov-v1/signing-linux-v1",
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': job['upstream-artifacts'],
                        'max-run-time': 3600},
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -19,17 +19,22 @@ import os
 
 
 # constants {{{1
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 VERSION_PATH = os.path.join(GECKO, "browser", "config", "version_display.txt")
 
 """Map signing scope aliases to sets of projects.
 
-Currently m-c and m-a use nightly signing; m-b and m-r use release signing.
+Currently m-c and DevEdition on m-b use nightly signing; Beta on m-b and m-r
+use release signing. These data structures aren't set-up to handle different
+scopes on the same repo, so we use a different set of them for DevEdition, and
+callers are responsible for using the correct one (by calling the appropriate
+helper below). More context on this in https://bugzilla.mozilla.org/show_bug.cgi?id=1358601.
+
 We will need to add esr support at some point. Eventually we want to add
 nuance so certain m-b and m-r tasks use dep or nightly signing, and we only
 release sign when we have a signed-off set of candidate builds.  This current
 approach works for now, though.
 
 This is a list of list-pairs, for ordering.
 """
 SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
@@ -47,16 +52,27 @@ SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
 """Map the signing scope aliases to the actual scopes.
 """
 SIGNING_CERT_SCOPES = {
     'all-release-branches': 'project:releng:signing:cert:release-signing',
     'all-nightly-branches': 'project:releng:signing:cert:nightly-signing',
     'default': 'project:releng:signing:cert:dep-signing',
 }
 
+DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
+    'beta', set([
+        'mozilla-beta',
+    ])
+]]
+
+DEVEDITION_SIGNING_CERT_SCOPES = {
+    'beta': 'project:releng:signing:cert:nightly-signing',
+    'default': 'project:releng:signing:cert:dep-signing',
+}
+
 """Map beetmover scope aliases to sets of projects.
 """
 BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [[
     'all-nightly-branches', set([
         'mozilla-central',
         'mozilla-aurora',
         'mozilla-beta',
         'mozilla-release',
@@ -309,16 +325,22 @@ def get_scope_from_target_method_and_pro
 
 
 get_signing_cert_scope = functools.partial(
     get_scope_from_project,
     SIGNING_SCOPE_ALIAS_TO_PROJECT,
     SIGNING_CERT_SCOPES
 )
 
+get_devedition_signing_cert_scope = functools.partial(
+    get_scope_from_project,
+    DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT,
+    DEVEDITION_SIGNING_CERT_SCOPES
+)
+
 get_beetmover_bucket_scope = functools.partial(
     get_scope_from_target_method_and_project,
     BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
     BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
     BEETMOVER_BUCKET_SCOPES
 )
 
 get_beetmover_action_scope = functools.partial(
--- a/testing/mozharness/configs/builds/branch_specifics.py
+++ b/testing/mozharness/configs/builds/branch_specifics.py
@@ -317,16 +317,31 @@ config = {
                 'update_channel': 'default',
             },
             'win64': {
                 'src_mozconfig': 'browser/config/mozconfigs/win64/release',
             },
             'win64-debug': {
                 'update_channel': 'default',
             },
+            'linux-devedition': {
+                "update_channel": "aurora-dev",
+            },
+            'linux64-devedition': {
+                "update_channel": "aurora-dev",
+            },
+            'macosx64-devedition': {
+                "update_channel": "aurora-dev",
+            },
+            'win32-devedition': {
+                "update_channel": "aurora-dev",
+            },
+            'win64-devedition': {
+                "update_channel": "aurora-dev",
+            },
         },
         'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
     },
     'larch': {
         'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
     },
     # 'maple': {},
     'oak': {