bug 1358601: set up CI builds and tests for DevEdition on mozilla-beta - fix build configuration and signing of DevEdition-on-beta builds. r=aki,rail a=ritu
authorBen Hearsum <bhearsum@mozilla.com>
Fri, 05 May 2017 16:54:36 -0400
changeset 393848 7868898154ed9e19d98d9aa4c53b7da57c2bd187
parent 393847 93700f9bdc9cb4e278d331049dff765c75771632
child 393849 9368d544cbc447f7e8d7b4f13d0b92cf6425cb56
push id7280
push userbhearsum@mozilla.com
push dateFri, 05 May 2017 20:55:44 +0000
treeherdermozilla-beta@7868898154ed [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersaki, rail, ritu
bugs1358601
milestone54.0
bug 1358601: set up CI builds and tests for DevEdition on mozilla-beta - fix build configuration and signing of DevEdition-on-beta builds. r=aki,rail a=ritu
browser/config/mozconfigs/linux32/devedition
browser/config/mozconfigs/linux64/devedition
browser/config/mozconfigs/macosx64/devedition
browser/config/mozconfigs/win32/devedition
browser/config/mozconfigs/win64/devedition
browser/confvars.sh
taskcluster/ci/build/linux.yml
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/signing.py
taskcluster/taskgraph/util/scriptworker.py
testing/mozharness/configs/builds/branch_specifics.py
--- a/browser/config/mozconfigs/linux32/devedition
+++ b/browser/config/mozconfigs/linux32/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-1}
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/browser/config/mozconfigs/linux32/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --enable-verify-mar
 
 # This will overwrite the default of stripping everything and keep the symbol table.
--- a/browser/config/mozconfigs/linux64/devedition
+++ b/browser/config/mozconfigs/linux64/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=${MOZ_AUTOMATION_UPLOAD_SYMBOLS-1}
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/browser/config/mozconfigs/linux64/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --enable-verify-mar
 
 # This will overwrite the default of stripping everything and keep the symbol table.
--- a/browser/config/mozconfigs/macosx64/devedition
+++ b/browser/config/mozconfigs/macosx64/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=1
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/browser/config/mozconfigs/macosx64/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --disable-install-strip
 ac_add_options --enable-verify-mar
 
--- a/browser/config/mozconfigs/win32/devedition
+++ b/browser/config/mozconfigs/win32/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=1
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/build/mozconfig.win-common"
 . "$topsrcdir/browser/config/mozconfigs/win32/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --enable-verify-mar
 
--- a/browser/config/mozconfigs/win64/devedition
+++ b/browser/config/mozconfigs/win64/devedition
@@ -1,8 +1,13 @@
+if [ -n "$ENABLE_RELEASE_PROMOTION" ]; then
+  MOZ_AUTOMATION_UPLOAD_SYMBOLS=1
+  MOZ_AUTOMATION_UPDATE_PACKAGING=1
+fi
+
 . "$topsrcdir/build/mozconfig.win-common"
 . "$topsrcdir/browser/config/mozconfigs/win64/common-win64"
 . "$topsrcdir/browser/config/mozconfigs/win64/common-opt"
 
 # Add-on signing is not required for DevEdition
 MOZ_REQUIRE_SIGNING=0
 
 ac_add_options --enable-verify-mar
--- a/browser/confvars.sh
+++ b/browser/confvars.sh
@@ -13,16 +13,17 @@ if test "$OS_ARCH" = "WINNT" -o \
   MOZ_BUNDLED_FONTS=1
 fi
 
 if test "$OS_ARCH" = "WINNT"; then
   MOZ_MAINTENANCE_SERVICE=1
   if ! test "$HAVE_64BIT_BUILD"; then
     if test "$MOZ_UPDATE_CHANNEL" = "nightly" -o \
             "$MOZ_UPDATE_CHANNEL" = "aurora" -o \
+            "$MOZ_UPDATE_CHANNEL" = "aurora-dev" -o \
             "$MOZ_UPDATE_CHANNEL" = "beta" -o \
             "$MOZ_UPDATE_CHANNEL" = "beta-dev" -o \
             "$MOZ_UPDATE_CHANNEL" = "release" -o \
             "$MOZ_UPDATE_CHANNEL" = "release-dev"; then
       if ! test "$MOZ_DEBUG"; then
         MOZ_STUB_INSTALLER=1
       fi
     fi
--- a/taskcluster/ci/build/linux.yml
+++ b/taskcluster/ci/build/linux.yml
@@ -65,33 +65,37 @@ linux64/debug:
             - builds/releng_base_linux_64_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         custom-build-variant-cfg: debug
         tooltool-downloads: public
         need-xvfb: true
 
-linux64-devedition/opt:
-    description: "Linux64 devedition Opt"
+linux64-devedition-nightly/opt:
+    description: "Linux64 devedition Nightly"
+    attributes:
+        nightly: true
     index:
         product: devedition
         job-name: linux64-opt
+        type: nightly
     treeherder:
         platform: linux64-devedition/opt
-        symbol: tc(B)
+        symbol: tc(N)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
     run:
         using: mozharness
         actions: [get-secrets build check-test generate-build-stats update]
         config:
             - builds/releng_base_linux_64_builds.py
+            - disable_signing.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
         custom-build-variant-cfg: devedition
     run-on-projects: [ 'mozilla-beta', ]
 
@@ -164,33 +168,37 @@ linux/pgo:
         config:
             - builds/releng_base_linux_32_builds.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
 
-linux-devedition/opt:
-    description: "Linux32 devedition Opt"
+linux-devedition-nightly/opt:
+    description: "Linux32 devedition Nightly"
+    attributes:
+        nightly: true
     index:
         product: devedition
         job-name: linux-opt
+        type: nightly
     treeherder:
         platform: linux32-devedition/opt
-        symbol: tc(B)
+        symbol: tc(N)
     worker-type: aws-provisioner-v1/gecko-{level}-b-linux
     worker:
         implementation: docker-worker
         max-run-time: 36000
     run:
         using: mozharness
         actions: [get-secrets build check-test generate-build-stats update]
         config:
             - builds/releng_base_linux_32_builds.py
+            - disable_signing.py
             - balrog/production.py
         script: "mozharness/scripts/fx_desktop_build.py"
         secrets: true
         tooltool-downloads: public
         need-xvfb: true
         custom-build-variant-cfg: devedition
     run-on-projects: [ 'mozilla-beta', ]
 
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -92,16 +92,18 @@ from voluptuous import Schema, Any, Requ
 _MOBILE_UPSTREAM_ARTIFACTS_SIGNED_MULTI = [
     "target.apk",
 ]
 
 
 UPSTREAM_ARTIFACT_UNSIGNED_PATHS = {
     'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
+    'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-x86-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'android-api-15-nightly': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_EN_US,
     'macosx64-nightly': [],
 
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'linux-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_UNSIGNED_L10N,
     'android-x86-nightly-multi': _MOBILE_UPSTREAM_ARTIFACTS_UNSIGNED_MULTI,
     'android-api-15-nightly-l10n': ["balrog_props.json"],
@@ -112,16 +114,24 @@ UPSTREAM_ARTIFACT_SIGNED_PATHS = {
     'linux64-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
     'linux-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
+    'linux64-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
+    'linux-devedition-nightly': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_EN_US + [
+        "target.tar.bz2",
+        "target.tar.bz2.asc",
+    ],
     'android-x86-nightly': ["en-US/target.apk"],
     'android-api-15-nightly': ["en-US/target.apk"],
     'macosx64-nightly': [],
 
     'linux64-nightly-l10n': _DESKTOP_UPSTREAM_ARTIFACTS_SIGNED_L10N + [
         "target.tar.bz2",
         "target.tar.bz2.asc",
     ],
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -4,17 +4,17 @@
 """
 Transform the signing task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.schema import validate_schema
-from taskgraph.util.scriptworker import get_signing_cert_scope
+from taskgraph.util.scriptworker import get_signing_cert_scope, get_devedition_signing_cert_scope
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Schema, Any, Required, Optional
 
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
@@ -101,17 +101,24 @@ def make_task_description(config, jobs):
             'nightly': dep_job.attributes.get('nightly', False),
             'build_platform': dep_job.attributes.get('build_platform'),
             'build_type': dep_job.attributes.get('build_type'),
         }
         if dep_job.attributes.get('chunk_locales'):
             # Used for l10n attribute passthrough
             attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
 
-        signing_cert_scope = get_signing_cert_scope(config)
+        # This code wasn't originally written with the possibility of using different
+        # signing cert scopes for different platforms on the same branch. This isn't
+        # ideal, but it's what we currently have to make this possible.
+        if dep_job.attributes.get('build_platform') in set(['linux64-devedition-nightly', 'linux-devedition-nightly']):
+            signing_cert_scope = get_devedition_signing_cert_scope(config)
+        else:
+            signing_cert_scope = get_signing_cert_scope(config)
+
         task = {
             'label': label,
             'description': "{} Signing".format(
                 dep_job.task["metadata"]["description"]),
             'worker-type': "scriptworker-prov-v1/signing-linux-v1",
             'worker': {'implementation': 'scriptworker-signing',
                        'upstream-artifacts': job['upstream-artifacts'],
                        'max-run-time': 3600},
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -19,17 +19,22 @@ import os
 
 
 # constants {{{1
 GECKO = os.path.realpath(os.path.join(__file__, '..', '..', '..', '..'))
 VERSION_PATH = os.path.join(GECKO, "browser", "config", "version_display.txt")
 
 """Map signing scope aliases to sets of projects.
 
-Currently m-c and m-a use nightly signing; m-b and m-r use release signing.
+Currently m-c and DevEdition on m-b use nightly signing; Beta on m-b and m-r
+use release signing. These data structures aren't set-up to handle different
+scopes on the same repo, so we use a different set of them for DevEdition, and
+callers are responsible for using the correct one (by calling the appropriate
+helper below). More context on this in https://bugzilla.mozilla.org/show_bug.cgi?id=1358601.
+
 We will need to add esr support at some point. Eventually we want to add
 nuance so certain m-b and m-r tasks use dep or nightly signing, and we only
 release sign when we have a signed-off set of candidate builds.  This current
 approach works for now, though.
 
 This is a list of list-pairs, for ordering.
 """
 SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
@@ -47,16 +52,27 @@ SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
 """Map the signing scope aliases to the actual scopes.
 """
 SIGNING_CERT_SCOPES = {
     'all-release-branches': 'project:releng:signing:cert:release-signing',
     'all-nightly-branches': 'project:releng:signing:cert:nightly-signing',
     'default': 'project:releng:signing:cert:dep-signing',
 }
 
+DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT = [[
+    'beta', set([
+        'mozilla-beta',
+    ])
+]]
+
+DEVEDITION_SIGNING_CERT_SCOPES = {
+    'beta': 'project:releng:signing:cert:nightly-signing',
+    'default': 'project:releng:signing:cert:dep-signing',
+}
+
 """Map beetmover scope aliases to sets of projects.
 """
 BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [[
     'all-nightly-branches', set([
         'mozilla-central',
         'mozilla-aurora',
         'mozilla-beta',
         'mozilla-release',
@@ -308,16 +324,22 @@ def get_scope_from_target_method_and_pro
 
 
 get_signing_cert_scope = functools.partial(
     get_scope_from_project,
     SIGNING_SCOPE_ALIAS_TO_PROJECT,
     SIGNING_CERT_SCOPES
 )
 
+get_devedition_signing_cert_scope = functools.partial(
+    get_scope_from_project,
+    DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT,
+    DEVEDITION_SIGNING_CERT_SCOPES
+)
+
 get_beetmover_bucket_scope = functools.partial(
     get_scope_from_target_method_and_project,
     BEETMOVER_SCOPE_ALIAS_TO_TARGET_TASK,
     BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
     BEETMOVER_BUCKET_SCOPES
 )
 
 get_beetmover_action_scope = functools.partial(
--- a/testing/mozharness/configs/builds/branch_specifics.py
+++ b/testing/mozharness/configs/builds/branch_specifics.py
@@ -380,16 +380,31 @@ config = {
                 'update_channel': 'default',
             },
             'win64': {
                 'src_mozconfig': 'browser/config/mozconfigs/win64/release',
             },
             'win64-debug': {
                 'update_channel': 'default',
             },
+            'linux-devedition': {
+                "update_channel": "aurora-dev",
+            },
+            'linux64-devedition': {
+                "update_channel": "aurora-dev",
+            },
+            'macosx64-devedition': {
+                "update_channel": "aurora-dev",
+            },
+            'win32-devedition': {
+                "update_channel": "aurora-dev",
+            },
+            'win64-devedition': {
+                "update_channel": "aurora-dev",
+            },
         },
         'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
     },
     'larch': {
         'stage_server': 'upload.ffxbld.productdelivery.prod.mozaws.net',
     },
     # 'maple': {},
     'oak': {