Bug 1554051 - Fennec Nightly: Cannot beetmove to nightly folder: release/candidates scopes are given to the task r=mtabara a=release FENNEC_68_0b4_BUILD1
authorJohan Lorenzo <jlorenzo@mozilla.com>
Fri, 24 May 2019 09:11:10 +0200
changeset 533361 2f46b19ec841c939b84fa40be8b326a11141e6da
parent 533360 b7bf7fa2d8c476df3a2462c20259e239ab1770c0
child 533362 825151707e44f7258d09b628c5fa71f0168d076d
push id11300
push userjlorenzo@mozilla.com
push dateFri, 24 May 2019 07:49:55 +0000
treeherdermozilla-beta@2f46b19ec841 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
reviewersmtabara, release
bugs1554051
milestone68.0
Bug 1554051 - Fennec Nightly: Cannot beetmove to nightly folder: release/candidates scopes are given to the task r=mtabara a=release Differential Revision: https://phabricator.services.mozilla.com/D32423
taskcluster/taskgraph/transforms/beetmover.py
taskcluster/taskgraph/transforms/beetmover_checksums.py
taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py
taskcluster/taskgraph/transforms/beetmover_push_to_release.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
taskcluster/taskgraph/transforms/beetmover_source_checksums.py
taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
taskcluster/taskgraph/transforms/signing.py
taskcluster/taskgraph/util/scriptworker.py
--- a/taskcluster/taskgraph/transforms/beetmover.py
+++ b/taskcluster/taskgraph/transforms/beetmover.py
@@ -181,18 +181,22 @@ def make_task_description(config, jobs):
         dependencies.update(signing_dependencies)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes.update(job.get('attributes', {}))
 
         if job.get('locale'):
             attributes['locale'] = job['locale']
 
-        bucket_scope = get_beetmover_bucket_scope(config)
-        action_scope = get_beetmover_action_scope(config)
+        bucket_scope = get_beetmover_bucket_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
+        action_scope = get_beetmover_action_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
 
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, bucket_scope),
             'scopes': [bucket_scope, action_scope],
             'dependencies': dependencies,
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/beetmover_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_checksums.py
@@ -81,18 +81,22 @@ def make_beetmover_checksums_description
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes.update(job.get('attributes', {}))
 
         if dep_job.attributes.get('locale'):
             treeherder['symbol'] = 'BMcs({})'.format(dep_job.attributes.get('locale'))
             attributes['locale'] = dep_job.attributes.get('locale')
 
-        bucket_scope = get_beetmover_bucket_scope(config)
-        action_scope = get_beetmover_action_scope(config)
+        bucket_scope = get_beetmover_bucket_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
+        action_scope = get_beetmover_action_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
 
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, bucket_scope),
             'scopes': [bucket_scope, action_scope],
             'dependencies': dependencies,
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_langpack_checksums.py
@@ -63,18 +63,22 @@ def make_beetmover_checksums_description
             if k.startswith('beetmover'):
                 dependencies[k] = v
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         if 'chunk_locales' in dep_job.attributes:
             attributes['chunk_locales'] = dep_job.attributes['chunk_locales']
         attributes.update(job.get('attributes', {}))
 
-        bucket_scope = get_beetmover_bucket_scope(config)
-        action_scope = get_beetmover_action_scope(config)
+        bucket_scope = get_beetmover_bucket_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
+        action_scope = get_beetmover_action_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
 
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, bucket_scope),
             'scopes': [bucket_scope, action_scope],
             'dependencies': dependencies,
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/beetmover_push_to_release.py
+++ b/taskcluster/taskgraph/transforms/beetmover_push_to_release.py
@@ -52,17 +52,19 @@ def make_beetmover_push_to_release_descr
 
         label = job['name']
         description = (
             "Beetmover push to release for '{product}'".format(
                 product=job['product']
             )
         )
 
-        bucket_scope = get_beetmover_bucket_scope(config)
+        bucket_scope = get_beetmover_bucket_scope(
+            config, job_release_type=job.get('attributes', {}).get('release-type')
+        )
         action_scope = add_scope_prefix(config, 'beetmover:action:push-to-releases')
 
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, bucket_scope),
             'scopes': [bucket_scope, action_scope],
             'product': job['product'],
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -223,18 +223,22 @@ def make_task_description(config, jobs):
         if repackage_signing_name in upstream_deps:
             dependencies["repackage-signing"] = upstream_deps[repackage_signing_name]
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes.update(job.get('attributes', {}))
         if job.get('locale'):
             attributes['locale'] = job['locale']
 
-        bucket_scope = get_beetmover_bucket_scope(config)
-        action_scope = get_beetmover_action_scope(config)
+        bucket_scope = get_beetmover_bucket_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
+        action_scope = get_beetmover_action_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
 
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, bucket_scope),
             'scopes': [bucket_scope, action_scope],
             'dependencies': dependencies,
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage_partner.py
@@ -135,20 +135,23 @@ def populate_scopes_and_worker_type(conf
     task['partner_public'] = partner_public
     if partner_public:
         task['label'] = "{}-public".format(task['label'])
     return task
 
 
 @transforms.add
 def split_public_and_private(config, jobs):
-    public_bucket_scope = get_beetmover_bucket_scope(config)
+
     partner_config = get_partner_config_by_kind(config, config.kind)
 
     for job in jobs:
+        public_bucket_scope = get_beetmover_bucket_scope(
+            config, job_release_type=job.get('attributes', {}).get('release-type')
+        )
         partner_bucket_scope = add_scope_prefix(config, job['partner-bucket-scope'])
         partner, subpartner, _ = job['extra']['repack_id'].split('/')
 
         if partner_config[partner][subpartner].get('upload_to_candidates'):
             # public
             yield populate_scopes_and_worker_type(
                 config, job, public_bucket_scope, partner_public=True
             )
--- a/taskcluster/taskgraph/transforms/beetmover_source_checksums.py
+++ b/taskcluster/taskgraph/transforms/beetmover_source_checksums.py
@@ -65,18 +65,22 @@ def make_beetmover_checksums_description
         dependencies = {dep_job.kind: dep_job.label}
         for k, v in dep_job.dependencies.items():
             if k.startswith('beetmover'):
                 dependencies[k] = v
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes.update(job.get('attributes', {}))
 
-        bucket_scope = get_beetmover_bucket_scope(config)
-        action_scope = get_beetmover_action_scope(config)
+        bucket_scope = get_beetmover_bucket_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
+        action_scope = get_beetmover_action_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
 
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, bucket_scope),
             'scopes': [bucket_scope, action_scope],
             'dependencies': dependencies,
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
+++ b/taskcluster/taskgraph/transforms/release_beetmover_signed_addons.py
@@ -97,18 +97,22 @@ def make_task_description(config, jobs):
         job['attributes']['chunk_locales'] = dep_job.attributes['chunk_locales']
 
         job['description'] = job['description'].format(
             locales='/'.join(job['attributes']['chunk_locales']),
             platform=job['attributes']['build_platform']
         )
 
         job['scopes'] = [
-            get_beetmover_bucket_scope(config),
-            get_beetmover_action_scope(config),
+            get_beetmover_bucket_scope(
+                config, job_release_type=attributes.get('release-type')
+            ),
+            get_beetmover_action_scope(
+                config, job_release_type=attributes.get('release-type')
+            ),
         ]
 
         job['dependencies'] = {dep_job.kind: dep_job.label}
 
         job['run-on-projects'] = dep_job.attributes['run_on_projects']
         job['treeherder'] = treeherder
         job['shipping-phase'] = dep_job.attributes['shipping_phase']
         job['shipping-product'] = dep_job.attributes['shipping_product']
--- a/taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
+++ b/taskcluster/taskgraph/transforms/release_generate_checksums_beetmover.py
@@ -84,18 +84,22 @@ def make_task_description(config, jobs):
         dependencies = {dep_job.kind: dep_job.label}
 
         if len(dep_job.dependencies) > 1:
             raise NotImplementedError(
                 "Can't beetmove a signing task with multiple dependencies")
         # update the dependencies with the dependencies of the signing task
         dependencies.update(dep_job.dependencies)
 
-        bucket_scope = get_beetmover_bucket_scope(config)
-        action_scope = get_beetmover_action_scope(config)
+        bucket_scope = get_beetmover_bucket_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
+        action_scope = get_beetmover_action_scope(
+            config, job_release_type=attributes.get('release-type')
+        )
 
         task = {
             'label': label,
             'description': description,
             'worker-type': get_worker_type_for_scope(config, bucket_scope),
             'scopes': [bucket_scope, action_scope],
             'dependencies': dependencies,
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/signing.py
+++ b/taskcluster/taskgraph/transforms/signing.py
@@ -126,17 +126,18 @@ def make_task_description(config, jobs):
         attributes = copy_attributes_from_dependent_job(dep_job)
         attributes['signed'] = True
 
         if dep_job.attributes.get('chunk_locales'):
             # Used for l10n attribute passthrough
             attributes['chunk_locales'] = dep_job.attributes.get('chunk_locales')
 
         signing_cert_scope = get_signing_cert_scope_per_platform(
-            build_platform, is_nightly, config, release_type=dep_job.attributes.get('release-type')
+            build_platform, is_nightly, config,
+            job_release_type=dep_job.attributes.get('release-type')
         )
         worker_type = get_worker_type_for_scope(config, signing_cert_scope)
 
         task = {
             'label': label,
             'description': description,
             'worker-type': worker_type,
             'worker': {'implementation': 'scriptworker-signing',
--- a/taskcluster/taskgraph/util/scriptworker.py
+++ b/taskcluster/taskgraph/util/scriptworker.py
@@ -127,24 +127,52 @@ BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [[
 """Map the beetmover scope aliases to the actual scopes.
 """
 BEETMOVER_BUCKET_SCOPES = {
     'all-release-branches': 'beetmover:bucket:release',
     'all-nightly-branches': 'beetmover:bucket:nightly',
     'default': 'beetmover:bucket:dep',
 }
 
+ANDROID_BEETMOVER_SCOPE_ALIAS_TO_PROJECT = [[
+    '68-release-train', set([
+        'mozilla-beta',
+        'mozilla-release',
+        'mozilla-esr68',
+    ])
+]]
+
+ANDROID_BEETMOVER_BUCKET_SCOPES = {
+    '68-release-train': {
+        'nightly': 'beetmover:bucket:nightly',
+        'beta': 'beetmover:bucket:release',
+        'release': 'beetmover:bucket:release',
+    },
+    'default': {
+        'nightly': 'beetmover:bucket:nightly',
+        'beta': 'beetmover:bucket:release',
+        'release': 'beetmover:bucket:release',
+    },
+}
+
 """Map the beetmover tasks aliases to the actual action scopes.
 """
 BEETMOVER_ACTION_SCOPES = {
     'nightly': 'beetmover:action:push-to-nightly',
     'nightly-oak': 'beetmover:action:push-to-nightly',
     'default': 'beetmover:action:push-to-candidates',
 }
 
+ANDROID_BEETMOVER_ACTION_SCOPES = {
+    'default': {
+        'nightly': 'beetmover:action:push-to-nightly',
+        'beta': 'beetmover:action:push-to-candidates',
+        'release': 'beetmover:action:push-to-candidates',
+    },
+}
 
 """Known balrog actions."""
 BALROG_ACTIONS = ('submit-locale', 'submit-toplevel', 'schedule')
 
 """Map balrog scope aliases to sets of projects.
 
 This is a list of list-pairs, for ordering.
 """
@@ -273,23 +301,23 @@ def get_scope_from_project(config, alias
     """
     for alias, projects in alias_to_project_map:
         if config.params['project'] in projects and alias in alias_to_scope_map:
             return alias_to_scope_map[alias]
     return alias_to_scope_map['default']
 
 
 @with_scope_prefix
-def get_scope_from_project_and_release_type(
-    config, release_type, alias_to_project_map, alias_to_scope_map
+def get_scope_from_project_and_job_release_type(
+    config, job_release_type, alias_to_project_map, alias_to_scope_map
 ):
     for alias, projects in alias_to_project_map:
         if config.params['project'] in projects and alias in alias_to_scope_map:
-            return alias_to_scope_map[alias][release_type]
-    return alias_to_scope_map['default'][release_type]
+            return alias_to_scope_map[alias][job_release_type]
+    return alias_to_scope_map['default'][job_release_type]
 
 
 @with_scope_prefix
 def get_scope_from_release_type(config, release_type_to_scope_map):
     """Determine the restricted scope from `config.params['target_tasks_method']`.
 
     Args:
         config (TransformConfig): The configuration for the kind being transformed.
@@ -330,46 +358,58 @@ def get_balrog_action_scope(config, acti
 
 get_signing_cert_scope = functools.partial(
     get_scope_from_project,
     alias_to_project_map=SIGNING_SCOPE_ALIAS_TO_PROJECT,
     alias_to_scope_map=SIGNING_CERT_SCOPES,
 )
 
 get_android_signing_cert_scope = functools.partial(
-    get_scope_from_project_and_release_type,
+    get_scope_from_project_and_job_release_type,
     alias_to_project_map=ANDROID_SIGNING_SCOPE_ALIAS_TO_PROJECT,
     alias_to_scope_map=ANDROID_SIGNING_CERT_SCOPES,
 )
 
 get_devedition_signing_cert_scope = functools.partial(
     get_scope_from_project,
     alias_to_project_map=DEVEDITION_SIGNING_SCOPE_ALIAS_TO_PROJECT,
     alias_to_scope_map=DEVEDITION_SIGNING_CERT_SCOPES,
 )
 
-get_beetmover_bucket_scope = functools.partial(
+get_beetmover_regular_bucket_scope = functools.partial(
     get_scope_from_project,
     alias_to_project_map=BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
     alias_to_scope_map=BEETMOVER_BUCKET_SCOPES,
 )
 
-get_beetmover_action_scope = functools.partial(
+get_beetmover_regular_action_scope = functools.partial(
     get_scope_from_release_type,
     release_type_to_scope_map=BEETMOVER_ACTION_SCOPES,
 )
 
+get_beetmover_android_bucket_scope = functools.partial(
+    get_scope_from_project_and_job_release_type,
+    alias_to_project_map=ANDROID_BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
+    alias_to_scope_map=ANDROID_BEETMOVER_BUCKET_SCOPES,
+)
+
+get_beetmover_android_action_scope = functools.partial(
+    get_scope_from_project_and_job_release_type,
+    alias_to_project_map=ANDROID_BEETMOVER_SCOPE_ALIAS_TO_PROJECT,
+    alias_to_scope_map=ANDROID_BEETMOVER_ACTION_SCOPES,
+)
+
 get_balrog_server_scope = functools.partial(
     get_scope_from_project,
     alias_to_project_map=BALROG_SCOPE_ALIAS_TO_PROJECT,
     alias_to_scope_map=BALROG_SERVER_SCOPES,
 )
 
 get_push_apk_scope = functools.partial(
-    get_scope_from_project_and_release_type,
+    get_scope_from_project_and_job_release_type,
     alias_to_project_map=PUSH_APK_SCOPE_ALIAS_TO_PROJECT,
     alias_to_scope_map=PUSH_APK_SCOPES,
 )
 
 cached_load_yaml = memoize(load_yaml)
 
 
 # release_config {{{1
@@ -406,27 +446,41 @@ def get_release_config(config):
     release_config['version'] = str(config.params['version'])
     release_config['appVersion'] = str(config.params['app_version'])
 
     release_config['next_version'] = str(config.params['next_version'])
     release_config['build_number'] = config.params['build_number']
     return release_config
 
 
-def get_signing_cert_scope_per_platform(build_platform, is_nightly, config, release_type=None):
-    if 'android' in build_platform and release_type is not None:
-        return get_android_signing_cert_scope(config, release_type)
+def get_signing_cert_scope_per_platform(build_platform, is_nightly, config, job_release_type=None):
+    if 'android' in build_platform and job_release_type is not None:
+        return get_android_signing_cert_scope(config, job_release_type)
     if 'devedition' in build_platform:
         return get_devedition_signing_cert_scope(config)
     elif is_nightly or build_platform in ('firefox-source', 'fennec-source', 'thunderbird-source'):
         return get_signing_cert_scope(config)
     else:
         return add_scope_prefix(config, 'signing:cert:dep-signing')
 
 
+def get_beetmover_bucket_scope(config, job_release_type=None):
+    if job_release_type:
+        return get_beetmover_android_bucket_scope(config, job_release_type)
+    else:
+        return get_beetmover_regular_bucket_scope(config)
+
+
+def get_beetmover_action_scope(config, job_release_type=None):
+    if job_release_type:
+        return get_beetmover_android_action_scope(config, job_release_type)
+    else:
+        return get_beetmover_regular_action_scope(config)
+
+
 def get_worker_type_for_scope(config, scope):
     """Get the scriptworker type that will accept the given scope.
 
     Args:
         config (TransformConfig): The configuration for the kind being transformed.
         scope (string): The scope being used.
 
     Returns: