Code cleanup DONTBUILD
authorSimon Fraser <sfraser@mozilla.com>
Wed, 06 Sep 2017 12:58:38 +0100
changeset 663921 64cc53763acd883c07141ff9e608031131ec007d
parent 663920 fdf54d666430a4e08e85540649aac46bd3fb27cb
child 663922 49c701ecf520ac1f374c672144e5ee2b72922a38
push id79558
push usercatlee@mozilla.com
push dateWed, 13 Sep 2017 13:53:28 +0000
milestone57.0a1
Code cleanup DONTBUILD
taskcluster/taskgraph/transforms/balrog.py
taskcluster/taskgraph/transforms/beetmover_partials.py
taskcluster/taskgraph/transforms/partials.py
taskcluster/taskgraph/transforms/partials_signing.py
taskcluster/taskgraph/transforms/task.py
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -51,35 +51,37 @@ def validate(config, jobs):
 
 
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
         treeherder = job.get('treeherder', {})
-        treeherder.setdefault('symbol', 'tc-Up(N)')
+        treeherder.setdefault('symbol', 'c-Up(N)')
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         treeherder.setdefault('platform',
                               "{}/opt".format(dep_th_platform))
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
 
         attributes = copy_attributes_from_dependent_job(dep_job)
 
+        locale = dep_job.attributes.get('locale', 'N')
+
         if dep_job.attributes.get('locale'):
-            treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
+            treeherder['symbol'] = 'c-Up({})'.format(locale)
             attributes['locale'] = dep_job.attributes.get('locale')
 
         label = job['label']
         if 'partials' in dep_job.kind:
             label = "partials-{}".format(label)
-            temp_locale = dep_job.attributes.get('locale', 'N')
-            treeherder['symbol'] = 'pUp({})'.format(temp_locale)
+            treeherder['symbol'] = 'cp-Up({})'.format(locale)
+
         description = (
             "Balrog submission for locale '{locale}' for build '"
             "{build_platform}/{build_type}'".format(
                 locale=attributes.get('locale', 'en-US'),
                 build_platform=attributes.get('build_platform'),
                 build_type=attributes.get('build_type')
             )
         )
@@ -93,17 +95,16 @@ def make_task_description(config, jobs):
         }]
 
         server_scope = get_balrog_server_scope(config)
         channel_scopes = get_balrog_channel_scopes(config)
 
         task = {
             'label': label,
             'description': description,
-            # do we have to define worker type somewhere?
             'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
             'worker': {
                 'implementation': 'balrog',
                 'upstream-artifacts': upstream_artifacts,
             },
             'scopes': [server_scope] + channel_scopes,
             'dependencies': {'beetmover': dep_job.label},
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/beetmover_partials.py
+++ b/taskcluster/taskgraph/transforms/beetmover_partials.py
@@ -12,20 +12,23 @@ from taskgraph.util.partials import (get
                                      get_partials_artifacts,
                                      get_partials_artifact_map)
 
 import logging
 logger = logging.getLogger(__name__)
 
 transforms = TransformSequence()
 
+
 def generate_upstream_artifacts(release_history, platform, locale=None):
     artifact_prefix = 'public/build'
     if locale:
         artifact_prefix = 'public/build/{}'.format(locale)
+    else:
+        locale = 'en-US'
 
     artifacts = get_partials_artifacts(release_history, platform, locale)
 
     upstream_artifacts = [{
         'taskId': {'task-reference': '<partials-signing>'},
         'taskType': 'signing',
         'paths': ["{}/{}".format(artifact_prefix, p)
                   for p in artifacts],
@@ -58,11 +61,12 @@ def make_partials_artifacts(config, jobs
         for artifact in artifact_map:
             extra.append({
                 'locale': locale,
                 'artifact_name': artifact,
                 'buildid': artifact_map[artifact],
                 'platform': platform,
             })
 
-        job['extra'] = {'partials': extra}
+        job.setdefault('extra', {})
+        job['extra']['partials'] = extra
 
         yield job
--- a/taskcluster/taskgraph/transforms/partials.py
+++ b/taskcluster/taskgraph/transforms/partials.py
@@ -52,38 +52,37 @@ def _generate_task_output_files(filename
 def make_task_description(config, jobs):
     # If no balrog release history, then don't generate partials
     if not config.params.get('release_history'):
         return
     for job in jobs:
         dep_job = job['dependent-task']
 
         treeherder = job.get('treeherder', {})
-        treeherder.setdefault('symbol', 'p(N)')  # TODO
+        treeherder.setdefault('symbol', 'p(N)')
 
         label = job.get('label', "partials-{}".format(dep_job.label))
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
 
         treeherder.setdefault('platform',
                               "{}/opt".format(dep_th_platform))
-        treeherder.setdefault('kind', 'build')  # TODO: update
+        treeherder.setdefault('kind', 'build')
         treeherder.setdefault('tier', 1)
 
         dependent_kind = str(dep_job.kind)
         dependencies = {dependent_kind: dep_job.label}
         signing_dependencies = dep_job.dependencies
         # This is so we get the build task etc in our dependencies to
         # have better beetmover support.
         dependencies.update(signing_dependencies)
 
         attributes = copy_attributes_from_dependent_job(dep_job)
         locale = dep_job.attributes.get('locale')
         if locale:
-            # locale = 'en-US'
             attributes['locale'] = locale
             treeherder['symbol'] = "p({})".format(locale)
 
         build_locale = locale or 'en-US'
 
         builds = get_builds(config.params['release_history'], dep_th_platform,
                             build_locale)
 
@@ -93,17 +92,17 @@ def make_task_description(config, jobs):
             continue
 
         signing_task = None
         for dependency in dependencies.keys():
             if 'repackage-signing' in dependency:
                 signing_task = dependency
         signing_task_ref = '<{}>'.format(signing_task)
 
-        extra = {'funsize': { 'partials': list()}}
+        extra = {'funsize': {'partials': list()}}
         update_number = 1
         artifact_path = "{}{}".format(_generate_taskcluster_prefix(signing_task_ref, locale=locale), 'target.complete.mar')
         for build in builds:
             extra['funsize']['partials'].append({
                 'locale': build_locale,
                 'from_mar': builds[build]['mar_url'],
                 'to_mar': {'task-reference': artifact_path},
                 'platform': get_friendly_platform_name(dep_th_platform),
@@ -113,63 +112,35 @@ def make_task_description(config, jobs):
             })
             update_number += 1
 
         cot = extra.setdefault('chainOfTrust', {})
         cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
 
         worker = {
             'artifacts': _generate_task_output_files(builds.keys(), build_locale),
-        }
-
-        run = {
-            'using': 'mozharness',
-            'script': 'mozharness/scripts/funsize.py',
-            'config': ['funsize.py'],
-            'job-script': 'taskcluster/scripts/builder/funsize.sh',
-            'actions': [],
-        }
-        level = config.params['level']
-
-        task = {
-            'label': label,
-            'description': "{} Partials".format(
-                dep_job.task["metadata"]["description"]),  # TODO reformat
-            'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
-            'dependencies': dependencies,
-            'attributes': attributes,
-            'run-on-projects': dep_job.attributes.get('run_on_projects'),
-            'treeherder': treeherder,
-            'extra': extra,
-            'worker': worker,
-            # 'run': run,
-        }
-
-        yield task
-
-
-@transforms.add
-def make_task_worker(config, jobs):
-    for job in jobs:
-        locale = job['attributes'].get('locale', 'en-US')
-
-        repackage_signing_task = 'repackage-signing'  # default
-        for dependency in job['dependencies'].keys():
-            if 'repackage-signing' in dependency:
-                repackage_signing_task = dependency
-
-        task_ref = '<' + str(repackage_signing_task) + '>'
-
-        worker = {
             'implementation': 'docker-worker',
             'docker-image': {'in-tree': 'funsize-update-generator'},
             'os': 'linux',
             'max-run-time': 3600,
             'chain-of-trust': True,
             'env': {
                 'SHA1_SIGNING_CERT': 'nightly_sha1',
                 'SHA384_SIGNING_CERT': 'nightly_sha384'
             }
         }
 
-        job["worker"].update(worker)
+        level = config.params['level']
 
-        yield job
+        task = {
+            'label': label,
+            'description': "{} Partials".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+            'extra': extra,
+            'worker': worker,
+        }
+
+        yield task
--- a/taskcluster/taskgraph/transforms/partials_signing.py
+++ b/taskcluster/taskgraph/transforms/partials_signing.py
@@ -50,17 +50,17 @@ def make_task_description(config, jobs):
 
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         label = job.get('label', "partials-signing-{}".format(dep_job.label))
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         treeherder.setdefault('platform',
                               "{}/opt".format(dep_th_platform))
-        treeherder.setdefault('kind', 'build')  # TODO: Update
+        treeherder.setdefault('kind', 'build')
         treeherder.setdefault('tier', 1)
 
         dependent_kind = str(dep_job.kind)
         dependencies = {dependent_kind: dep_job.label}
         signing_dependencies = dep_job.dependencies
         # This is so we get the build task etc in our dependencies to
         # have better beetmover support.
         dependencies.update(signing_dependencies)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -488,33 +488,33 @@ GROUP_NAMES = {
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
     'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
     'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
     'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster',
-    'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
+    'c-Up': 'Balrog submission of updates',
     'tc-cs': 'Checksum signing executed by Taskcluster',
     'tc-rs': 'Repackage signing executed by Taskcluster',
     'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'I': 'Docker Image Builds',
     'TL': 'Toolchain builds for Linux 64-bits',
     'TM': 'Toolchain builds for OSX',
     'TW32': 'Toolchain builds for Windows 32-bits',
     'TW64': 'Toolchain builds for Windows 64-bits',
     'SM-tc': 'Spidermonkey builds',
     'pub': 'APK publishing',
     'p': 'Partial generation',
     'ps': 'Partials signing',
     'pBM': 'Beetmover for partials',
-    'pUp': 'Partials Balrog upload',
+    'cp-Up': 'Balrog submission of updates, including partials',
 }
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",