Bug 1342392 Migrate partial update generation in-tree r=rail draft
authorSimon Fraser <sfraser@mozilla.com>
Mon, 18 Sep 2017 13:36:36 +0100
changeset 666295 dbea33ec8d96cc9a0e016e4828f9f989fca7131b
parent 666274 ffe6cc09ccf38cca6f0e727837bbc6cb722d1e71
child 732051 95ec6f8f5532fdabab3907883882828945d43a9a
push id80352
push usersfraser@mozilla.com
push dateMon, 18 Sep 2017 12:37:58 +0000
reviewersrail
bugs1342392
milestone57.0a1
Bug 1342392 Migrate partial update generation in-tree r=rail MozReview-Commit-ID: G6EFXY0UzSa
taskcluster/ci/beetmover-repackage/kind.yml
taskcluster/ci/docker-image/kind.yml
taskcluster/ci/partials-signing/kind.yml
taskcluster/ci/partials/kind.yml
taskcluster/docker/funsize-update-generator/Dockerfile
taskcluster/docker/funsize-update-generator/scripts/funsize.py
taskcluster/docs/kinds.rst
taskcluster/docs/parameters.rst
taskcluster/mach_commands.py
taskcluster/taskgraph/decision.py
taskcluster/taskgraph/parameters.py
taskcluster/taskgraph/transforms/balrog.py
taskcluster/taskgraph/transforms/beetmover_repackage.py
taskcluster/taskgraph/transforms/partials.py
taskcluster/taskgraph/transforms/partials_signing.py
taskcluster/taskgraph/transforms/repackage.py
taskcluster/taskgraph/transforms/repackage_signing.py
taskcluster/taskgraph/transforms/task.py
taskcluster/taskgraph/util/partials.py
taskcluster/taskgraph/util/taskcluster.py
--- a/taskcluster/ci/beetmover-repackage/kind.yml
+++ b/taskcluster/ci/beetmover-repackage/kind.yml
@@ -7,15 +7,16 @@ loader: taskgraph.loader.single_dep:load
 transforms:
    - taskgraph.transforms.name_sanity:transforms
    - taskgraph.transforms.beetmover_repackage_l10n:transforms
    - taskgraph.transforms.beetmover_repackage:transforms
    - taskgraph.transforms.task:transforms
 
 kind-dependencies:
   - repackage-signing
+  - partials-signing
 
 only-for-build-platforms:
   - linux-nightly/opt
   - linux64-nightly/opt
   - macosx64-nightly/opt
   - win32-nightly/opt
   - win64-nightly/opt
--- a/taskcluster/ci/docker-image/kind.yml
+++ b/taskcluster/ci/docker-image/kind.yml
@@ -21,8 +21,10 @@ jobs:
   valgrind-build:
     symbol: I(vb)
   lint:
     symbol: I(lnt)
   android-gradle-build:
     symbol: I(agb)
   index-task:
     symbol: I(idx)
+  funsize-update-generator:
+    symbol: I(pg)
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/partials-signing/kind.yml
@@ -0,0 +1,13 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+  - taskgraph.transforms.name_sanity:transforms
+  - taskgraph.transforms.partials_signing:transforms
+  - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - partials
new file mode 100644
--- /dev/null
+++ b/taskcluster/ci/partials/kind.yml
@@ -0,0 +1,23 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+loader: taskgraph.loader.single_dep:loader
+
+transforms:
+  - taskgraph.transforms.name_sanity:transforms
+  - taskgraph.transforms.partials:transforms
+  - taskgraph.transforms.task:transforms
+
+kind-dependencies:
+  - repackage-signing
+
+only-for-attributes:
+  - nightly
+
+only-for-build-platforms:
+  - macosx64-nightly/opt
+  - win32-nightly/opt
+  - win64-nightly/opt
+  - linux-nightly/opt
+  - linux64-nightly/opt
--- a/taskcluster/docker/funsize-update-generator/Dockerfile
+++ b/taskcluster/docker/funsize-update-generator/Dockerfile
@@ -20,18 +20,21 @@ RUN for i in 1 2 3 4 5; do freshclam --v
 # python-pip installs a lot of dependencies increasing the size of an image
 # drastically. Using easy_install saves us almost 200M.
 RUN easy_install pip
 RUN pip install -r /tmp/requirements.txt
 
 # scripts
 RUN mkdir /home/worker/bin
 COPY scripts/* /home/worker/bin/
+
 COPY runme.sh /runme.sh
 COPY recompress.sh /recompress.sh
 RUN chmod 755 /home/worker/bin/* /*.sh
 RUN mkdir /home/worker/keys
 COPY *.pubkey /home/worker/keys/
 
 ENV           HOME          /home/worker
 ENV           SHELL         /bin/bash
 ENV           USER          worker
 ENV           LOGNAME       worker
+
+CMD ["/runme.sh"]
--- a/taskcluster/docker/funsize-update-generator/scripts/funsize.py
+++ b/taskcluster/docker/funsize-update-generator/scripts/funsize.py
@@ -1,10 +1,14 @@
 #!/usr/bin/env python
-from __future__ import absolute_import, print_function
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
 
 import ConfigParser
 import argparse
 import functools
 import hashlib
 import json
 import logging
 import os
@@ -22,16 +26,17 @@ log = logging.getLogger(__name__)
 ALLOWED_URL_PREFIXES = [
     "http://download.cdn.mozilla.net/pub/mozilla.org/firefox/nightly/",
     "http://download.cdn.mozilla.net/pub/firefox/nightly/",
     "https://mozilla-nightly-updates.s3.amazonaws.com",
     "https://queue.taskcluster.net/",
     "http://ftp.mozilla.org/",
     "http://download.mozilla.org/",
     "https://archive.mozilla.org/",
+    "https://queue.taskcluster.net/v1/task/",
 ]
 
 DEFAULT_FILENAME_TEMPLATE = "{appName}-{branch}-{version}-{platform}-" \
                             "{locale}-{from_buildid}-{to_buildid}.partial.mar"
 
 
 def verify_signature(mar, certs):
     log.info("Checking %s signature", mar)
@@ -281,17 +286,21 @@ def main():
                       "previousBuildNumber", "toVersion",
                       "toBuildNumber"):
             if field in e:
                 mar_data[field] = e[field]
         mar_data.update(complete_mars)
         # if branch not set explicitly use repo-name
         mar_data["branch"] = e.get("branch",
                                    mar_data["repo"].rstrip("/").split("/")[-1])
-        mar_name = args.filename_template.format(**mar_data)
+        if 'dest_mar' in e:
+            mar_name = e['dest_mar']
+        else:
+            # default to formatted name if not specified
+            mar_name = args.filename_template.format(**mar_data)
         mar_data["mar"] = mar_name
         dest_mar = os.path.join(work_env.workdir, mar_name)
         # TODO: download these once
         work_env.download_buildsystem_bits(repo=mar_data["repo"],
                                            revision=mar_data["revision"])
         generate_partial(work_env, from_path, path, dest_mar,
                          mar_data["ACCEPTED_MAR_CHANNEL_IDS"],
                          mar_data["version"],
--- a/taskcluster/docs/kinds.rst
+++ b/taskcluster/docs/kinds.rst
@@ -228,8 +228,18 @@ repackage-l10n
 --------------
 Repackage-L10n is a ```Repackage``` task split up to be suitable for use after l10n repacks.
 
 
 repackage-signing
 -----------------
 Repackage-signing take the repackaged installers (windows) and update packaging (with
 the signed internal bits) and signs them.
+
+partials
+--------
+Partials takes the complete.mar files produced in previous tasks and generates partial
+updates between previous nightly releases and the new one. Requires a release_history
+in the parameters. See ``mach release-history`` if doing this manually.
+
+partials-signing
+----------------
+Partials-signing takes the partial updates produced in Partials and signs them.
--- a/taskcluster/docs/parameters.rst
+++ b/taskcluster/docs/parameters.rst
@@ -102,16 +102,22 @@ syntax or reading a project-specific con
     one of the functions in ``taskcluster/taskgraph/target_tasks.py``.
 
 ``optimize_target_tasks``
     If true, then target tasks are eligible for optimization.
 
 ``include_nightly``
     If true, then nightly tasks are eligible for optimization.
 
+``release_history``
+   History of recent releases by platform and locale, used when generating
+   partial updates for nightly releases.
+   Suitable contents can be generated with ``mach release-history``,
+   which will print to the console by default.
+
 Morphed Set
 -----------
 
 ``morph_templates``
     Dict of JSON-e templates to apply to each task, keyed by template name.
     Values are extra context that will be available to the template under the
     ``input.<template>`` key. Available templates live in
     ``taskcluster/taskgraph/templates``. Enabled on try only.
--- a/taskcluster/mach_commands.py
+++ b/taskcluster/mach_commands.py
@@ -499,8 +499,28 @@ class TaskClusterImagesProvider(object):
         try:
             if context_only is None:
                 build_image(image_name)
             else:
                 build_context(image_name, context_only)
         except Exception:
             traceback.print_exc()
             sys.exit(1)
+
+
+@CommandProvider
+class TaskClusterPartialsData(object):
+    @Command('release-history', category="ci",
+             description="Query balrog for release history used by enable partials generation")
+    @CommandArgument('-b', '--branch',
+                     help="The gecko project branch used in balrog, such as "
+                          "mozilla-central, release, date")
+    @CommandArgument('--product', default='Firefox',
+                     help="The product identifier, such as 'Firefox'")
+    def generate_partials_builds(self, product, branch):
+        from taskgraph.util.partials import populate_release_history
+        try:
+            import yaml
+            release_history = {'release_history': populate_release_history(product, branch)}
+            print(yaml.safe_dump(release_history, allow_unicode=True, default_flow_style=False))
+        except Exception:
+            traceback.print_exc()
+            sys.exit(1)
--- a/taskcluster/taskgraph/decision.py
+++ b/taskcluster/taskgraph/decision.py
@@ -13,16 +13,17 @@ import re
 import time
 import yaml
 
 from .generator import TaskGraphGenerator
 from .create import create_tasks
 from .parameters import Parameters
 from .taskgraph import TaskGraph
 from .actions import render_actions_json
+from taskgraph.util.partials import populate_release_history
 from . import GECKO
 
 from taskgraph.util.templates import Templates
 from taskgraph.util.time import (
     json_time_from_now,
     current_json_time,
 )
 
@@ -102,16 +103,17 @@ def taskgraph_decision(options):
      * processing decision task command-line options into parameters
      * running task-graph generation exactly the same way the other `mach
        taskgraph` commands do
      * generating a set of artifacts to memorialize the graph
      * calling TaskCluster APIs to create the graph
     """
 
     parameters = get_decision_parameters(options)
+
     # create a TaskGraphGenerator instance
     tgg = TaskGraphGenerator(
         root_dir=options['root'],
         parameters=parameters)
 
     # write out the parameters used to generate this graph
     write_artifact('parameters.yml', dict(**parameters))
 
@@ -197,16 +199,23 @@ def get_decision_parameters(options):
             task_config = json.load(fh)
         parameters['morph_templates'] = task_config.get('templates', {})
         parameters['target_task_labels'] = task_config.get('tasks')
 
     # `target_tasks_method` has higher precedence than `project` parameters
     if options.get('target_tasks_method'):
         parameters['target_tasks_method'] = options['target_tasks_method']
 
+    # If the target method is nightly, we should build partials. This means
+    # knowing what has been released previously.
+    # An empty release_history is fine, it just means no partials will be built
+    parameters.setdefault('release_history', dict())
+    if 'nightly' in parameters.get('target_tasks_method', ''):
+        parameters['release_history'] = populate_release_history('Firefox', project)
+
     return Parameters(parameters)
 
 
 def write_artifact(filename, data):
     logger.info('writing artifact file `{}`'.format(filename))
     if not os.path.isdir(ARTIFACTS_DIR):
         os.mkdir(ARTIFACTS_DIR)
     path = os.path.join(ARTIFACTS_DIR, filename)
--- a/taskcluster/taskgraph/parameters.py
+++ b/taskcluster/taskgraph/parameters.py
@@ -23,16 +23,17 @@ PARAMETER_NAMES = set([
     'message',
     'morph_templates',
     'moz_build_date',
     'optimize_target_tasks',
     'owner',
     'project',
     'pushdate',
     'pushlog_id',
+    'release_history',
     'target_task_labels',
     'target_tasks_method',
 ])
 
 TRY_ONLY_PARAMETERS = set([
     'morph_templates',
     'target_task_labels',
 ])
--- a/taskcluster/taskgraph/transforms/balrog.py
+++ b/taskcluster/taskgraph/transforms/balrog.py
@@ -51,31 +51,34 @@ def validate(config, jobs):
 
 
 @transforms.add
 def make_task_description(config, jobs):
     for job in jobs:
         dep_job = job['dependent-task']
 
         treeherder = job.get('treeherder', {})
-        treeherder.setdefault('symbol', 'tc-Up(N)')
+        treeherder.setdefault('symbol', 'c-Up(N)')
         dep_th_platform = dep_job.task.get('extra', {}).get(
             'treeherder', {}).get('machine', {}).get('platform', '')
         treeherder.setdefault('platform',
                               "{}/opt".format(dep_th_platform))
         treeherder.setdefault('tier', 1)
         treeherder.setdefault('kind', 'build')
 
         attributes = copy_attributes_from_dependent_job(dep_job)
 
+        treeherder_job_symbol = dep_job.attributes.get('locale', 'N')
+
         if dep_job.attributes.get('locale'):
-            treeherder['symbol'] = 'tc-Up({})'.format(dep_job.attributes.get('locale'))
+            treeherder['symbol'] = 'c-Up({})'.format(treeherder_job_symbol)
             attributes['locale'] = dep_job.attributes.get('locale')
 
         label = job['label']
+
         description = (
             "Balrog submission for locale '{locale}' for build '"
             "{build_platform}/{build_type}'".format(
                 locale=attributes.get('locale', 'en-US'),
                 build_platform=attributes.get('build_platform'),
                 build_type=attributes.get('build_type')
             )
         )
@@ -89,17 +92,16 @@ def make_task_description(config, jobs):
         }]
 
         server_scope = get_balrog_server_scope(config)
         channel_scopes = get_balrog_channel_scopes(config)
 
         task = {
             'label': label,
             'description': description,
-            # do we have to define worker type somewhere?
             'worker-type': 'scriptworker-prov-v1/balrogworker-v1',
             'worker': {
                 'implementation': 'balrog',
                 'upstream-artifacts': upstream_artifacts,
             },
             'scopes': [server_scope] + channel_scopes,
             'dependencies': {'beetmover': dep_job.label},
             'attributes': attributes,
--- a/taskcluster/taskgraph/transforms/beetmover_repackage.py
+++ b/taskcluster/taskgraph/transforms/beetmover_repackage.py
@@ -4,16 +4,19 @@
 """
 Transform the beetmover task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.partials import (get_balrog_platform_name,
+                                     get_partials_artifacts,
+                                     get_partials_artifact_map)
 from taskgraph.util.schema import validate_schema, Schema
 from taskgraph.util.scriptworker import (get_beetmover_bucket_scope,
                                          get_beetmover_action_scope)
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
 import logging
 import re
@@ -210,16 +213,24 @@ def make_task_description(config, jobs):
 
         repackage_name = "repackage"
         # repackage-l10n actually uses the repackage depname here
         repackage_dependencies = {"repackage":
                                   dep_job.dependencies[repackage_name]
                                   }
         dependencies.update(repackage_dependencies)
 
+        # If this isn't a direct dependency, it won't be in there.
+        if 'repackage-signing' not in dependencies:
+            repackage_signing_name = "repackage-signing"
+            repackage_signing_deps = {"repackage-signing":
+                                      dep_job.dependencies[repackage_signing_name]
+                                      }
+            dependencies.update(repackage_signing_deps)
+
         attributes = copy_attributes_from_dependent_job(dep_job)
         if job.get('locale'):
             attributes['locale'] = job['locale']
 
         bucket_scope = get_beetmover_bucket_scope(config)
         action_scope = get_beetmover_action_scope(config)
 
         task = {
@@ -268,44 +279,64 @@ def generate_upstream_artifacts(build_ta
 
     for ref, tasktype, mapping in zip(task_refs, tasktypes, mapping):
         plarform_was_previously_matched_by_regex = None
         for platform_regex, paths in mapping.iteritems():
             if platform_regex.match(platform) is not None:
                 _check_platform_matched_only_one_regex(
                     tasktype, platform, plarform_was_previously_matched_by_regex, platform_regex
                 )
-
                 upstream_artifacts.append({
                     "taskId": {"task-reference": ref},
                     "taskType": tasktype,
                     "paths": ["{}/{}".format(artifact_prefix, path) for path in paths],
                     "locale": locale or "en-US",
                 })
                 plarform_was_previously_matched_by_regex = platform_regex
 
     return upstream_artifacts
 
 
+def generate_partials_upstream_artifacts(artifacts, platform, locale=None):
+    if not locale or locale == 'en-US':
+        artifact_prefix = 'public/build'
+    else:
+        artifact_prefix = 'public/build/{}'.format(locale)
+
+    upstream_artifacts = [{
+        'taskId': {'task-reference': '<partials-signing>'},
+        'taskType': 'signing',
+        'paths': ["{}/{}".format(artifact_prefix, p)
+                  for p in artifacts],
+        'locale': locale or 'en-US',
+    }]
+
+    return upstream_artifacts
+
+
 def _check_platform_matched_only_one_regex(
     task_type, platform, plarform_was_previously_matched_by_regex, platform_regex
 ):
     if plarform_was_previously_matched_by_regex is not None:
         raise Exception('In task type "{task_type}", platform "{platform}" matches at \
 least 2 regular expressions. First matched: "{first_matched}". Second matched: \
 "{second_matched}"'.format(
             task_type=task_type, platform=platform,
             first_matched=plarform_was_previously_matched_by_regex.pattern,
             second_matched=platform_regex.pattern
         ))
 
 
 def is_valid_beetmover_job(job):
-    # windows builds don't have docker-image, so fewer dependencies
-    if any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):
+    # beetmover after partials-signing should have six dependencies.
+    # windows builds w/o partials don't have docker-image, so fewer
+    # dependencies
+    if 'partials-signing' in job['dependencies'].keys():
+        expected_dep_count = 6
+    elif any(b in job['attributes']['build_platform'] for b in _WINDOWS_BUILD_PLATFORMS):
         expected_dep_count = 4
     else:
         expected_dep_count = 5
 
     return (len(job["dependencies"]) == expected_dep_count and
             any(['repackage' in j for j in job['dependencies']]))
 
 
@@ -316,16 +347,17 @@ def make_task_worker(config, jobs):
             raise NotImplementedError("Beetmover_repackage must have five dependencies.")
 
         locale = job["attributes"].get("locale")
         platform = job["attributes"]["build_platform"]
         build_task = None
         build_signing_task = None
         repackage_task = None
         repackage_signing_task = None
+
         for dependency in job["dependencies"].keys():
             if 'repackage-signing' in dependency:
                 repackage_signing_task = dependency
             elif 'repackage' in dependency:
                 repackage_task = dependency
             elif 'signing' in dependency:
                 # catches build-signing and nightly-l10n-signing
                 build_signing_task = dependency
@@ -343,8 +375,62 @@ def make_task_worker(config, jobs):
 
         worker = {'implementation': 'beetmover',
                   'upstream-artifacts': upstream_artifacts}
         if locale:
             worker["locale"] = locale
         job["worker"] = worker
 
         yield job
+
+
+@transforms.add
+def make_partials_artifacts(config, jobs):
+    for job in jobs:
+        locale = job["attributes"].get("locale")
+        if not locale:
+            locale = 'en-US'
+
+        # Remove when proved reliable
+        # job['treeherder']['tier'] = 3
+
+        platform = job["attributes"]["build_platform"]
+
+        balrog_platform = get_balrog_platform_name(platform)
+
+        artifacts = get_partials_artifacts(config.params.get('release_history'),
+                                           balrog_platform, locale)
+
+        # Dependency:        | repackage-signing | partials-signing
+        # Partials artifacts |              Skip | Populate & yield
+        # No partials        |             Yield |         continue
+        if len(artifacts) == 0:
+            if 'partials-signing' in job['dependencies']:
+                continue
+            else:
+                yield job
+                continue
+        else:
+            if 'partials-signing' not in job['dependencies']:
+                continue
+
+        upstream_artifacts = generate_partials_upstream_artifacts(
+            artifacts, balrog_platform, locale
+        )
+
+        job['worker']['upstream-artifacts'].extend(upstream_artifacts)
+
+        extra = list()
+
+        artifact_map = get_partials_artifact_map(
+            config.params.get('release_history'), balrog_platform, locale)
+        for artifact in artifact_map:
+            extra.append({
+                'locale': locale,
+                'artifact_name': artifact,
+                'buildid': artifact_map[artifact],
+                'platform': balrog_platform,
+            })
+
+        job.setdefault('extra', {})
+        job['extra']['partials'] = extra
+
+        yield job
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/partials.py
@@ -0,0 +1,137 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the partials task into an actual task description.
+"""
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.partials import get_balrog_platform_name, get_builds
+from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
+
+import logging
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+def _generate_task_output_files(filenames, locale=None):
+    locale_output_path = '{}/'.format(locale) if locale else ''
+
+    data = list()
+    for filename in filenames:
+        data.append({
+            'type': 'file',
+            'path': '/home/worker/artifacts/{}'.format(filename),
+            'name': 'public/build/{}{}'.format(locale_output_path, filename)
+        })
+    data.append({
+        'type': 'file',
+        'path': '/home/worker/artifacts/manifest.json',
+        'name': 'public/build/{}manifest.json'.format(locale_output_path)
+    })
+    return data
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    # If no balrog release history, then don't generate partials
+    if not config.params.get('release_history'):
+        return
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'p(N)')
+
+        label = job.get('label', "partials-{}".format(dep_job.label))
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('kind', 'build')
+        treeherder.setdefault('tier', 1)
+
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+        signing_dependencies = dep_job.dependencies
+        # This is so we get the build task etc in our dependencies to
+        # have better beetmover support.
+        dependencies.update(signing_dependencies)
+
+        attributes = copy_attributes_from_dependent_job(dep_job)
+        locale = dep_job.attributes.get('locale')
+        if locale:
+            attributes['locale'] = locale
+            treeherder['symbol'] = "p({})".format(locale)
+
+        build_locale = locale or 'en-US'
+
+        builds = get_builds(config.params['release_history'], dep_th_platform,
+                            build_locale)
+
+        # If the list is empty there's no available history for this platform
+        # and locale combination, so we can't build any partials.
+        if not builds:
+            continue
+
+        signing_task = None
+        for dependency in sorted(dependencies.keys()):
+            if 'repackage-signing' in dependency:
+                signing_task = dependency
+                break
+        signing_task_ref = '<{}>'.format(signing_task)
+
+        extra = {'funsize': {'partials': list()}}
+        update_number = 1
+        artifact_path = "{}{}".format(
+            get_taskcluster_artifact_prefix(signing_task_ref, locale=locale),
+            'target.complete.mar'
+        )
+        for build in builds:
+            extra['funsize']['partials'].append({
+                'locale': build_locale,
+                'from_mar': builds[build]['mar_url'],
+                'to_mar': {'task-reference': artifact_path},
+                'platform': get_balrog_platform_name(dep_th_platform),
+                'branch': config.params['project'],
+                'update_number': update_number,
+                'dest_mar': build,
+            })
+            update_number += 1
+
+        cot = extra.setdefault('chainOfTrust', {})
+        cot.setdefault('inputs', {})['docker-image'] = {"task-reference": "<docker-image>"}
+
+        worker = {
+            'artifacts': _generate_task_output_files(builds.keys(), locale),
+            'implementation': 'docker-worker',
+            'docker-image': {'in-tree': 'funsize-update-generator'},
+            'os': 'linux',
+            'max-run-time': 3600,
+            'chain-of-trust': True,
+            'env': {
+                'SHA1_SIGNING_CERT': 'nightly_sha1',
+                'SHA384_SIGNING_CERT': 'nightly_sha384'
+            }
+        }
+
+        level = config.params['level']
+
+        task = {
+            'label': label,
+            'description': "{} Partials".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': 'aws-provisioner-v1/gecko-%s-b-linux' % level,
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+            'extra': extra,
+            'worker': worker,
+        }
+
+        yield task
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/transforms/partials_signing.py
@@ -0,0 +1,96 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+"""
+Transform the partials task into an actual task description.
+"""
+from __future__ import absolute_import, print_function, unicode_literals
+
+from taskgraph.transforms.base import TransformSequence
+from taskgraph.util.attributes import copy_attributes_from_dependent_job
+from taskgraph.util.scriptworker import get_signing_cert_scope_per_platform
+from taskgraph.util.partials import get_balrog_platform_name, get_partials_artifacts
+
+import logging
+logger = logging.getLogger(__name__)
+
+transforms = TransformSequence()
+
+
+def generate_upstream_artifacts(release_history, platform, locale=None):
+    artifact_prefix = 'public/build'
+    if locale:
+        artifact_prefix = 'public/build/{}'.format(locale)
+    else:
+        locale = 'en-US'
+
+    artifacts = get_partials_artifacts(release_history, platform, locale)
+
+    upstream_artifacts = [{
+        "taskId": {"task-reference": '<partials>'},
+        "taskType": 'partials',
+        "paths": ["{}/{}".format(artifact_prefix, p)
+                  for p in artifacts],
+        "formats": ["mar_sha384"],
+    }]
+
+    return upstream_artifacts
+
+
+@transforms.add
+def make_task_description(config, jobs):
+    for job in jobs:
+        dep_job = job['dependent-task']
+
+        treeherder = job.get('treeherder', {})
+        treeherder.setdefault('symbol', 'ps(N)')
+
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        label = job.get('label', "partials-signing-{}".format(dep_job.label))
+        dep_th_platform = dep_job.task.get('extra', {}).get(
+            'treeherder', {}).get('machine', {}).get('platform', '')
+        treeherder.setdefault('platform',
+                              "{}/opt".format(dep_th_platform))
+        treeherder.setdefault('kind', 'build')
+        treeherder.setdefault('tier', 1)
+
+        dependent_kind = str(dep_job.kind)
+        dependencies = {dependent_kind: dep_job.label}
+        signing_dependencies = dep_job.dependencies
+        # This is so we get the build task etc in our dependencies to
+        # have better beetmover support.
+        dependencies.update(signing_dependencies)
+
+        attributes = copy_attributes_from_dependent_job(dep_job)
+        locale = dep_job.attributes.get('locale')
+        if locale:
+            attributes['locale'] = locale
+            treeherder['symbol'] = 'ps({})'.format(locale)
+
+        balrog_platform = get_balrog_platform_name(dep_th_platform)
+        upstream_artifacts = generate_upstream_artifacts(
+            config.params['release_history'], balrog_platform, locale)
+
+        build_platform = dep_job.attributes.get('build_platform')
+        is_nightly = dep_job.attributes.get('nightly')
+        signing_cert_scope = get_signing_cert_scope_per_platform(
+            build_platform, is_nightly, config
+        )
+        scopes = [signing_cert_scope, 'project:releng:signing:format:mar_sha384']
+        task = {
+            'label': label,
+            'description': "{} Partials".format(
+                dep_job.task["metadata"]["description"]),
+            'worker-type': 'scriptworker-prov-v1/signing-linux-v1',
+            'worker': {'implementation': 'scriptworker-signing',
+                           'upstream-artifacts': upstream_artifacts,
+                           'max-run-time': 3600},
+            'dependencies': dependencies,
+            'attributes': attributes,
+            'scopes': scopes,
+            'run-on-projects': dep_job.attributes.get('run_on_projects'),
+            'treeherder': treeherder,
+        }
+
+        yield task
--- a/taskcluster/taskgraph/transforms/repackage.py
+++ b/taskcluster/taskgraph/transforms/repackage.py
@@ -5,22 +5,20 @@
 Transform the repackage task into an actual task description.
 """
 
 from __future__ import absolute_import, print_function, unicode_literals
 
 from taskgraph.transforms.base import TransformSequence
 from taskgraph.util.attributes import copy_attributes_from_dependent_job
 from taskgraph.util.schema import validate_schema, Schema
+from taskgraph.util.taskcluster import get_taskcluster_artifact_prefix
 from taskgraph.transforms.task import task_description_schema
 from voluptuous import Any, Required, Optional
 
-_TC_ARTIFACT_LOCATION = \
-        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
-
 transforms = TransformSequence()
 
 # Voluptuous uses marker objects as dictionary *keys*, but they are not
 # comparable, so we cast all of the keys back to regular strings
 task_description_schema = {str(k): v for k, v in task_description_schema.schema.iteritems()}
 
 # shortcut for a string where task references are allowed
 taskref_or_string = Any(
@@ -198,18 +196,18 @@ def _generate_task_mozharness_config(bui
             return ['repackage/linux{}_signed.py'.format(bits)]
         elif build_platform.startswith('win'):
             return ['repackage/win{}_signed.py'.format(bits)]
 
     raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
 
 
 def _generate_task_env(build_platform, build_task_ref, signing_task_ref, locale=None):
-    mar_prefix = _generate_taskcluster_prefix(build_task_ref, postfix='host/bin/', locale=None)
-    signed_prefix = _generate_taskcluster_prefix(signing_task_ref, locale=locale)
+    mar_prefix = get_taskcluster_artifact_prefix(build_task_ref, postfix='host/bin/', locale=None)
+    signed_prefix = get_taskcluster_artifact_prefix(signing_task_ref, locale=locale)
 
     if build_platform.startswith('linux') or build_platform.startswith('macosx'):
         tarball_extension = 'bz2' if build_platform.startswith('linux') else 'gz'
         return {
             'SIGNED_INPUT': {'task-reference': '{}target.tar.{}'.format(
                 signed_prefix, tarball_extension
             )},
             'UNSIGNED_MAR': {'task-reference': '{}mar'.format(mar_prefix)},
@@ -226,23 +224,16 @@ def _generate_task_env(build_platform, b
             task_env['SIGNED_SETUP_STUB'] = {
                 'task-reference': '{}setup-stub.exe'.format(signed_prefix),
             }
         return task_env
 
     raise NotImplementedError('Unsupported build_platform: "{}"'.format(build_platform))
 
 
-def _generate_taskcluster_prefix(task_id, postfix='', locale=None):
-    if locale:
-        postfix = '{}/{}'.format(locale, postfix)
-
-    return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)
-
-
 def _generate_task_output_files(build_platform, locale=None):
     locale_output_path = '{}/'.format(locale) if locale else ''
 
     if build_platform.startswith('linux') or build_platform.startswith('macosx'):
         output_files = [{
             'type': 'file',
             'path': '/builds/worker/workspace/build/artifacts/{}target.complete.mar'
                     .format(locale_output_path),
--- a/taskcluster/taskgraph/transforms/repackage_signing.py
+++ b/taskcluster/taskgraph/transforms/repackage_signing.py
@@ -124,28 +124,14 @@ def make_repackage_signing_description(c
                        'max-run-time': 3600},
             'scopes': scopes,
             'dependencies': dependencies,
             'attributes': attributes,
             'run-on-projects': dep_job.attributes.get('run_on_projects'),
             'treeherder': treeherder,
         }
 
-        funsize_platforms = [
-            'linux-nightly',
-            'linux64-nightly',
-            'macosx64-nightly',
-            'win32-nightly',
-            'win64-nightly'
-        ]
-        if build_platform in funsize_platforms and is_nightly:
-            route_template = "project.releng.funsize.level-{level}.{project}"
-            task['routes'] = [
-                route_template.format(project=config.params['project'],
-                                      level=config.params['level'])
-            ]
-
         yield task
 
 
 def _generate_worker_type(signing_cert_scope):
     worker_type = 'depsigning' if 'dep-signing' in signing_cert_scope else 'signing-linux-v1'
     return 'scriptworker-prov-v1/{}'.format(worker_type)
--- a/taskcluster/taskgraph/transforms/task.py
+++ b/taskcluster/taskgraph/transforms/task.py
@@ -510,30 +510,33 @@ GROUP_NAMES = {
     'tc-W': 'Web platform tests executed by TaskCluster',
     'tc-W-e10s': 'Web platform tests executed by TaskCluster with e10s',
     'tc-X': 'Xpcshell tests executed by TaskCluster',
     'tc-X-e10s': 'Xpcshell tests executed by TaskCluster with e10s',
     'tc-L10n': 'Localised Repacks executed by Taskcluster',
     'tc-L10n-Rpk': 'Localized Repackaged Repacks executed by Taskcluster',
     'tc-BM-L10n': 'Beetmover for locales executed by Taskcluster',
     'tc-BMR-L10n': 'Beetmover repackages for locales executed by Taskcluster',
-    'tc-Up': 'Balrog submission of updates, executed by Taskcluster',
+    'c-Up': 'Balrog submission of complete updates',
     'tc-cs': 'Checksum signing executed by Taskcluster',
     'tc-rs': 'Repackage signing executed by Taskcluster',
     'tc-BMcs': 'Beetmover checksums, executed by Taskcluster',
     'Aries': 'Aries Device Image',
     'Nexus 5-L': 'Nexus 5-L Device Image',
     'I': 'Docker Image Builds',
     'TL': 'Toolchain builds for Linux 64-bits',
     'TM': 'Toolchain builds for OSX',
     'TW32': 'Toolchain builds for Windows 32-bits',
     'TW64': 'Toolchain builds for Windows 64-bits',
     'SM-tc': 'Spidermonkey builds',
     'pub': 'APK publishing',
+    'p': 'Partial generation',
+    'ps': 'Partials signing',
 }
+
 UNKNOWN_GROUP_NAME = "Treeherder group {} has no name; add it to " + __file__
 
 V2_ROUTE_TEMPLATES = [
     "index.gecko.v2.{project}.latest.{product}.{job-name}",
     "index.gecko.v2.{project}.pushdate.{build_date_long}.{product}.{job-name}",
     "index.gecko.v2.{project}.pushlog-id.{pushlog_id}.{product}.{job-name}",
     "index.gecko.v2.{project}.revision.{head_rev}.{product}.{job-name}",
 ]
new file mode 100644
--- /dev/null
+++ b/taskcluster/taskgraph/util/partials.py
@@ -0,0 +1,193 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, print_function, unicode_literals
+
+import requests
+import redo
+
+import logging
+logger = logging.getLogger(__name__)
+
+BALROG_API_ROOT = 'https://aus5.mozilla.org/api/v1'
+
+PLATFORM_RENAMES = {
+    'windows2012-32': 'win32',
+    'windows2012-64': 'win64',
+    'osx-cross': 'macosx64',
+}
+
+BALROG_PLATFORM_MAP = {
+    "linux": [
+        "Linux_x86-gcc3"
+    ],
+    "linux64": [
+        "Linux_x86_64-gcc3"
+    ],
+    "macosx64": [
+        "Darwin_x86_64-gcc3-u-i386-x86_64",
+        "Darwin_x86-gcc3-u-i386-x86_64",
+        "Darwin_x86-gcc3",
+        "Darwin_x86_64-gcc3"
+    ],
+    "win32": [
+        "WINNT_x86-msvc",
+        "WINNT_x86-msvc-x86",
+        "WINNT_x86-msvc-x64"
+    ],
+    "win64": [
+        "WINNT_x86_64-msvc",
+        "WINNT_x86_64-msvc-x64"
+    ]
+}
+
+
+def get_balrog_platform_name(platform):
+    """Convert build platform names into balrog platform names"""
+    if '-nightly' in platform:
+        platform = platform.replace('-nightly', '')
+    if '-devedition' in platform:
+        platform = platform.replace('-devedition', '')
+    return PLATFORM_RENAMES.get(platform, platform)
+
+
+def _sanitize_platform(platform):
+    platform = get_balrog_platform_name(platform)
+    if platform not in BALROG_PLATFORM_MAP:
+        return platform
+    return BALROG_PLATFORM_MAP[platform][0]
+
+
+def get_builds(release_history, platform, locale):
+    """Examine cached balrog release history and return the list of
+    builds we need to generate diffs from"""
+    platform = _sanitize_platform(platform)
+    return release_history.get(platform, {}).get(locale, {})
+
+
+def get_partials_artifacts(release_history, platform, locale):
+    platform = _sanitize_platform(platform)
+    return release_history.get(platform, {}).get(locale, {}).keys()
+
+
+def get_partials_artifact_map(release_history, platform, locale):
+    platform = _sanitize_platform(platform)
+    return {k: release_history[platform][locale][k]['buildid']
+            for k in release_history.get(platform, {}).get(locale, {})}
+
+
+def _retry_on_http_errors(url, verify, params, errors):
+    if params:
+        params_str = "&".join("=".join([k, str(v)])
+                              for k, v in params.iteritems())
+    else:
+        params_str = ''
+    logger.info("Connecting to %s?%s", url, params_str)
+    for _ in redo.retrier(sleeptime=5, max_sleeptime=30, attempts=10):
+        try:
+            req = requests.get(url, verify=verify, params=params, timeout=4)
+            req.raise_for_status()
+            return req
+        except requests.HTTPError as e:
+            if e.response.status_code in errors:
+                logger.exception("Got HTTP %s trying to reach %s",
+                                 e.response.status_code, url)
+            else:
+                raise
+    else:
+        raise
+
+
+def get_sorted_releases(product, branch):
+    """Returns a list of release names from Balrog.
+    :param product: product name, AKA appName
+    :param branch: branch name, e.g. mozilla-central
+    :return: a sorted list of release names, most recent first.
+    """
+    url = "{}/releases".format(BALROG_API_ROOT)
+    params = {
+        "product": product,
+        # Adding -nightly-2 (2 stands for the beginning of build ID
+        # based on date) should filter out release and latest blobs.
+        # This should be changed to -nightly-3 in 3000 ;)
+        "name_prefix": "{}-{}-nightly-2".format(product, branch),
+        "names_only": True
+    }
+    req = _retry_on_http_errors(
+        url=url, verify=True, params=params,
+        errors=[500])
+    releases = req.json()["names"]
+    releases = sorted(releases, reverse=True)
+    return releases
+
+
+def get_release_builds(release):
+    url = "{}/releases/{}".format(BALROG_API_ROOT, release)
+    req = _retry_on_http_errors(
+        url=url, verify=True, params=None,
+        errors=[500])
+    return req.json()
+
+
+def populate_release_history(product, branch, maxbuilds=4, maxsearch=10):
+    """Find relevant releases in Balrog
+    Not all releases have all platforms and locales, due
+    to Taskcluster migration.
+
+        Args:
+            product (str): capitalized product name, AKA appName, e.g. Firefox
+            branch (str): branch name (mozilla-central)
+            maxbuilds (int): Maximum number of historical releases to populate
+            maxsearch(int): Traverse at most this many releases, to avoid
+                working through the entire history.
+        Returns:
+            json object based on data from balrog api
+
+            results = {
+                'platform1': {
+                    'locale1': {
+                        'buildid1': mar_url,
+                        'buildid2': mar_url,
+                        'buildid3': mar_url,
+                    },
+                    'locale2': {
+                        'target.partial-1.mar': {'buildid1': 'mar_url'},
+                    }
+                },
+                'platform2': {
+                }
+            }
+        """
+    last_releases = get_sorted_releases(product, branch)
+
+    partial_mar_tmpl = 'target.partial-{}.mar'
+
+    builds = dict()
+    for release in last_releases[:maxsearch]:
+        # maxbuilds in all categories, don't make any more queries
+        full = len(builds) > 0 and all(
+            len(builds[platform][locale]) >= maxbuilds
+            for platform in builds for locale in builds[platform])
+        if full:
+            break
+        history = get_release_builds(release)
+
+        for platform in history['platforms']:
+            if 'alias' in history['platforms'][platform]:
+                continue
+            if platform not in builds:
+                builds[platform] = dict()
+            for locale in history['platforms'][platform]['locales']:
+                if locale not in builds[platform]:
+                    builds[platform][locale] = dict()
+                if len(builds[platform][locale]) >= maxbuilds:
+                    continue
+                buildid = history['platforms'][platform]['locales'][locale]['buildID']
+                url = history['platforms'][platform]['locales'][locale]['completes'][0]['fileUrl']
+                nextkey = len(builds[platform][locale]) + 1
+                builds[platform][locale][partial_mar_tmpl.format(nextkey)] = {
+                    'buildid': buildid,
+                    'mar_url': url,
+                }
+    return builds
--- a/taskcluster/taskgraph/util/taskcluster.py
+++ b/taskcluster/taskgraph/util/taskcluster.py
@@ -8,16 +8,19 @@ from __future__ import absolute_import, 
 
 import functools
 import yaml
 import requests
 from mozbuild.util import memoize
 from requests.packages.urllib3.util.retry import Retry
 from requests.adapters import HTTPAdapter
 
+_TC_ARTIFACT_LOCATION = \
+        'https://queue.taskcluster.net/v1/task/{task_id}/artifacts/public/build/{postfix}'
+
 
 @memoize
 def get_session():
     session = requests.Session()
     retry = Retry(total=5, backoff_factor=0.1,
                   status_forcelist=[500, 502, 503, 504])
     session.mount('http://', HTTPAdapter(max_retries=retry))
     session.mount('https://', HTTPAdapter(max_retries=retry))
@@ -96,8 +99,15 @@ def get_task_url(task_id, use_proxy=Fals
     else:
         TASK_URL = 'https://queue.taskcluster.net/v1/task/{}'
     return TASK_URL.format(task_id)
 
 
 def get_task_definition(task_id, use_proxy=False):
     response = _do_request(get_task_url(task_id, use_proxy))
     return response.json()
+
+
+def get_taskcluster_artifact_prefix(task_id, postfix='', locale=None):
+    if locale:
+        postfix = '{}/{}'.format(locale, postfix)
+
+    return _TC_ARTIFACT_LOCATION.format(task_id=task_id, postfix=postfix)