Bug 1264006 - release promotion - create part 2 of RC firefox task graph in release tasks, r=rail
authorJordan Lund <jlund@mozilla.com>
Tue, 12 Apr 2016 16:19:31 -0700
changeset 6718 97ce4f0a2c802f3846587354a2b6ac382217ecb7
parent 6717 c9553e6cc74442238606a39c5f174fcbc44143a2
child 6719 dfbf99b435239f9ec6948e3c859f34d66ed9223b
push id5018
push userjlund@mozilla.com
push dateMon, 18 Apr 2016 14:59:40 +0000
reviewersrail
bugs1264006
Bug 1264006 - release promotion - create part 2 of RC firefox task graph in release tasks, r=rail MozReview-Commit-ID: Chb4sMO8Kz1 new command example: python releasetasks_graph_gen.py --release-runner-ini release-runner.ini --branch-and-product-config $BUD/releasetasks/releasetasks/release_configs/dev_mozilla-release_firefox_rc_graph_2.yml --common-task-id T2IXQW55R8Gio8Y3QwHZYg
buildfarm/release/release-runner.py
buildfarm/release/releasetasks_graph_gen.py
lib/python/kickoff/__init__.py
--- a/buildfarm/release/release-runner.py
+++ b/buildfarm/release/release-runner.py
@@ -13,27 +13,28 @@ import shutil
 import tempfile
 import requests
 from os import path
 from optparse import OptionParser
 from twisted.python.lockfile import FilesystemLock
 
 site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
 
-from kickoff.api import Releases, Release, ReleaseL10n
+from kickoff import get_partials, ReleaseRunner, make_task_graph_strict_kwargs
+from kickoff import get_l10n_config, get_en_US_config
+from kickoff import email_release_drivers
+from kickoff import bump_version
 from release.info import readBranchConfig
 from release.l10n import parsePlainL10nChangesets
 from release.versions import getAppVersion
-from releasetasks import make_task_graph
 from taskcluster import Scheduler, Index, Queue
 from taskcluster.utils import slugId
 from util.hg import mercurial
 from util.retry import retry
 from util.file import load_config, get_config
-from util.sendmail import sendmail
 
 log = logging.getLogger(__name__)
 
 
 # both CHECKSUMS and ALL_FILES have been defined to improve the release sanity
 # en-US binaries timing by whitelisting artifacts of interest - bug 1251761
 CHECKSUMS = set([
     '.checksums',
@@ -47,52 +48,20 @@ ALL_FILES = set([
     '.complete.mar',
     '.exe',
     '.dmg',
     'i686.tar.bz2',
     'x86_64.tar.bz2',
 ])
 
 
-# temporary regex to filter out anything but mozilla-beta and mozilla-release
-# within release promotion. Once migration to release promotion is completed
-# for all types of releases, we will backout this filtering
-# regex beta tracking bug is 1252333,
-# regex release tracking bug is 1263976
-RELEASE_PATTERNS = [
-    r"Firefox-\d+\.0b\d+-build\d+",
-    r"Firefox-\d+\.\d+(\.\d+)?-build\d+"
-]
-
-
 class SanityException(Exception):
     pass
 
 
-# FIXME: the following function should be removed and we should use
-# next_version provided by ship-it
-def bump_version(version):
-    """Bump last digit"""
-    split_by = "."
-    digit_index = 2
-    if "b" in version:
-        split_by = "b"
-        digit_index = 1
-    v = version.split(split_by)
-    if len(v) < digit_index + 1:
-        # 45.0 is 45.0.0 actually
-        v.append("0")
-    v[-1] = str(int(v[-1]) + 1)
-    return split_by.join(v)
-
-
-def matches(name, patterns):
-    return any([re.search(p, name) for p in patterns])
-
-
 def is_candidate_release(channels):
     """determine if this is a candidate release or not
 
     Because ship-it can not tell us if this is a candidate release (yet!), we assume it is when we
     have determined, based on version, that we are planning to ship to more than one update_channel
     e.g. for candidate releases we have:
      1) one channel to test the 'candidate' release with: to 'beta' channel users
      2) once verified, we ship to the main channel: to 'release' channel users
@@ -131,187 +100,16 @@ def get_display_version(repo_path, revis
     def _get():
         req = requests.get(url, timeout=60)
         req.raise_for_status()
         return req.content.strip()
 
     return retry(_get)
 
 
-def long_revision(repo, revision):
-    """Convert short revision to long using JSON API
-
-    >>> long_revision("releases/mozilla-beta", "59f372c35b24")
-    u'59f372c35b2416ac84d6572d64c49227481a8a6c'
-
-    >>> long_revision("releases/mozilla-beta", "59f372c35b2416ac84d6572d64c49227481a8a6c")
-    u'59f372c35b2416ac84d6572d64c49227481a8a6c'
-    """
-    url = "https://hg.mozilla.org/{}/json-rev/{}".format(repo, revision)
-
-    def _get():
-        req = requests.get(url, timeout=60)
-        req.raise_for_status()
-        return req.json()["node"]
-
-    return retry(_get)
-
-
-class ReleaseRunner(object):
-    def __init__(self, api_root=None, username=None, password=None,
-                 timeout=60):
-        self.new_releases = []
-        self.releases_api = Releases((username, password), api_root=api_root,
-                                     timeout=timeout)
-        self.release_api = Release((username, password), api_root=api_root,
-                                   timeout=timeout)
-        self.release_l10n_api = ReleaseL10n((username, password),
-                                            api_root=api_root, timeout=timeout)
-
-    def get_release_requests(self):
-        new_releases = self.releases_api.getReleases()
-        if new_releases['releases']:
-            new_releases = [self.release_api.getRelease(name) for name in
-                            new_releases['releases']]
-            our_releases = [r for r in new_releases if
-                            matches(r['name'], RELEASE_PATTERNS)]
-            if our_releases:
-                # make sure to use long revision
-                for r in our_releases:
-                    r["mozillaRevision"] = long_revision(r["branch"], r["mozillaRevision"])
-                self.new_releases = our_releases
-                log.info("Releases to handle are %s", self.new_releases)
-                return True
-            else:
-                log.info("No releases to handle in %s", new_releases)
-                return False
-        else:
-            log.info("No new releases: %s" % new_releases)
-            return False
-
-    def get_release_l10n(self, release):
-        return self.release_l10n_api.getL10n(release)
-
-    def update_status(self, release, status):
-        log.info('updating status for %s to %s' % (release['name'], status))
-        try:
-            self.release_api.update(release['name'], status=status)
-        except requests.HTTPError, e:
-            log.warning('Caught HTTPError: %s' % e.response.content)
-            log.warning('status update failed, continuing...', exc_info=True)
-
-    def mark_as_completed(self, release):#, enUSPlatforms):
-        log.info('mark as completed %s' % release['name'])
-        self.release_api.update(release['name'], complete=True,
-                                status='Started')
-
-    def mark_as_failed(self, release, why):
-        log.info('mark as failed %s' % release['name'])
-        self.release_api.update(release['name'], ready=False, status=why)
-
-
-def getPartials(rr, release):
-    partials = {}
-    for p in release['partials'].split(','):
-        partialVersion, buildNumber = p.split('build')
-        partial_release_name = '{}-{}-build{}'.format(
-            release['product'].capitalize(), partialVersion, buildNumber,
-        )
-        partials[partialVersion] = {
-            'appVersion': getAppVersion(partialVersion),
-            'buildNumber': buildNumber,
-            'locales': parsePlainL10nChangesets(
-                rr.get_release_l10n(partial_release_name)).keys(),
-        }
-    return partials
-
-
-def email_release_drivers(smtp_server, from_, to, release, graph_id):
-    # Send an email to the mailing after the build
-
-    content = """\
-A new build has been submitted through ship-it:
-
-Commit: https://hg.mozilla.org/{path}/rev/{revision}
-Task graph: https://tools.taskcluster.net/task-graph-inspector/#{task_graph_id}/
-
-Created by {submitter}
-Started by {starter}
-
-
-""".format(path=release["branch"], revision=release["mozillaRevision"],
-           submitter=release["submitter"], starter=release["starter"],
-           task_graph_id=graph_id)
-
-    comment = release.get("comment")
-    if comment:
-        content += "Comment:\n" + comment + "\n\n"
-
-    # On r-d, we prefix the subject of the email in order to simplify filtering
-    if "Fennec" in release["name"]:
-        subject_prefix = "[mobile] "
-    if "Firefox" in release["name"]:
-        subject_prefix = "[desktop] "
-
-    subject = subject_prefix + 'Build of %s' % release["name"]
-
-    sendmail(from_=from_, to=to, subject=subject, body=content,
-             smtp_server=smtp_server)
-
-
-def get_platform_locales(l10n_changesets, platform):
-    # hardcode ja/ja-JP-mac exceptions
-    if platform == "macosx64":
-        ignore = "ja"
-    else:
-        ignore = "ja-JP-mac"
-
-    return [l for l in l10n_changesets.keys() if l != ignore]
-
-
-def get_l10n_config(release, branchConfig, branch, l10n_changesets, index):
-    l10n_platforms = {}
-    for platform in branchConfig["l10n_release_platforms"]:
-        task = index.findTask("buildbot.revisions.{revision}.{branch}.{platform}".format(
-            revision=release["mozillaRevision"],
-            branch=branch,
-            platform=platform,
-        ))
-        url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
-            taskid=task["taskId"]
-        )
-        l10n_platforms[platform] = {
-            "locales": get_platform_locales(l10n_changesets, platform),
-            "en_us_binary_url": url,
-            "chunks": branchConfig["platforms"][platform].get("l10n_chunks", 10),
-        }
-
-    return {
-        "platforms": l10n_platforms,
-        "changesets": l10n_changesets,
-    }
-
-
-def get_en_US_config(release, branchConfig, branch, index):
-    platforms = {}
-    for platform in branchConfig["release_platforms"]:
-        task = index.findTask("buildbot.revisions.{revision}.{branch}.{platform}".format(
-            revision=release["mozillaRevision"],
-            branch=branch,
-            platform=platform,
-        ))
-        platforms[platform] = {
-            "task_id": task["taskId"],
-        }
-
-    return {
-        "platforms": platforms,
-    }
-
-
 def validate_version(repo_path, revision, version):
     actual_version = get_display_version(repo_path, revision)
     if version != actual_version:
         raise SanityException(
             "In-tree version '%s' doesn't match ship-it version '%s'" %
             (actual_version, version))
     else:
         log.info("In-tree version '%s' matches ship-it version '%s'",
@@ -589,50 +387,57 @@ def main(options):
                 "buildNumber": release["buildNumber"],
                 "source_enabled": True,
                 "checksums_enabled": True,
                 "repo_path": release["branch"],
                 "revision": release["mozillaRevision"],
                 "product": release["product"],
                 # if mozharness_revision is not passed, use 'revision'
                 "mozharness_changeset": release.get('mh_changeset') or release['mozillaRevision'],
-                "partial_updates": getPartials(rr, release),
+                "partial_updates": get_partials(rr, release['partials'], release['product']),
                 "branch": branch,
                 "updates_enabled": bool(release["partials"]),
-                "l10n_config": get_l10n_config(release, branchConfig, branch, l10n_changesets, index),
-                "en_US_config": get_en_US_config(release, branchConfig, branch, index),
+                "l10n_config": get_l10n_config(
+                    release['mozillaRevision'], branchConfig['platforms'],
+                    branchConfig['l10n_release_platforms'], branch, l10n_changesets, index
+                ),
+                "en_US_config": get_en_US_config(
+                    release['mozillaRevision'], branchConfig['release_platforms'], branch, index
+                ),
                 "verifyConfigs": {},
                 "balrog_api_root": branchConfig["balrog_api_root"],
                 "funsize_balrog_api_root": branchConfig["funsize_balrog_api_root"],
                 "balrog_username": balrog_username,
                 "balrog_password": balrog_password,
                 "beetmover_aws_access_key_id": beetmover_aws_access_key_id,
                 "beetmover_aws_secret_access_key": beetmover_aws_secret_access_key,
                 # TODO: stagin specific, make them configurable
                 "signing_class": "release-signing",
                 "bouncer_enabled": branchConfig["bouncer_enabled"],
+                "updates_builder_enabled": branchConfig["updates_builder_enabled"],
+                "update_verify_enabled": branchConfig["update_verify_enabled"],
                 "release_channels": release_channels,
                 "final_verify_channels": final_verify_channels,
+                "final_verify_platforms": branchConfig['release_platforms'],
                 "signing_pvt_key": signing_pvt_key,
                 "build_tools_repo_path": branchConfig['build_tools_repo_path'],
                 "push_to_candidates_enabled": branchConfig['push_to_candidates_enabled'],
                 "postrelease_bouncer_aliases_enabled": postrelease_bouncer_aliases_enabled,
                 "tuxedo_server_url": branchConfig['tuxedoServerUrl'],
                 "postrelease_version_bump_enabled": postrelease_enabled,
                 "push_to_releases_enabled": push_to_releases_enabled,
                 "push_to_releases_automatic": branchConfig['push_to_releases_automatic'],
                 "beetmover_candidates_bucket": branchConfig["beetmover_buckets"][release["product"]],
                 "partner_repacks_platforms": branchConfig.get("partner_repacks_platforms", []),
                 "l10n_changesets": l10n_changesets,
+                "extra_balrog_submitter_params": extra_balrog_submitter_params,
             }
-            if extra_balrog_submitter_params:
-                kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params
 
             validate_graph_kwargs(queue, gpg_key_path, **kwargs)
-            graph = make_task_graph(**kwargs)
+            graph = make_task_graph_strict_kwargs(**kwargs)
             rr.update_status(release, "Submitting task graph")
             log.info("Task graph generated!")
             import pprint
             log.debug(pprint.pformat(graph, indent=4, width=160))
             print scheduler.createTaskGraph(graph_id, graph)
 
             rr.mark_as_completed(release)
             email_release_drivers(smtp_server=smtp_server, from_=notify_from,
new file mode 100644
--- /dev/null
+++ b/buildfarm/release/releasetasks_graph_gen.py
@@ -0,0 +1,206 @@
+#!/usr/bin/env python
+
+import logging
+import os
+from optparse import OptionParser
+import site
+import yaml
+
+site.addsitedir(os.path.join(os.path.dirname(__file__), "../../lib/python"))
+
+from kickoff import get_partials, ReleaseRunner, make_task_graph_strict_kwargs
+from kickoff import get_l10n_config, get_en_US_config
+from kickoff import bump_version
+
+from release.versions import getAppVersion
+from util.file import load_config, get_config
+
+from taskcluster import Scheduler, Index, Queue
+from taskcluster.utils import slugId
+
+log = logging.getLogger(__name__)
+
+
+def main(release_runner_config, release_config, tc_config):
+
+    api_root = release_runner_config.get('api', 'api_root')
+    username = release_runner_config.get('api', 'username')
+    password = release_runner_config.get('api', 'password')
+
+    scheduler = Scheduler(tc_config)
+    index = Index(tc_config)
+
+    rr = ReleaseRunner(api_root=api_root, username=username, password=password)
+    graph_id = slugId()
+    log.info('Generating task graph')
+    kwargs = {
+        # release-runner.ini
+        "signing_pvt_key": release_config['signing_pvt_key'],
+        "public_key": release_config['docker_worker_key'],
+        "balrog_username": release_config['balrog_username'],
+        "balrog_password": release_config['balrog_password'],
+        "beetmover_aws_access_key_id": release_config['beetmover_aws_access_key_id'],
+        "beetmover_aws_secret_access_key": release_config['beetmover_aws_secret_access_key'],
+        "signing_class": "release-signing",  # TODO: stagin specific, make them configurable
+
+        # ship-it items
+        "version": release_config["version"],
+        "revision": release_config["mozilla_revision"],
+        "mozharness_changeset": release_config["mozharness_revision"] or release_config["mozilla_revision"],
+        "buildNumber": release_config["build_number"],
+        "l10n_changesets": release_config["l10n_changesets"],
+
+        # was branchConfig items
+        "funsize_balrog_api_root": release_config["funsize_balrog_api_root"],
+        "balrog_api_root": release_config["balrog_api_root"],
+        "build_tools_repo_path": release_config['build_tools_repo_path'],
+        "tuxedo_server_url": release_config['tuxedo_server_url'],
+        "beetmover_candidates_bucket": release_config["beetmover_candidates_bucket"],
+        "bouncer_enabled": release_config["bouncer_enabled"],
+        "updates_builder_enabled": release_config["updates_builder_enabled"],
+        "update_verify_enabled": release_config["update_verify_enabled"],
+        "push_to_candidates_enabled": release_config['push_to_candidates_enabled'],
+        "postrelease_bouncer_aliases_enabled": release_config['postrelease_bouncer_aliases_enabled'],
+        "postrelease_version_bump_enabled": release_config['postrelease_version_bump_enabled'],
+        "push_to_releases_automatic": release_config['push_to_releases_automatic'],
+        "partner_repacks_platforms": release_config["partner_repacks_platforms"],
+
+        "repo_path": release_config["repo_path"],
+        "branch": release_config["branch"],
+        "product": release_config["product"],
+        "release_channels": release_config['channels'],
+        "final_verify_channels": release_config['final_verify_channels'],
+        "final_verify_platforms": release_config['final_verify_platforms'],
+        "source_enabled": release_config["source_enabled"],
+        "checksums_enabled": release_config["checksums_enabled"],
+        "updates_enabled": release_config["updates_enabled"],
+        "push_to_releases_enabled": release_config["push_to_releases_enabled"],
+
+        "verifyConfigs": {},
+        "next_version": bump_version(release_config["version"]),
+        "appVersion": getAppVersion(release_config["version"]),
+        "partial_updates": get_partials(rr, release_config["partials"],
+                                        release_config['product']),
+        # in release-runner.py world we have a concept of branchConfig and release (shipit) vars
+        # todo fix get_en_US_config and en_US_config helper methods to not require both
+        "l10n_config": get_l10n_config(
+            release_config["mozilla_revision"], release_config['platforms'],
+            release_config['l10n_release_platforms'] or {}, release_config["branch"],
+            release_config["l10n_changesets"], index
+        ),
+        "en_US_config": get_en_US_config(
+            release_config["mozilla_revision"], release_config['release_platforms'] or [],
+            release_config["branch"], index
+        ),
+        "extra_balrog_submitter_params": release_config['extra_balrog_submitter_params']
+    }
+
+    graph = make_task_graph_strict_kwargs(**kwargs)
+    log.info("Submitting task graph")
+    import pprint
+    log.info(pprint.pformat(graph, indent=4, width=160))
+    if not options.dry_run:
+        print scheduler.createTaskGraph(graph_id, graph)
+
+
+def get_items_from_common_tc_task(common_task_id, tc_config):
+    tc_task_items = {}
+    queue = Queue(tc_config)
+    task = queue.task(common_task_id)
+    tc_task_items["version"] = task["extra"]["build_props"]["version"]
+    tc_task_items["build_number"] = task["extra"]["build_props"]["build_number"]
+    tc_task_items["mozilla_revision"] = task["extra"]["build_props"]["revision"]
+    return tc_task_items
+
+
+def get_unique_release_items(options, tc_config):
+    unique_items = {}
+
+    if options.common_task_id:
+        # sometimes, we make a release based on a previous release. e.g. a graph that represents
+        # part 2 of a Firefox Release Candidate release
+        # TODO extract partials, mozharness_revision, and l10n_changesets from common taskgroup
+        unique_items.update(get_items_from_common_tc_task(options.common_task_id, tc_config))
+    else:
+        unique_items['version'] = options.version
+        unique_items['build_number'] = options.build_number
+        unique_items['mozilla_revision'] = options.mozilla_revision
+
+    unique_items['partials'] = options.partials
+    unique_items['mozharness_revision'] = options.mozharness_revision
+    # TODO have ability to pass l10n_changesets whether based on previous release or new one
+    unique_items["l10n_changesets"] = {}
+
+    return unique_items
+
+
+def get_release_items_from_runner_config(release_runner_ini):
+    ini_items = {}
+    ini_items['signing_pvt_key'] = release_runner_ini.get('signing', 'pvt_key')
+    ini_items['docker_worker_key'] = release_runner_ini.get('release-runner', 'docker_worker_key')
+    ini_items['balrog_username'] = release_runner_ini.get("balrog", "username")
+    ini_items['balrog_password'] = release_runner_ini.get("balrog", "password")
+    ini_items['beetmover_aws_access_key_id'] = release_runner_ini.get("beetmover", "aws_access_key_id")
+    ini_items['beetmover_aws_secret_access_key'] = release_runner_ini.get("beetmover", "aws_secret_access_key")
+    ini_items['extra_balrog_submitter_params'] = get_config(release_runner_ini, "balrog",
+                                                            "extra_balrog_submitter_params", None)
+    return ini_items
+
+
+def load_branch_and_product_config(config_file):
+    with open(config_file, 'r') as rc_file:
+        return yaml.load(rc_file)
+
+
+if __name__ == '__main__':
+    parser = OptionParser(__doc__)
+    parser.add_option('--release-runner-ini', dest='release_runner_ini',
+                      help='ini file that contains things like sensitive credentials')
+    parser.add_option('--branch-and-product-config', dest='branch_and_product_config',
+                      help='config items specific to certain product and branch')
+    parser.add_option('--version', dest='version', help='full version of release, e.g. 46.0b1')
+    parser.add_option('--build-number', dest='build_number', help='build number of release')
+    parser.add_option('--partials', type="string", dest='partials',
+                      help='list of partials for the release')
+    parser.add_option('--mozilla-revision', dest='mozilla_revision',
+                      help='gecko revision to build ff from')
+    parser.add_option('--mozharness-revision', dest='mozharness_revision',
+                      help='gecko revision for mozharness')
+    parser.add_option('--common-task-id', dest='common_task_id',
+                      help='a task id of a task that shares the same release info')
+    parser.add_option('--dry-run', dest='dry_run', action='store_true', default=False,
+                      help="render the task graph from yaml tmpl but don't submit to taskcluster")
+
+    options = parser.parse_args()[0]
+
+    if not options.release_runner_ini:
+        parser.error('Need to pass a release runner config')
+    if not options.branch_and_product_config:
+        parser.error('Need to pass a branch and product config')
+
+    # load config files
+    release_runner_config = load_config(options.release_runner_ini)
+    tc_config = {
+        "credentials": {
+            "clientId": get_config(release_runner_config, "taskcluster", "client_id", None),
+            "accessToken": get_config(release_runner_config, "taskcluster", "access_token", None),
+        }
+    }
+    branch_product_config = load_branch_and_product_config(options.branch_and_product_config)
+
+    if release_runner_config.getboolean('release-runner', 'verbose'):
+        log_level = logging.DEBUG
+    else:
+        log_level = logging.INFO
+    logging.basicConfig(filename='releasetasks_graph_gen.log',
+                        format="%(asctime)s - %(levelname)s - %(message)s",
+                        level=log_level)
+
+
+    # create releasetasks graph args from config files
+    releasetasks_kwargs = {}
+    releasetasks_kwargs.update(branch_product_config)
+    releasetasks_kwargs.update(get_release_items_from_runner_config(release_runner_config))
+    releasetasks_kwargs.update(get_unique_release_items(options, tc_config))
+
+    main(release_runner_config, releasetasks_kwargs, tc_config)
--- a/lib/python/kickoff/__init__.py
+++ b/lib/python/kickoff/__init__.py
@@ -0,0 +1,280 @@
+import re
+import requests
+import logging
+
+from kickoff.api import Releases, Release, ReleaseL10n
+from releasetasks import make_task_graph
+from release.l10n import parsePlainL10nChangesets
+from release.versions import getAppVersion
+from util.sendmail import sendmail
+from util.retry import retry
+
+log = logging.getLogger(__name__)
+
+# temporary regex to filter out anything but mozilla-beta and mozilla-release
+# within release promotion. Once migration to release promotion is completed
+# for all types of releases, we will backout this filtering
+# regex beta tracking bug is 1252333,
+# regex release tracking bug is 1263976
+RELEASE_PATTERNS = [
+    r"Firefox-\d+\.0b\d+-build\d+",
+    r"Firefox-\d+\.\d+(\.\d+)?-build\d+"
+]
+
+
+def matches(name, patterns):
+    return any([re.search(p, name) for p in patterns])
+
+
+def long_revision(repo, revision):
+    """Convert short revision to long using JSON API
+
+    >>> long_revision("releases/mozilla-beta", "59f372c35b24")
+    u'59f372c35b2416ac84d6572d64c49227481a8a6c'
+
+    >>> long_revision("releases/mozilla-beta", "59f372c35b2416ac84d6572d64c49227481a8a6c")
+    u'59f372c35b2416ac84d6572d64c49227481a8a6c'
+    """
+    url = "https://hg.mozilla.org/{}/json-rev/{}".format(repo, revision)
+
+    def _get():
+        req = requests.get(url, timeout=60)
+        req.raise_for_status()
+        return req.json()["node"]
+
+    return retry(_get)
+
+
+class ReleaseRunner(object):
+    def __init__(self, api_root=None, username=None, password=None,
+                 timeout=60):
+        self.new_releases = []
+        self.releases_api = Releases((username, password), api_root=api_root,
+                                     timeout=timeout)
+        self.release_api = Release((username, password), api_root=api_root,
+                                   timeout=timeout)
+        self.release_l10n_api = ReleaseL10n((username, password),
+                                            api_root=api_root, timeout=timeout)
+
+    def get_release_requests(self):
+        new_releases = self.releases_api.getReleases()
+        if new_releases['releases']:
+            new_releases = [self.release_api.getRelease(name) for name in
+                            new_releases['releases']]
+            our_releases = [r for r in new_releases if
+                            matches(r['name'], RELEASE_PATTERNS)]
+            if our_releases:
+                # make sure to use long revision
+                for r in our_releases:
+                    r["mozillaRevision"] = long_revision(r["branch"], r["mozillaRevision"])
+                self.new_releases = our_releases
+                log.info("Releases to handle are %s", self.new_releases)
+                return True
+            else:
+                log.info("No releases to handle in %s", new_releases)
+                return False
+        else:
+            log.info("No new releases: %s" % new_releases)
+            return False
+
+    def get_release_l10n(self, release):
+        return self.release_l10n_api.getL10n(release)
+
+    def update_status(self, release, status):
+        log.info('updating status for %s to %s' % (release['name'], status))
+        try:
+            self.release_api.update(release['name'], status=status)
+        except requests.HTTPError, e:
+            log.warning('Caught HTTPError: %s' % e.response.content)
+            log.warning('status update failed, continuing...', exc_info=True)
+
+    def mark_as_completed(self, release):#, enUSPlatforms):
+        log.info('mark as completed %s' % release['name'])
+        self.release_api.update(release['name'], complete=True,
+                                status='Started')
+
+    def mark_as_failed(self, release, why):
+        log.info('mark as failed %s' % release['name'])
+        self.release_api.update(release['name'], ready=False, status=why)
+
+def email_release_drivers(smtp_server, from_, to, release, graph_id):
+    # Send an email to the mailing after the build
+
+    content = """\
+A new build has been submitted through ship-it:
+
+Commit: https://hg.mozilla.org/{path}/rev/{revision}
+Task graph: https://tools.taskcluster.net/task-graph-inspector/#{task_graph_id}/
+
+Created by {submitter}
+Started by {starter}
+
+
+""".format(path=release["branch"], revision=release["mozillaRevision"],
+           submitter=release["submitter"], starter=release["starter"],
+           task_graph_id=graph_id)
+
+    comment = release.get("comment")
+    if comment:
+        content += "Comment:\n" + comment + "\n\n"
+
+    # On r-d, we prefix the subject of the email in order to simplify filtering
+    if "Fennec" in release["name"]:
+        subject_prefix = "[mobile] "
+    if "Firefox" in release["name"]:
+        subject_prefix = "[desktop] "
+
+    subject = subject_prefix + 'Build of %s' % release["name"]
+
+    sendmail(from_=from_, to=to, subject=subject, body=content,
+             smtp_server=smtp_server)
+
+
+def get_partials(rr, partial_versions, product):
+    partials = {}
+    if not partial_versions:
+        return partials
+    for p in partial_versions.split(','):
+        partialVersion, buildNumber = p.split('build')
+        partial_release_name = '{}-{}-build{}'.format(
+            product.capitalize(), partialVersion, buildNumber,
+        )
+        partials[partialVersion] = {
+            'appVersion': getAppVersion(partialVersion),
+            'buildNumber': buildNumber,
+            'locales': parsePlainL10nChangesets(
+                rr.get_release_l10n(partial_release_name)).keys(),
+        }
+    return partials
+
+
+def get_platform_locales(l10n_changesets, platform):
+    # hardcode ja/ja-JP-mac exceptions
+    if platform == "macosx64":
+        ignore = "ja"
+    else:
+        ignore = "ja-JP-mac"
+
+    return [l for l in l10n_changesets.keys() if l != ignore]
+
+
+def get_l10n_config(mozilla_revision, platforms, l10n_platforms, branch, l10n_changesets, index):
+    l10n_platform_configs = {}
+    for platform in l10n_platforms:
+        task = index.findTask("buildbot.revisions.{revision}.{branch}.{platform}".format(
+            revision=mozilla_revision,
+            branch=branch,
+            platform=platform,
+        ))
+        url = "https://queue.taskcluster.net/v1/task/{taskid}/artifacts/public/build".format(
+            taskid=task["taskId"]
+        )
+        l10n_platform_configs[platform] = {
+            "locales": get_platform_locales(l10n_changesets, platform),
+            "en_us_binary_url": url,
+            "chunks": platforms[platform].get("l10n_chunks", 10),
+        }
+
+    return {
+        "platforms": l10n_platform_configs,
+        "changesets": l10n_changesets,
+    }
+
+
+def get_en_US_config(mozilla_revision, platforms, branch, index):
+    platform_configs = {}
+    for platform in platforms:
+        task = index.findTask("buildbot.revisions.{revision}.{branch}.{platform}".format(
+            revision=mozilla_revision,
+            branch=branch,
+            platform=platform,
+        ))
+        platform_configs[platform] = {
+            "task_id": task["taskId"],
+        }
+
+    return {
+        "platforms": platform_configs,
+    }
+
+
+# FIXME: the following function should be removed and we should use
+# next_version provided by ship-it
+def bump_version(version):
+    """Bump last digit"""
+    split_by = "."
+    digit_index = 2
+    if "b" in version:
+        split_by = "b"
+        digit_index = 1
+    v = version.split(split_by)
+    if len(v) < digit_index + 1:
+        # 45.0 is 45.0.0 actually
+        v.append("0")
+    v[-1] = str(int(v[-1]) + 1)
+    return split_by.join(v)
+
+
+def make_task_graph_strict_kwargs(appVersion, balrog_api_root, balrog_password, balrog_username,
+                                  beetmover_aws_access_key_id, beetmover_aws_secret_access_key,
+                                  beetmover_candidates_bucket, bouncer_enabled, branch, buildNumber,
+                                  build_tools_repo_path, checksums_enabled, en_US_config,
+                                  extra_balrog_submitter_params, final_verify_channels,
+                                  final_verify_platforms, funsize_balrog_api_root, l10n_config,
+                                  l10n_changesets, mozharness_changeset, next_version,
+                                  partial_updates, partner_repacks_platforms,
+                                  postrelease_bouncer_aliases_enabled, postrelease_version_bump_enabled,
+                                  product, public_key, push_to_candidates_enabled,
+                                  push_to_releases_automatic, push_to_releases_enabled, release_channels,
+                                  repo_path, revision, signing_class, signing_pvt_key, source_enabled,
+                                  tuxedo_server_url, update_verify_enabled, updates_builder_enabled,
+                                  updates_enabled, verifyConfigs, version):
+    """simple wrapper that sanitizes whatever calls make_task_graph uses universally known kwargs"""
+
+    kwargs = dict(
+        appVersion=appVersion,
+        balrog_api_root=balrog_api_root,
+        balrog_password=balrog_password,
+        balrog_username=balrog_username,
+        beetmover_aws_access_key_id=beetmover_aws_access_key_id,
+        beetmover_aws_secret_access_key=beetmover_aws_secret_access_key,
+        beetmover_candidates_bucket=beetmover_candidates_bucket,
+        bouncer_enabled=bouncer_enabled,
+        branch=branch,
+        buildNumber=buildNumber,
+        build_tools_repo_path=build_tools_repo_path,
+        checksums_enabled=checksums_enabled,
+        en_US_config=en_US_config,
+        final_verify_channels=final_verify_channels,
+        final_verify_platforms=final_verify_platforms,
+        funsize_balrog_api_root=funsize_balrog_api_root,
+        l10n_changesets=l10n_changesets,
+        l10n_config=l10n_config,
+        mozharness_changeset=mozharness_changeset,
+        next_version=next_version,
+        partial_updates=partial_updates,
+        partner_repacks_platforms=partner_repacks_platforms,
+        postrelease_bouncer_aliases_enabled=postrelease_bouncer_aliases_enabled,
+        postrelease_version_bump_enabled=postrelease_version_bump_enabled,
+        product=product,
+        public_key=public_key,
+        push_to_candidates_enabled=push_to_candidates_enabled,
+        push_to_releases_automatic=push_to_releases_automatic,
+        push_to_releases_enabled=push_to_releases_enabled,
+        release_channels=release_channels,
+        repo_path=repo_path,
+        revision=revision,
+        signing_class=signing_class,
+        signing_pvt_key=signing_pvt_key,
+        source_enabled=source_enabled,
+        tuxedo_server_url=tuxedo_server_url,
+        update_verify_enabled=update_verify_enabled,
+        updates_builder_enabled=updates_builder_enabled,
+        updates_enabled=updates_enabled,
+        verifyConfigs=verifyConfigs,
+        version=version
+    )
+    if extra_balrog_submitter_params:
+        kwargs["extra_balrog_submitter_params"] = extra_balrog_submitter_params
+
+    return make_task_graph(**kwargs)