Bug 1347635 - Add releaserunner2 logic to handle Fennec integration. r=rail
authorMihai Tabara <mtabara@mozilla.com>
Fri, 15 Sep 2017 17:20:16 +0300
changeset 7852 e7efa4f81256
parent 7851 ad871db03cd8
child 7853 6798845e667a
push id5807
push usermtabara@mozilla.com
push dateFri, 15 Sep 2017 14:42:40 +0000
reviewersrail
bugs1347635
Bug 1347635 - Add releaserunner2 logic to handle Fennec integration. r=rail
buildfarm/release/release-runner2.py
buildfarm/release/releasetasks_graph_gen.py
--- a/buildfarm/release/release-runner2.py
+++ b/buildfarm/release/release-runner2.py
@@ -1,26 +1,29 @@
 #!/usr/bin/env python
 
 import site
 import time
 import logging
-import subprocess
 import sys
 import os
 import re
 from os import path
 from optparse import OptionParser
 from twisted.python.lockfile import FilesystemLock
 import yaml
 
 site.addsitedir(path.join(path.dirname(__file__), "../../lib/python"))
 
-from kickoff import ReleaseRunner, long_revision
+from kickoff import ReleaseRunner, long_revision, email_release_drivers
 from kickoff.sanity.revisions import RevisionsSanitizer
+from releasetasks_graph_gen import (get_release_items_from_runner_config,
+                                    get_unique_release_items,
+                                    load_branch_and_product_config)
+from releasetasks_graph_gen import main as gen_main
 
 
 log = logging.getLogger(__name__)
 
 
 def check_and_assign_long_revision(release_runner, release, releases_config):
     # Revisions must be checked before trying to get the long one.
     RevisionsSanitizer(**release).run()
@@ -91,20 +94,24 @@ def main(options):
     logging.getLogger("util.retry").setLevel(logging.WARN)
 
     api_root = config['api']['api_root']
     username = config['api']['username']
     password = config['api']['password']
 
     rr_config = config['release-runner']
     sleeptime = rr_config['sleeptime']
+    smtp_server = rr_config.get('smtp_server', 'localhost')
+    notify_from = rr_config.get('notify_from')
+    notify_to = rr_config.get('notify_to_announce')
+    if isinstance(notify_to, basestring):
+        notify_to = [x.strip() for x in notify_to.split(',')]
 
     rr = ReleaseRunner(api_root=api_root, username=username, password=password)
 
-    # Main loop waits for new releases, processes them and exits.
     while True:
         try:
             log.debug('Fetching release requests')
             rr.get_release_requests([r['pattern'] for r in config['releases']])
             if rr.new_releases:
                 new_releases = run_prebuild_sanity_checks(
                     rr, config['releases'])
                 break
@@ -118,39 +125,53 @@ def main(options):
 
     rc = 0
     for release in new_releases:
         try:
             relconfigs_tmpl = '{}_{}_fennec_full_graph.yml'.format(rr_config['relconfigs_prefix'],
                                                                              release['branchShortName'])
             configs = '/'.join([rr_config['relconfigs_root'], relconfigs_tmpl])
 
-            cmd = [
-                rr_config['python'],
-                rr_config['gen_script'],
-                '--release-runner-config',
-                rr_config['release_runner_config'],
-                '--branch-and-product-config={}'.format(configs),
-                '--version={}'.format(release['version']),
-                '--build-number={}'.format(release['buildNumber']),
-                '--mozilla-revision={}'.format(release["mozillaRevision"]),
-            ]
+            # process stuff for releasetasks_graph_gen.py script
+            release_runner_config = yaml.safe_load(open(rr_config['release_runner_config']))
+            tc_config = {
+                "credentials": {
+                    "clientId": release_runner_config["taskcluster"].get("client_id"),
+                    "accessToken": release_runner_config["taskcluster"].get("access_token"),
+                },
+                "maxRetries": 12,
+            }
+            branch_product_config = load_branch_and_product_config(configs)
+
+            # hack: we simulate the options argument for the
+            # releasetasks_graph_gen.py script in order to reuse some of the
+            # functions defined there
+            fake_options = OptionParser()
+            fake_options.version = release['version']
+            fake_options.build_number = release['buildNumber']
+            fake_options.mozilla_revision = release['mozillaRevision']
+            fake_options.common_task_id = None
+            fake_options.partials = ''
+            fake_options.dry_run = False
+
+            releasetasks_kwargs = {}
+            releasetasks_kwargs.update(branch_product_config)
+            releasetasks_kwargs.update(get_release_items_from_runner_config(release_runner_config))
+            releasetasks_kwargs.update(get_unique_release_items(fake_options, tc_config))
 
             rr.update_status(release, 'Generating task graph')
 
-            # TODO: might want to investigate if we can just call a function
-            # here instead of the entire script, so that we can send the email
-            # directly from this script, and not the gen.py one
-            subprocess.check_call(cmd)
+            task_group_id = gen_main(release_runner_config, releasetasks_kwargs, tc_config, options=fake_options)
 
             rr.mark_as_completed(release)
-            # TODO: normally sending the email should be done here but
-            # since we don't have the actual Taskgroup ID, we need to munge
-            # the other script to do that
-        except subprocess.CalledProcessError as exception:
+            l10n_url = rr.release_l10n_api.getL10nFullUrl(release['name'])
+            email_release_drivers(smtp_server=smtp_server, from_=notify_from,
+                                  to=notify_to, release=release,
+                                  task_group_id=task_group_id, l10n_url=l10n_url)
+        except Exception as exception:
             # We explicitly do not raise an error here because there's no
             # reason not to start other releases if creating the Task Graph
             # fails for another one. We _do_ need to set this in order to exit
             # with the right code, though.
             rc = 2
             rr.mark_as_failed(
                 release,
                 'Failed to start release promotion. Error(s): %s' % (exception)
--- a/buildfarm/release/releasetasks_graph_gen.py
+++ b/buildfarm/release/releasetasks_graph_gen.py
@@ -16,17 +16,17 @@ from kickoff.tc import resolve_task, sub
 from release.versions import getAppVersion
 
 from taskcluster import Index, Queue
 from taskcluster.utils import slugId
 
 log = logging.getLogger(__name__)
 
 
-def main(release_runner_config, release_config, tc_config):
+def main(release_runner_config, release_config, tc_config, options):
 
     api_root = release_runner_config['api']['api_root']
     username = release_runner_config['api']['username']
     password = release_runner_config['api']['password']
 
     queue = Queue(tc_config)
     index = Index(tc_config)
 
@@ -131,17 +131,18 @@ def main(release_runner_config, release_
     if not options.dry_run:
         submit_parallelized(queue, tasks)
         resolve_task(queue, toplevel_task_id)
         log_line = 'Task graph submitted: https://tools.taskcluster.net/groups/{}'.format(task_group_id)
         log.info(log_line)
         # TODO: We shouldn't need this extra print, but at the moment, calling the script in verbose
         # mode doesn't output anything.
         print log_line
-        # TODO: import logic to send an email with task id
+
+    return task_group_id
 
 
 def get_items_from_common_tc_task(common_task_id, tc_config):
     tc_task_items = {}
     queue = Queue(tc_config)
     task = queue.task(common_task_id)
     tc_task_items["version"] = task["extra"]["build_props"]["version"]
     tc_task_items["build_number"] = task["extra"]["build_props"]["build_number"]
@@ -233,9 +234,9 @@ if __name__ == '__main__':
 
 
     # create releasetasks graph args from config files
     releasetasks_kwargs = {}
     releasetasks_kwargs.update(branch_product_config)
     releasetasks_kwargs.update(get_release_items_from_runner_config(release_runner_config))
     releasetasks_kwargs.update(get_unique_release_items(options, tc_config))
 
-    main(release_runner_config, releasetasks_kwargs, tc_config)
+    main(release_runner_config, releasetasks_kwargs, tc_config, options=options)