bug 1083319 and 1083324: Trigger nightly builds and bundles after merge day uplift automatically. r=rail
authorBen Hearsum <bhearsum@mozilla.com>
Tue, 18 Nov 2014 10:00:49 -0500 (2014-11-18)
changeset 3396 c4060dfc1b65d903bd4574498b9c3d87c7d962d3
parent 3395 ca641817e1a4393da9e8b6e4cdaba6195b8ab657
child 3397 187d3e64f28f900c07243f08cacb4db29ee9cc46
push id2623
push userbhearsum@mozilla.com
push dateTue, 18 Nov 2014 15:01:35 +0000 (2014-11-18)
reviewersrail
bugs1083319, 1083324
bug 1083319 and 1083324: Trigger nightly builds and bundles after merge day uplift automatically. r=rail
configs/merge_day/aurora_to_beta.py
configs/merge_day/beta_to_release.py
configs/merge_day/central_to_aurora.py
configs/selfserve/production.py
configs/selfserve/staging.py
mozharness/mozilla/selfserve.py
scripts/merge_day/gecko_migration.py
--- a/configs/merge_day/aurora_to_beta.py
+++ b/configs/merge_day/aurora_to_beta.py
@@ -13,9 +13,20 @@ config = {
     "tools_repo_revision": "default",
     "from_repo_url": "ssh://hg.mozilla.org/releases/mozilla-aurora",
     "to_repo_url": "ssh://hg.mozilla.org/releases/mozilla-beta",
 
     "base_tag": "FIREFOX_BETA_%(major_version)s_BASE",
     "end_tag": "FIREFOX_BETA_%(major_version)s_END",
 
     "migration_behavior": "aurora_to_beta",
+
+    "virtualenv_modules": [
+        "requests==2.2.1",
+    ],
+
+    "post_merge_builders": [
+        "mozilla-beta hg bundle",
+    ],
+    "post_merge_nightly_branches": [
+        # No nightlies on mozilla-beta
+    ],
 }
--- a/configs/merge_day/beta_to_release.py
+++ b/configs/merge_day/beta_to_release.py
@@ -15,9 +15,20 @@ config = {
     "to_repo_url": "ssh://hg.mozilla.org/releases/mozilla-release",
 
     "base_tag": "FIREFOX_RELEASE_%(major_version)s_BASE",
     "end_tag": "FIREFOX_RELEASE_%(major_version)s_END",
 
     "migration_behavior": "beta_to_release",
     "require_remove_locales": False,
     "pull_all_branches": True,
+
+    "virtualenv_modules": [
+        "requests==2.2.1",
+    ],
+
+    "post_merge_builders": [
+        "mozilla-release hg bundle",
+    ],
+    "post_merge_nightly_branches": [
+        # No nightlies on mozilla-release
+    ],
 }
--- a/configs/merge_day/central_to_aurora.py
+++ b/configs/merge_day/central_to_aurora.py
@@ -40,9 +40,22 @@ config = {
     "migration_behavior": "central_to_aurora",
 
     "balrog_rules_to_lock": [
         8, # Fennec aurora channel
         10, # Firefox aurora channel
         18, # MetroFirefox aurora channel
     ],
     "balrog_credentials_file": "oauth.txt",
+
+    "virtualenv_modules": [
+        "requests==2.2.1",
+    ],
+
+    "post_merge_builders": [
+        "mozilla-aurora hg bundle",
+        "mozilla-central hg bundle",
+    ],
+    "post_merge_nightly_branches": [
+        "mozilla-central",
+        "mozilla-aurora",
+    ],
 }
new file mode 100644
--- /dev/null
+++ b/configs/selfserve/production.py
@@ -0,0 +1,3 @@
+config = {
+    "selfserve_url": "http://buildapi.pvt.build.mozilla.org/buildapi/self-serve",
+}
new file mode 100644
--- /dev/null
+++ b/configs/selfserve/staging.py
@@ -0,0 +1,3 @@
+config = {
+    "selfserve_url": "https://secure-pub-build.allizom.org/buildapi/self-serve",
+}
new file mode 100644
--- /dev/null
+++ b/mozharness/mozilla/selfserve.py
@@ -0,0 +1,47 @@
+import json
+import os
+import site
+
+# SelfServeMixin {{{1
+class SelfServeMixin(object):
+    def _get_session(self):
+        site_packages_path = self.query_python_site_packages_path()
+        site.addsitedir(site_packages_path)
+        import requests
+        session = requests.Session()
+        adapter = requests.adapters.HTTPAdapter(max_retries=5)
+        session.mount("http://", adapter)
+        return session
+
+    def _get_base_url(self):
+        return self.config["selfserve_url"].rstrip("/")
+
+    def trigger_nightly_builds(self, branch, revision):
+        session = self._get_session()
+
+        selfserve_base = self._get_base_url()
+        url = "%s/%s/rev/%s/nightly" % (selfserve_base, branch, revision)
+
+        data = {
+            "revision": revision,
+        }
+        self.info("Triggering nightly builds via %s" % url)
+        return session.post(url, data=data)
+
+    def trigger_arbitrary_job(self, builder, branch, revision, files=None):
+        session = self._get_session()
+
+        selfserve_base = self._get_base_url()
+        url = "%s/%s/builders/%s/%s" % (selfserve_base, branch, builder, revision)
+
+        data = {
+            "properties": json.dumps({
+                "branch": branch,
+                "revision": revision
+            }),
+        }
+        if files:
+            data["files"] = json.dumps(files)
+
+        self.info("Triggering arbritrary job at %s" % url)
+        return session.post(url, data=data)
--- a/scripts/merge_day/gecko_migration.py
+++ b/scripts/merge_day/gecko_migration.py
@@ -20,27 +20,29 @@ import os
 import pprint
 import subprocess
 import sys
 
 sys.path.insert(1, os.path.dirname(os.path.dirname(sys.path[0])))
 
 from mozharness.base.errors import HgErrorList
 from mozharness.base.log import INFO, FATAL
+from mozharness.base.python import VirtualenvMixin, virtualenv_config_options
 from mozharness.base.vcs.vcsbase import MercurialScript
 from mozharness.base.vcs.mercurial import MercurialVCS
+from mozharness.mozilla.selfserve import SelfServeMixin
 from mozharness.mozilla.updates.balrog import BalrogMixin
 
 VALID_MIGRATION_BEHAVIORS = (
     "beta_to_release", "aurora_to_beta", "central_to_aurora", "release_to_esr"
 )
 
 
 # GeckoMigration {{{1
-class GeckoMigration(MercurialScript, BalrogMixin):
+class GeckoMigration(MercurialScript, BalrogMixin, VirtualenvMixin, SelfServeMixin):
     config_options = [
         [['--hg-user', ], {
             "action": "store",
             "dest": "hg_user",
             "type": "string",
             "default": "ffxbld <release@mozilla.com>",
             "help": "Specify what user to use to commit to hg.",
         }],
@@ -63,25 +65,27 @@ class GeckoMigration(MercurialScript, Ba
             "type": "string",
             "help": "Comma separated list of locales to remove from the 'to' repo.",
         }],
     ]
     gecko_repos = None
 
     def __init__(self, require_config_file=True):
         super(GeckoMigration, self).__init__(
-            config_options=self.config_options,
+            config_options=virtualenv_config_options + self.config_options,
             all_actions=[
                 'clobber',
+                'create-virtualenv',
                 'clean-repos',
                 'pull',
                 'lock-update-paths',
                 'migrate',
                 'commit-changes',
                 'push',
+                'trigger-builders',
             ],
             default_actions=[
                 'clean-repos',
                 'pull',
                 'migrate',
             ],
             require_config_file=require_config_file
         )
@@ -674,13 +678,33 @@ the script (--clean-repos --pull --migra
                     message = """m-c push failed!
 You may be able to fix by |hg rebase| and rerunning --push if successful.
 If not, try rerunning the script (--clean-repos --pull --migrate).
 The second run will be faster."""
                 else:
                     message = error_message
                 self.fatal(message)
 
+    def trigger_builders(self):
+        """Triggers builders that should be run directly after a merge.
+        There are two different types of things we trigger:
+        1) Nightly builds ("post_merge_nightly_branches" in the config).
+           These are triggered with buildapi's nightly build endpoint to avoid
+           duplicating all of the nightly builder names into the gecko
+           migration mozharness configs. (Which would surely get out of date
+           very quickly).
+        2) Arbitrary builders ("post_merge_builders"). These are additional
+           builders to trigger that aren't part of the nightly builder set.
+           For example: hg bundle generation builders.
+        """
+        dirs = self.query_abs_dirs()
+        branch = self.config["to_repo_url"].rstrip("/").split("/")[-1]
+        revision = self.query_to_revision()
+        for builder in self.config["post_merge_builders"]:
+            self.trigger_arbitrary_job(builder, branch, revision)
+        for nightly_branch in self.config["post_merge_nightly_branches"]:
+            nightly_revision = self.query_hg_revision(os.path.join(dirs["abs_work_dir"], nightly_branch))
+            self.trigger_nightly_builds(nightly_branch, nightly_revision)
 
 # __main__ {{{1
 if __name__ == '__main__':
     gecko_migration = GeckoMigration()
     gecko_migration.run_and_exit()