Bug 1083923 - Autoland should stop squashing changesets when transplanting
authorDan Minor <dminor@mozilla.com>
Thu, 16 Oct 2014 20:09:18 -0400
changeset 1901 4cebc6dc5210c1cb1cd3ecddbc4fce560594036e
parent 1900 dc528e0b3a88dbae19ab0ac6321c48cbee29d3b0
child 1902 b7e98165a939b90c2437c08cb6fc015442e434bf
push id517
push usergszorc@mozilla.com
push dateTue, 27 Jan 2015 22:09:25 +0000
treeherderversion-control-tools@8e091ac4f8a1 [default view] [failures only]
perfherder[talos] [build metrics] [platform microbench] (compared to previous push)
bugs1083923
Bug 1083923 - Autoland should stop squashing changesets when transplanting
autoland/autoland/autoland.py
autoland/autoland/autoland_pulse.py
autoland/autoland/test_autoland.py
autoland/autoland/test_autoland_pulse.py
autoland/autoland/transplant.py
autoland/sql/schema.sql
--- a/autoland/autoland/autoland.py
+++ b/autoland/autoland/autoland.py
@@ -27,86 +27,89 @@ BUGZILLA_COMMENT_LIMIT = 10  # max comme
 
 def handle_single_failure(logger, auth, dbconn, tree, rev, buildername, build_id):
     logger.debug('autoland request %s %s needs to retry job for %s' % (tree, rev, buildername))
     job_id = selfserve.rebuild_job(auth, tree, build_id)
     if job_id:
         logger.info('submitted rebuild request %s for autoland job %s %s' % (job_id, tree, rev))
         cursor = dbconn.cursor()
         query = """
-            update AutolandRequest set last_updated=%s
+            update Autoland set last_updated=%s
             where tree=%s and revision=%s
         """
         cursor.execute(query, (datetime.datetime.now(), tree, rev))
         dbconn.commit()
     else:
         logger.info('could not rebuild %s for autoland job %s %s' % (build_id, tree, rev))
 
 
 def handle_insufficient_permissions(logger, dbconn, tree, rev, bugid, blame):
     cursor = dbconn.cursor()
     query = """
-        update AutolandRequest set can_be_landed=false, last_updated=%s
+        update Autoland set can_be_landed=false, last_updated=%s
         where tree=%s and revision=%s
     """
     cursor.execute(query, (datetime.datetime.now(), tree, rev))
     dbconn.commit()
 
     comment = 'Autoland request failed. User %s has insufficient permissions to land on tree %s.' %  (blame, tree)
     add_bugzilla_comment(dbconn, bugid, comment)
 
 def handle_failure(logger, dbconn, tree, rev, bugid, buildernames):
     logger.info('autoland request %s %s has too many failures for %s' % (tree, rev, ', '.join(buildernames)))
 
     cursor = dbconn.cursor()
     query = """
-        update AutolandRequest set can_be_landed=FALSE,last_updated=%s
+        update Autoland set can_be_landed=FALSE,last_updated=%s
         where tree=%s and revision=%s
     """
     cursor.execute(query, (datetime.datetime.now(), tree, rev))
     dbconn.commit()
 
     #TODO: add treeherder/tbpl link for job
     comment = 'Autoland request failed. Too many failures for %s.' %  (', '.join(buildernames))
     add_bugzilla_comment(dbconn, bugid, comment)
 
 def handle_can_be_landed(logger, dbconn, tree, rev):
     logger.info('autoland request %s %s can be landed' % (tree, rev))
     cursor = dbconn.cursor()
     query = """
-        update AutolandRequest set can_be_landed=true,last_updated=%s
+        update Autoland set can_be_landed=true,last_updated=%s
         where tree=%s and revision=%s
     """
     cursor.execute(query, (datetime.datetime.now(), tree, rev))
     dbconn.commit()
 
 def handle_pending_transplants(logger, dbconn):
     cursor = dbconn.cursor()
     query = """
-        select tree,revision,bugid,message from AutolandRequest
+        select tree,revision,bugid from Autoland
         where can_be_landed is true and landed is null
     """
     cursor.execute(query)
 
     landed = []
     for row in cursor.fetchall():
-        tree, rev, bugid, message = row
+        tree, rev, bugid = row
 
         pushlog = mercurial.get_pushlog(tree, rev)
         if not pushlog:
             logger.debug('could not get pushlog for tree: %s rev %s' % (tree, rev))
             return
 
         changesets = []
         for key in pushlog:
             for changeset in pushlog[key]['changesets']:
-                changesets.append(changeset)
+                # we assume by convention head revision is empty and should
+                # not be landed
+                if changeset != rev:
+                    changesets.append(changeset)
 
         # TODO: allow for transplant to other trees than 'mozilla-inbound'
-        result = transplant.transplant(tree, 'mozilla-inbound', changesets, message)
+        result = transplant.transplant(tree, 'mozilla-inbound', changesets)
 
         if not result:
             logger.debug('could not connect to transplant server: tree: %s rev %s' % (tree, rev))
             continue
 
         if 'error' in result:
             succeeded = False
             logger.info('transplant failed: tree: %s rev: %s error: %s' % (tree, rev, json.dumps(result)))
@@ -115,17 +118,17 @@ def handle_pending_transplants(logger, d
             succeeded = True
             comment = 'Autoland request succeeded: mozilla-inbound tip: %s' % result['tip']
 
         landed.append([succeeded, json.dumps(result), datetime.datetime.now(), tree, rev])
         add_bugzilla_comment(dbconn, bugid, comment)
 
     if landed:
         query = """
-            update AutolandRequest set landed=%s,transplant_result=%s,last_updated=%s
+            update Autoland set landed=%s,transplant_result=%s,last_updated=%s
             where tree=%s and revision=%s
         """
         cursor.executemany(query, landed)
         dbconn.commit()
 
 def handle_autoland_request(logger, auth, dbconn, tree, rev):
 
     logger.info('looking at autoland request %s %s' % (tree, rev))
@@ -136,25 +139,25 @@ def handle_autoland_request(logger, auth
         return
 
     cursor = dbconn.cursor()
     if not status['job_complete']:
         logger.debug('autoland request %s %s job not complete' % (tree, rev))
         # update pending so we won't look at this job again
         # until we get another update over pulse
         query = """
-            update AutolandRequest set pending=null,last_updated=%s
+            update Autoland set pending=null,last_updated=%s
             where tree=%s and revision=%s
         """
         cursor.execute(query, (datetime.datetime.now(), tree, rev))
         dbconn.commit()
         return
 
     query = """
-        select bugid, blame from AutolandRequest
+        select bugid, blame from Autoland
         where tree=%s and revision=%s
     """
     cursor.execute(query, (tree, rev))
     bugid, blame = cursor.fetchone()
 
     # check ldap group
     blame = blame.strip('{}')
     result = mozilla_ldap.check_group(mozilla_ldap.read_credentials(),
@@ -176,17 +179,17 @@ def handle_autoland_request(logger, auth
 
     if not builds:
         logger.debug('could not get jobs for revision')
         return
 
     if len(pending) > 0 or len(running) > 0:
         logger.info('autoland request %s %s still has pending or running jobs: %d %d' % (tree, rev, len(pending), len(running)))
         query = """
-            update AutolandRequest set pending=%s,running=%s,
+            update Autoland set pending=%s,running=%s,
                                        builds=%s,last_updated=%s
             where tree=%s and revision=%s
         """
         cursor.execute(query, (len(pending), len(running), len(builds), datetime.datetime.now(), tree, rev))
         dbconn.commit()
         return
 
     # organize build results by job type
@@ -286,27 +289,27 @@ def main():
     old_job = datetime.timedelta(minutes=30)
 
     while True:
         try:
             cursor = dbconn.cursor()
             now = datetime.datetime.now()
 
             # handle potentially finished autoland jobs
-            query = """select tree,revision from AutolandRequest
+            query = """select tree,revision from Autoland
                        where pending=0 and running=0 and last_updated<=%(time)s
                        and can_be_landed is null"""
             cursor.execute(query, ({'time': now - stable_delay}))
             for row in cursor.fetchall():
                 tree, rev = row
                 handle_autoland_request(logger, auth, dbconn, tree, rev)
 
             # we also look for any older jobs - maybe something got missed
             # in pulse
-            query = """select tree,revision from AutolandRequest
+            query = """select tree,revision from Autoland
                        where last_updated<=%(time)s
                        and can_be_landed is null"""
             cursor.execute(query, ({'time': now - old_job}))
             for row in cursor.fetchall():
                 tree, rev = row
                 handle_autoland_request(logger, auth, dbconn, tree, rev)
 
             #
--- a/autoland/autoland/autoland_pulse.py
+++ b/autoland/autoland/autoland_pulse.py
@@ -34,21 +34,21 @@ def read_credentials():
 
 def extract_bugid(patch):
     #TODO: check to see if there is an "official" re for this
     bugid = re.compile('[Bb]ug (\d+)')
     m = re.search(bugid, patch)
     if m:
         return m.groups()[0]
 
-def is_known_autoland_job(dbconn, tree, rev): 
+def is_known_autoland_job(dbconn, tree, rev):
     cursor = dbconn.cursor()
 
     # see if we know already know about this autoland request
-    query = """select revision from AutolandRequest
+    query = """select revision from Autoland
                where tree=%(tree)s
                and substring(revision, 0, %(len)s)=%(rev)s"""
     cursor.execute(query, {'tree': tree,
                            'len': len(rev) + 1,
                            'rev': rev})
     row = cursor.fetchone()
     return row is not None
 
@@ -63,31 +63,29 @@ def handle_message(data, message):
             json.dump(data, f, indent=2, sort_keys=True)
 
     if key.find('started') != -1:
         blame = payload['build']['blame']
 
         tree = None
         rev = None
         autoland = False
-        message = None
         bugid = None
         for prop in payload['build']['properties']:
             if prop[0] == 'revision':
                 rev = prop[1]
             elif prop[0] == 'branch':
                 tree = prop[1]
         try:
             for change in payload['build']['sourceStamp']['changes']:
                 comments = change['comments']
                 index = comments.find('--autoland')
                 if index > -1:
                     autoland = True
-                    message = comments[index + len('--autoland') + 1:].strip()
-                    bugid = extract_bugid(message)
+                    bugid = extract_bugid(comments)
         except KeyError:
             pass
 
         if autoland:
             logger.info('found autoland job: %s %s' % (tree, rev))
 
             if not bugid:
                 logger.info('autoland job missing bugid')
@@ -97,21 +95,21 @@ def handle_message(data, message):
 
             if is_known_autoland_job(dbconn, tree, rev):
                 return
 
             logger.info('found new autoland job')
 
             # insert into database
             query = """
-                insert into AutolandRequest(tree,revision,bugid,blame,message,last_updated)
-                values(%s,%s,%s,%s,%s,%s)
+                insert into Autoland(tree,revision,bugid,blame,last_updated)
+                values(%s,%s,%s,%s,%s)
             """
             cursor = dbconn.cursor()
-            cursor.execute(query, (tree, rev, bugid, blame, message, datetime.datetime.now()))
+            cursor.execute(query, (tree, rev, bugid, blame, datetime.datetime.now()))
             dbconn.commit()
     elif key.find('finished') != -1:
         rev = None
         tree = None
         for prop in payload['build']['properties']:
             if prop[0] == 'revision':
                 rev = prop[1]
             elif prop[0] == 'branch':
@@ -120,17 +118,17 @@ def handle_message(data, message):
         if tree and rev:
             if is_known_autoland_job(dbconn, tree, rev):
                 logger.info('updating autoland job: %s %s' % (tree, rev))
 
                 pending, running, builds = selfserve.jobs_for_revision(auth, tree, rev)
                 logger.info('pending: %d running: %d builds: %d' % (len(pending), len(running), len(builds)))
 
                 query = """
-                    update AutolandRequest set pending=%s,
+                    update Autoland set pending=%s,
                         running=%s,builds=%s,last_updated=%s
                     where tree=%s
                     and substring(revision, 0, %s)=%s"""
                 cursor = dbconn.cursor()
                 cursor.execute(query, (len(pending), len(running), len(builds), datetime.datetime.now(), tree, len(rev) + 1, rev))
                 dbconn.commit()
 
 def main():
--- a/autoland/autoland/test_autoland.py
+++ b/autoland/autoland/test_autoland.py
@@ -4,18 +4,18 @@ import responses
 import unittest
 
 import autoland
 
 class TestAutoland(unittest.TestCase):
 
     def clear_database(self):
         cursor = self.dbconn.cursor()
-        cursor.execute('delete from autolandrequest')
-        cursor.execute('delete from bugzillacomment')
+        cursor.execute('delete from Autoland')
+        cursor.execute('delete from BugzillaComment')
         self.dbconn.commit()
 
     def setUp(self):
         dsn = 'dbname=testautoland user=autoland host=localhost password=autoland'
         self.dbconn = psycopg2.connect(dsn)
         self.clear_database()
 
         self.logger = logging.getLogger(__name__)
@@ -25,25 +25,25 @@ class TestAutoland(unittest.TestCase):
 
     def test_handle_insufficient_permissions(self):
         tree = 'try'
         rev = 'a revision' 
         bugid = '1'
         blame = 'cthulhu@mozilla.com'
 
         cursor = self.dbconn.cursor()
-        query = """insert into AutolandRequest(tree, revision)
+        query = """insert into Autoland(tree, revision)
                    values(%s, %s)"""
         cursor.execute(query, (tree, rev))
         self.dbconn.commit()
 
         autoland.handle_insufficient_permissions(self.logger, self.dbconn,
             tree, rev, bugid, blame)
 
-        query = """select can_be_landed, last_updated from AutolandRequest
+        query = """select can_be_landed, last_updated from Autoland
                    where tree=%s and revision=%s"""
         cursor.execute(query, (tree, rev))
         can_be_landed, last_updated = cursor.fetchone()
         self.assertEqual(can_be_landed, False)
         self.assertIsNotNone(last_updated)
 
     def test_add_bugzilla_comment(self):
         autoland.add_bugzilla_comment(self.dbconn, '1', 'a comment')
--- a/autoland/autoland/test_autoland_pulse.py
+++ b/autoland/autoland/test_autoland_pulse.py
@@ -10,17 +10,17 @@ import autoland_pulse
 class DummyMessage(object):
     def ack(self):
         pass
 
 class TestAutolandPulse(unittest.TestCase):
 
     def clear_database(self):
         cursor = self.dbconn.cursor()
-        cursor.execute('delete from autolandrequest')
+        cursor.execute('delete from Autoland')
         self.dbconn.commit()
 
     def setUp(self):
         dsn = 'dbname=testautoland user=autoland host=localhost password=autoland'
         self.dbconn = psycopg2.connect(dsn)
         self.clear_database()
 
         autoland_pulse.dbconn = self.dbconn
@@ -32,17 +32,17 @@ class TestAutolandPulse(unittest.TestCas
             for comment in comments:
                 bugid = autoland_pulse.extract_bugid(comment['comment'])
                 self.assertEqual(bugid, comment['bugid'], '%s should match %s' %
                     (bugid, comment['bugid']))
 
     def test_is_known_autoland_job(self):
         tree = 'try'
         rev = 'd28403874a12f2f5449190ce267a58d7abab350a'
-        query = """insert into AutolandRequest(tree, revision)
+        query = """insert into Autoland(tree, revision)
                    values(%s, %s)
         """
         cursor = self.dbconn.cursor()
         cursor.execute(query, (tree, rev))
         self.dbconn.commit()
 
         # we need to match "short" revisions against the full string
         for i in xrange(0, len(rev) + 1):
@@ -75,17 +75,17 @@ class TestAutolandPulse(unittest.TestCas
 
         # read and replay canned messages
         cursor = self.dbconn.cursor()
         with gzip.open('test-data/pulse-messages.json.gz') as f:
             message_data = json.load(f)
         for i, data in enumerate(message_data):
             autoland_pulse.handle_message(data, DummyMessage())
 
-            cursor.execute('select pending,running,builds from AutolandRequest')
+            cursor.execute('select pending,running,builds from Autoland')
             if i < 176:
                 self.assertEqual(cursor.rowcount, 0, 'should be no autoland jobs')
             else:
                 self.assertEqual(cursor.rowcount, 1, 'autoland job not found')
                 pending, running, builds = cursor.fetchone()
 
                 if i < 5307:
                     self.assertIsNone(pending, "pending should be none")
--- a/autoland/autoland/transplant.py
+++ b/autoland/autoland/transplant.py
@@ -1,23 +1,19 @@
 import json
 import requests
 
-#TRANSPLANT_URL = 'http://transplant.infinity.com.ua'
 TRANSPLANT_URL = 'http://localhost:5000'
 
-def transplant(src, dest, changesets, message):
-    """Transplant changsets from src to dest using the provided message"""
+def transplant(src, dest, changesets):
+    """Transplant changesets from src to dest"""
     url = TRANSPLANT_URL + '/transplant'
     headers = {'Content-Type': 'application/json'}
     data = {
         'src': src,
         'dst': dest,
-        'items': [
-            {'revset': ' + '.join(changesets),
-             'message': message}
-        ]
+        'items': [{'commit': changeset} for changeset in changesets]
     }
     try:
         r = requests.post(url, data=json.dumps(data), headers=headers)
     except requests.exceptions.ConnectionError:
         return
     return json.loads(r.text)
--- a/autoland/sql/schema.sql
+++ b/autoland/sql/schema.sql
@@ -1,26 +1,25 @@
 alter user autoland with password 'autoland';
 
-create table AutolandRequest (
+create table Autoland (
     tree varchar(20),
     revision varchar(40),
     bugid integer,
     blame varchar(120),
-    message text,
     pending integer,
     running integer,
     builds integer,
     last_updated timestamp,
     can_be_landed boolean,
     landed boolean,
     transplant_result text,
     primary key(tree, revision)
 );
-grant all privileges on table AutolandRequest to autoland;
+grant all privileges on table Autoland to autoland;
 
 create table BugzillaComment (
     sequence bigserial,
     bugid integer,
     bug_comment text,
     primary key(sequence)
 );
 grant all privileges on table BugzillaComment to autoland;