Ability to sync pushlogs and query push info for a changeset
authorGregory Szorc <gps@mozilla.com>
Thu, 25 Jul 2013 11:47:08 -0700
changeset 30 7204cbc9f33b97c753f51746f806d68984a615cb
parent 29 3f702b69b06c1a9054aa09af8464481234715171
child 31 c28a3b4ed986ee887895a40e869a49078ed7edf6
push id14
push usergszorc@mozilla.com
push dateThu, 25 Jul 2013 18:47:14 +0000
Ability to sync pushlogs and query push info for a changeset
__init__.py
mozautomation/changetracker.py
mozautomation/repository.py
--- a/__init__.py
+++ b/__init__.py
@@ -82,16 +82,17 @@ this will analyze the current working di
 clean, the tip changeset will be analyzed. By default, only changed lines are
 reported on.
 
 Static analysis is also performed automatically during qrefresh and commit
 operations. To disable this behavior, add "noautocritic = True" to the
 [mozext] section in your hgrc.
 """
 
+import datetime
 import errno
 import os
 import shutil
 import sys
 
 import mercurial.commands as commands
 
 from mercurial.i18n import _
@@ -111,18 +112,24 @@ from mercurial import (
     cmdutil,
     demandimport,
     encoding,
     extensions,
     hg,
     util,
 )
 
+from mozautomation.changetracker import (
+    ChangeTracker,
+)
+
 from mozautomation.repository import (
     MercurialRepository,
+    RELEASE_TREES,
+    REPOS,
     resolve_trees_to_official,
     resolve_trees_to_uris,
     resolve_uri_to_tree,
 )
 
 import bzauth
 import bz
 
@@ -357,16 +364,60 @@ def critic(ui, repo, rev='.', entire=Fal
     """Perform a critique of a changeset.
 
     This will perform static analysis on a given changeset and report any
     issues found.
     """
     critique(ui, repo, node=rev, entire=entire, **opts)
 
 
+@command('pushlogsync', [], _('hg pushlogsync'))
+def syncpushinfo(ui, repo, tree=None, **opts):
+    """Synchronize the pushlog information for all known Gecko trees.
+
+    The pushlog info contains who, when, and where individual changesets were
+    pushed.
+
+    After running this command, you can query for push information for specific
+    changesets.
+    """
+    tracker = ChangeTracker(repo.join('changetracker.db'))
+
+    for i, tree in enumerate(sorted(REPOS)):
+        tracker.load_pushlog(tree)
+        ui.progress('pushlogsync', i, total=len(REPOS))
+
+    ui.progress('pushlogsync', None)
+
+
+@command('changesetpushes',
+    [('a', 'all', False, _('Show all trees, not just release trees.'), '')],
+    _('hg changesetpushes REV'))
+def changesetpushes(ui, repo, rev, all=False, **opts):
+    """Display pushlog information for a changeset.
+
+    This command prints pushlog entries for a given changeset. It is used to
+    answer the question: how did a changeset propagate to all the trees.
+    """
+    ctx = repo[rev]
+    node = ctx.hex()
+
+    tracker = ChangeTracker(repo.join('changetracker.db'))
+    pushes = [p for p in tracker.pushes_for_changeset(node) if all or p[0] in
+        RELEASE_TREES]
+    longest_tree = max(len(p[0]) for p in pushes) + 2
+
+    ui.write(ctx.rev(), ':', str(ctx), ' ', ctx.description(), '\n')
+
+    ui.write('Tree'.ljust(longest_tree), 'Date'.ljust(19), ' Username\n')
+    for tree, push_id, when, user, head_changeset in pushes:
+        date = datetime.datetime.fromtimestamp(when)
+        ui.write(tree.ljust(longest_tree), date.isoformat(), ' ', user, '\n')
+
+
 def critic_hook(ui, repo, node=None, **opts):
     critique(ui, repo, node=node, **opts)
     return 0
 
 
 class remoterefs(dict):
     """Represents a remote refs file."""
 
@@ -475,12 +526,33 @@ def reposetup(ui, repo):
 
         def _update_remote_refs(self, remote, tree):
             for branch, nodes in remote.branchmap().items():
                 for node in nodes:
                     self.remoterefs['%s/%s' % (tree, branch)] = node
 
             self.remoterefs.write()
 
+        def _milestone_changesets(self):
+            """Look up Gecko milestone changes.
+
+            Returns a mapping of changeset node to milestone text.
+            """
+            m = {}
+
+            for rev in self.file('config/milestone.txt'):
+                ctx = self.filectx('config/milestone.txt', fileid=rev)
+
+                lines = ctx.data().splitlines()
+                lines = [l for l in lines if not l.startswith('#') and
+                    l.strip()]
+
+                if len(lines) != 1:
+                    continue
+
+                m[ctx.node()] = lines[0]
+
+            return m
+
     repo.__class__ = remotestrackingrepo
     if not ui.configbool('mozext', 'noautocritic'):
         ui.setconfig('hooks', 'commit.critic', critic_hook)
         ui.setconfig('hooks', 'qrefresh.critic', critic_hook)
new file mode 100644
--- /dev/null
+++ b/mozautomation/changetracker.py
@@ -0,0 +1,97 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import sqlite3
+
+from .repository import (
+    MercurialRepository,
+    resolve_trees_to_uris,
+)
+
+
+class ChangeTracker(object):
+    """Data store for tracking changes and bugs and repository events."""
+
+    def __init__(self, path):
+        self._db = sqlite3.connect(path)
+
+        if not self._schema_current():
+            self._create_schema()
+
+    def _schema_current(self):
+        return self._db.execute('SELECT COUNT(*) FROM SQLITE_MASTER WHERE '
+            'name="trees"').fetchone()[0] == 1
+
+    def _create_schema(self):
+        with self._db:
+            self._db.execute('CREATE TABLE trees ('
+                'id INTEGER PRIMARY KEY AUTOINCREMENT, '
+                'name TEXT, '
+                'url TEXT '
+                ')')
+
+            self._db.execute('CREATE TABLE pushes ('
+                'push_id INTEGER, '
+                'tree_id INTEGER, '
+                'time INTEGER, '
+                'user TEXT, '
+                'PRIMARY KEY (push_id, tree_id) '
+                ')')
+
+            self._db.execute('CREATE TABLE changeset_pushes ('
+                'changeset TEXT, '
+                'head_changeset TEXT, '
+                'push_id INTEGER, '
+                'tree_id INTEGER, '
+                'UNIQUE (changeset, tree_id) '
+                ')')
+
+    def tree_id(self, tree, url=None):
+        with self._db:
+            field = self._db.execute('SELECT id FROM trees WHERE name=? LIMIT 1',
+                [tree]).fetchone()
+
+            if field:
+                return field[0]
+
+            self._db.execute('INSERT INTO trees (name, url) VALUES (?, ?)',
+                [tree, url])
+
+            return self._db.execute('SELECT id FROM trees WHERE name=? LIMIT 1',
+                [tree]).fetchone()[0]
+
+    def load_pushlog(self, tree):
+        tree, url = resolve_trees_to_uris([tree])[0]
+        repo = MercurialRepository(url)
+
+        tree_id = self.tree_id(tree, url)
+
+        last_push_id = self._db.execute('SELECT push_id FROM pushes WHERE '
+            'tree_id=? ORDER BY push_id DESC LIMIT 1', [tree_id]).fetchone()
+
+        last_push_id = last_push_id[0] if last_push_id else -1
+
+        with self._db:
+            for push_id, push in repo.push_info(start_id=last_push_id + 1):
+                self._db.execute('INSERT INTO pushes (push_id, tree_id, time, '
+                'user) VALUES (?, ?, ?, ?)', [push_id, tree_id, push['date'],
+                    push['user']])
+
+                head = push['changesets'][0]
+
+                for changeset in push['changesets']:
+                    self._db.execute('INSERT INTO changeset_pushes VALUES '
+                        '(?, ?, ?, ?)', [changeset, head, push_id, tree_id])
+
+    def pushes_for_changeset(self, changeset):
+        for row in self._db.execute('SELECT trees.name, pushes.push_id, '
+            'pushes.time, pushes.user, changeset_pushes.head_changeset '
+            'FROM trees, pushes, changeset_pushes '
+            'WHERE pushes.push_id = changeset_pushes.push_id AND '
+            'pushes.tree_id = changeset_pushes.tree_id AND '
+            'trees.id = pushes.tree_id AND changeset_pushes.changeset=? '
+            'ORDER BY pushes.time ASC', [changeset]):
+            yield row
--- a/mozautomation/repository.py
+++ b/mozautomation/repository.py
@@ -82,16 +82,18 @@ OFFICIAL_MAP = {
     'services': 'services-central',
     'release': 'mozilla-release',
     'aurora': 'mozilla-aurora',
     'beta': 'mozilla-beta',
     'build': 'build-system',
     'esr17': 'mozilla-esr17',
 }
 
+RELEASE_TREES = set(['central', 'aurora', 'beta', 'release', 'b2g18', 'esr17'])
+
 
 def resolve_trees_to_official(trees):
     mapped = []
     for tree in trees:
         mapped.extend(TREE_ALIASES.get(tree, [tree]))
     mapped = [OFFICIAL_MAP.get(tree, tree) for tree in mapped]
 
     return mapped
@@ -182,8 +184,22 @@ class MercurialRepository(object):
         response = self._opener.open(request)
         o = json.load(response)
 
         if not o:
             return None
 
         push_id = o.keys()[0]
         return PushInfo(push_id, o[push_id])
+
+    def push_info(self, full=False, start_id=0):
+        """Obtain all pushlog info for a repository."""
+
+        url = '%s/json-pushes?startID=%d' % (self.url, start_id)
+        if full:
+            url += '&full=1'
+        request = urllib2.Request(url)
+
+        response = self._opener.open(request)
+        pushes = json.load(response)
+
+        for push_id in sorted(int(k) for k in pushes):
+            yield push_id, pushes[str(push_id)]