Add mozautomation
authorGregory Szorc <gps@mozilla.com>
Sun, 21 Jul 2013 15:19:48 -0700
changeset 0 2f314876bbddbd468757149edecad87ed1c8c04a
child 1 53f2eafb1b3de15049934d4565a5d1ea765f53e3
push id1
push usergszorc@mozilla.com
push dateSun, 21 Jul 2013 22:40:54 +0000
Add mozautomation
mozautomation/mozautomation/__init__.py
mozautomation/mozautomation/buildstatus.py
mozautomation/mozautomation/repository.py
mozautomation/mozautomation/selfserve.py
mozautomation/mozautomation/treestatus.py
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/mozautomation/mozautomation/buildstatus.py
@@ -0,0 +1,203 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import json
+import urllib2
+
+
+PLATFORMS = dict(
+    ANDROID_4_0_PANDA=('Android 4.0', 10),
+    ANDROID_ARMV6_TEGRA_250=('Android 2.2 Armv6', 10),
+    ANDROID_TEGRA_250=('', 10),
+    ANDROID_TEGRA_250_NOION=('', 10),
+    B2G_EMULATOR_VM=('B2G Emu (VM)', 10),
+    B2G_EMULATOR=('B2G Emu', 10),
+    OSX_10_8=('OS X 10.8', 10),
+    OSX_10_7=('OS X 10.7', 10),
+    OSX_10_6=('OS X 10.6', 10),
+    UBUNTU_HW_1204_64=('Ubuntu64', 10),
+    UBUNTU_HW_1204_32=('Ubuntu32', 10),
+    UBUNTU_VM_1204_64=('Ubuntu64', 10),
+    UBUNTU_VM_1204_32=('Ubuntu32', 10),
+    WINDOWS_XP_32=('WinXP', 10),
+    WINDOWS_7_32=('Win7', 10),
+    WINDOWS_8=('Win8', 10),
+)
+
+PREFIXES = [
+    ('Windows XP 32-bit', 'WINDOWS_XP_32'),
+    ('Windows 7 32-bit', 'WINDOWS_7_32'),
+    ('WINNT 6.2', 'WINDOWS_8'),
+    ('Ubuntu HW 12.04 x64', 'UBUNTU_HW_1204_64'),
+    ('Ubuntu HW 12.04', 'UBUNTU_HW_1204_32'),
+    ('Ubuntu VM 12.04 x64', 'UBUNTU_VM_1204_64'),
+    ('Ubuntu VM 12.04', 'UBUNTU_VM_1204_32'),
+    ('Rev5 MacOSX Mountain Lion 10.8', 'OSX_10_8'),
+    ('Rev4 MacOSX Lion 10.7', 'OSX_10_7'),
+    ('Rev4 MacOSX Snow Leopard 10.6', 'OSX_10_6'),
+    ('Android 4.0 Panda', 'ANDROID_4_0_PANDA'),
+    ('Android Armv6 Tegra 250', 'ANDROID_ARMV6_TEGRA_250'),
+    ('Android Tegra 250', 'ANDROID_TEGRA_250'),
+    ('b2g_emulator_vm', 'B2G_EMULATOR_VM'),
+    ('b2g_emulator', 'B2G_EMULATOR'),
+
+    # This is where it starts to get a little inconsistent.
+    ('Android no-ionmonkey Tegra 250', 'ANDROID_TEGRA_250_NOION'),
+    ('Android no-ionmonkey', 'ANDROID_TEGRA_250_NOION'),
+]
+
+TREES = {'mozilla-central', 'mozilla-inbound'}
+
+JOBS = {
+    'build',
+    'chromez',
+    'crashtest',
+    'crashtest-1',
+    'crashtest-2',
+    'crashtest-3',
+    'crashtest-ipc',
+    'dirtypaint',
+    'dromaeojs',
+    'jetpack',
+    'jsreftest',
+    'jsreftest-1',
+    'jsreftest-2',
+    'jsreftest-3',
+    'hsreftest',
+    'marionette',
+    'marionette-webapi',
+    'mochitest-1',
+    'mochitest-2',
+    'mochitest-3',
+    'mochitest-4',
+    'mochitest-5',
+    'mochitest-6',
+    'mochitest-7',
+    'mochitest-8',
+    'mochitest-9',
+    'mochitest-browser-chrome',
+    'mochitest-gl',
+    'mochitest-metro-chrome',
+    'mochitest-other',
+    'other',
+    'plain-reftest-1',
+    'plain-reftest-2',
+    'plain-reftest-3',
+    'plain-reftest-4',
+    'reftest',
+    'reftest-1',
+    'reftest-2',
+    'reftest-3',
+    'reftest-4',
+    'reftest-5',
+    'reftest-6',
+    'reftest-7',
+    'reftest-8',
+    'reftest-9',
+    'reftest-10',
+    'reftest-ipc',
+    'reftest-no-accel',
+    'remote-tp4m_chochrome',
+    'remote-tp4m_nochrome',
+    'remote-trobocheck2',
+    'remote-trobopan',
+    'remote-troboprovider',
+    'remote-ts',
+    'remote-tsvg',
+    'robocop-1',
+    'robocop-2',
+    'svgr',
+    'talos',
+    'tp5o',
+    'xpcshell',
+}
+
+
+def parse_builder_name(b):
+    """Parse a builder name into metadata."""
+
+    platform = None
+
+    remaining = ''
+
+    for prefix, key in PREFIXES:
+        if b.startswith(prefix):
+            platform = key
+            remaining = b[len(prefix)+1:]
+            break
+
+    tree = None
+    opt_level = None
+    job_type = None
+    props = set(remaining.split())
+    try:
+        props.remove('test')
+    except KeyError:
+        pass
+
+    for t in TREES:
+        if t in props:
+            tree = t
+            props.remove(t)
+            break
+
+    if 'opt' in props:
+        opt_level = 'opt'
+    elif 'debug' in props:
+        opt_level = 'debug'
+    elif 'pgo' in props:
+        opt_level = 'pgo'
+
+    if opt_level:
+        props.remove(opt_level)
+
+    for job in JOBS:
+        if job in props:
+            job_type = job
+            props.remove(job)
+
+
+class JobResult(object):
+    """Represents the result of an individual automation job."""
+
+    def __init__(self, d):
+        self.build_id = d['_id']
+        self.builder_name = d['buildername']
+        self.slave = d['slave']
+        self.result = d['result']
+        self.start_time = int(d['starttime'])
+        self.end_time = int(d['endtime'])
+        self.log = d['log']
+        self.notes = d['notes']
+
+
+class BuildStatusResult(object):
+    def __init__(self, o):
+        self.jobs = []
+        for job in o:
+            self.jobs.append(JobResult(job))
+
+
+class BuildStatusClient(object):
+    """Client to interface with build status API."""
+
+    def __init__(self, base_uri='https://tbpl.mozilla.org/php/'):
+        self._base_uri = base_uri
+        self._opener = urllib2.build_opener()
+
+    def revision_builds(self, repo, changeset):
+        """Obtain the build status for a single changeset in a repository."""
+
+        # The API only accepts 12 digit short form changesets.
+        if len(changeset) > 12:
+            changeset = changeset[0:12]
+
+        request = urllib2.Request('%sgetRevisionBuilds.php?branch=%s&rev=%s' %
+            (self._base_uri, repo, changeset))
+
+        response = self._opener.open(request)
+
+        return BuildStatusResult(json.load(response))
new file mode 100644
--- /dev/null
+++ b/mozautomation/mozautomation/repository.py
@@ -0,0 +1,171 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import json
+import urllib2
+
+
+TREE_ALIASES = {
+    'mozilla-central': ('central',),
+    'mc': ('central',),
+    'm-c': ('central',),
+    'mozilla-inbound': ('inbound',),
+    'm-i': ('inbound',),
+    'mi': ('inbound',),
+    'inbound': ('inbound',),
+    'in': ('inbound',),
+    'fx': ('fx-team',),
+    'mozilla-services': ('services',),
+    's-c': ('services',),
+    'sc': ('services',),
+    'bs': ('build',),
+    'b-s': ('build',),
+    'build-system': ('build',),
+    'gfx': ('graphics',),
+    'mozilla-release': ('release',),
+    'mozilla-aurora': ('aurora',),
+    'mozilla-beta': ('beta',),
+
+    'releases': ('esr17', 'release', 'beta', 'aurora', 'central'),
+}
+
+BASE_READ_URI = 'https://hg.mozilla.org/'
+BASE_WRITE_URI = 'ssh://hg.mozilla.org/'
+
+REPOS = {
+    # Release repositories.
+    'central': 'mozilla-central',
+    'aurora': 'releases/mozilla-aurora',
+    'beta': 'releases/mozilla-beta',
+    'release': 'releases/mozilla-release',
+    'esr17': 'releases/mozilla-esr17',
+
+    # Integration repositories.
+    'build': 'projects/build-system',
+    'fx-team': 'integration/fx-team',
+    'graphics': 'projects/graphics',
+    'inbound': 'integration/mozilla-inbound',
+    'places': 'projects/places',
+    'services': 'services/services-central',
+
+    # Twigs
+    'alder': 'projects/alder',
+    'ash': 'projects/ash',
+    'birch': 'projects/birch',
+    'cedar': 'projects/cedar',
+    'cypress': 'projects/cypress',
+    'date': 'projects/date',
+    'elm': 'projects/elm',
+    'fig': 'projects/fig',
+    'gum': 'projects/gum',
+    'holly': 'projects/holly',
+    'jamun': 'projects/jamun',
+    'larch': 'projects/larch',
+    'maple': 'projects/maple',
+    'oak': 'projects/oak',
+    'pine': 'projects/pine',
+
+    # Misc
+    'try': 'try',
+}
+
+OFFICIAL_MAP = {
+    'central': 'mozilla-central',
+    'inbound': 'mozilla-inbound',
+    'services': 'services-central',
+    'release': 'mozilla-release',
+    'aurora': 'mozilla-aurora',
+    'beta': 'mozilla-beta',
+    'esr17': 'mozilla-esr17',
+}
+
+
+def resolve_trees_to_official(trees):
+    mapped = []
+    for tree in trees:
+        mapped.extend(TREE_ALIASES.get(tree, [tree]))
+    mapped = [OFFICIAL_MAP.get(tree, tree) for tree in mapped]
+
+    return mapped
+
+
+def resolve_trees_to_uris(trees, write_access=False):
+    """Resolve tree names to repositories URIs.
+
+    The caller passes in an iterable of tree names. These can be common names,
+    aliases, or official names.
+
+    A list of 2-tuples is returned. If a repository could be resolved to a URI,
+    the tuple is (common_name, uri). If a repository could not be resolved to a
+    URI, the tuple is (specified_name, None).
+    """
+    mapped = []
+    for tree in trees:
+        mapped.extend(TREE_ALIASES.get(tree, [tree]))
+    repos = [REPOS.get(tree, None) for tree in mapped]
+
+    base = BASE_WRITE_URI if write_access else BASE_READ_URI
+
+    uris = []
+    for i, tree in enumerate(repos):
+        if tree is None:
+            uris.append((trees[i], None))
+        else:
+            uris.append((mapped[i], '%s%s' % (base, tree)))
+
+    return uris
+
+
+class PushInfo(object):
+    """Represents an entry from the repository pushlog."""
+
+    def __init__(self, push_id, d):
+        self.push_id = push_id
+        self.date = d['date']
+        self.changesets = []
+
+        for changeset in d['changesets']:
+            entry = changeset
+            entry['tags'] = set(entry['tags']) if entry['tags'] else set()
+            self.changesets.append(entry)
+
+    @property
+    def nodes(self):
+        """All the changesets pushed in this push."""
+        return [c['node'] for c in self.changesets]
+
+    @property
+    def first_node(self):
+        return self.nodes[0]
+
+    @property
+    def last_node(self):
+        return self.nodes[-1]
+
+
+class MercurialRepository(object):
+    """Interface with a Mozilla Mercurial repository."""
+
+    def __init__(self, url):
+        self.url = url
+        self._opener = urllib2.build_opener()
+
+    def push_info_for_changeset(self, changeset):
+        """Obtain the push information for a single changeset.
+
+        Returns a PushInfo on success or None if no push info is available.
+        """
+        request = urllib2.Request('%s/json-pushes?full=1&changeset=%s' % ( self.url,
+            changeset))
+
+        response = self._opener.open(request)
+        o = json.load(response)
+
+        if not o:
+            return None
+
+        push_id = o.keys()[0]
+        return PushInfo(push_id, o[push_id])
new file mode 100644
--- /dev/null
+++ b/mozautomation/mozautomation/selfserve.py
@@ -0,0 +1,104 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This file is a client to the self-serve HTTP API.
+
+import json
+import urllib2
+
+
+class Branch(object):
+    def __init__(self, client, name, meta=None):
+        """Create a Branch bound to a client.
+
+        The name is the name/identifier of the branch on the server.
+        meta is metadata about the branch (returned from the branches() call on
+        the main client. It is optional.
+
+        Instances of this class are not meant to be instantiated outside this
+        module.
+        """
+        self._client = client
+        self.name = name
+
+        if meta:
+            self.graph_branches = meta['graph_branches']
+            self.repo = meta['repo']
+            self.repo_type = meta['repo']
+
+    def builds(self):
+        """Returns a list of builds on this branch."""
+        return self._request()
+
+    def rebuild(self, build_id):
+        """Rebuild the build specified by its ID."""
+
+    def cancel_build(self, build_id):
+        """Cancel a build specified by its ID."""
+
+    def build(self, build_id):
+        """Obtain info about a build specified by its ID."""
+        return self._request('build', build_id)
+
+    def builders(self):
+        """Return info on builders building for this branch."""
+        return self._request('builders')
+
+    def builder(self, builder):
+        """Return info on a single bingler."""
+        return self._request('builders', builder)
+
+    def _request(self, *paths):
+        return self._client._request(self.name, *paths)
+
+
+class SelfServeClient(object):
+    def __init__(self, uri, opener=None):
+        self._uri = uri
+
+        if not opener:
+            opener = urllib2.build_opener()
+
+        self._opener = opener
+
+    def branches(self):
+        """Returns all the branches as Branch instances."""
+        for name, meta in self._request('branches').items():
+            yield Branch(self, name, meta)
+
+    def jobs(self):
+        """Returns a list of past self-serve request."""
+        return self._request('jobs')
+
+    def get_job(self, job_id):
+        """Return information about a specific job."""
+        return self._request('jobs', job_id)
+
+    def __getitem__(self, key):
+        """Dictionary like access retrives branches."""
+        return Branch(self, key)
+
+    def _request(self, *paths):
+        uri = self._uri
+        for p in paths:
+            uri += '/%s' % p
+
+        request = urllib2.Request(uri, None,
+            {'Accept': 'application/json'})
+
+        response = self._opener.open(request)
+        return json.load(response)
+
+
+def get_mozilla_self_serve(username, password):
+    uri = 'https://secure.pub.build.mozilla.org/buildapi/self-serve'
+
+    handler = urllib2.HTTPBasicAuthHandler()
+    handler.add_password(realm='Mozilla Contributors - LDAP Authentication',
+        uri=uri, user=username, passwd=password)
+
+    opener = urllib2.build_opener(handler)
+
+    return SelfServeClient(uri, opener)
+
new file mode 100644
--- /dev/null
+++ b/mozautomation/mozautomation/treestatus.py
@@ -0,0 +1,108 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import unicode_literals
+
+import json
+import urllib2
+
+from time import strptime
+
+
+class TreeStatus(object):
+    """Represents the status of an individual tree."""
+
+    APPROVAL_REQUEST = 'approval required'
+    OPEN = 'open'
+    CLOSED = 'closed'
+
+    def __init__(self, d):
+        self.status = None
+        self.motd = None
+        self.tree = None
+        self.reason = None
+
+        for k in d:
+            if k == 'status':
+                self.status = d[k]
+            elif k == 'message_of_the_day':
+                self.motd = d[k]
+            elif k == 'tree':
+                self.tree = d[k]
+            elif k == 'reason':
+                self.reason = d[k]
+            else:
+                raise Exception('Unknown key in Tree Status response: %s' % k)
+
+    @property
+    def open(self):
+        return self.status == self.OPEN
+
+    @property
+    def closed(self):
+        return self.status == self.CLOSED
+
+    @property
+    def approval_required(self):
+        return self.status == self.APPROVAL_REQUIRED
+
+
+class TreeLog(object):
+    """Represents a change in a tree's status."""
+    def __init__(self, d):
+        self.reason = d['reason'] or None
+        self.tags = set(d['tags']) if d['tags'] else set()
+        self.tree = d['tree']
+        self.who = d['who']
+        # FUTURE return a datetime with appropriate timezone info set.
+        self.when = strptime(d['when'], '%Y-%m-%dT%H:%M:%S')
+
+
+class TreeStatusClient(object):
+    """Client to the Mozilla Tree Status API.
+
+    The tree status API controls whether Mozilla's main source repositories are
+    open or closed.
+    """
+
+    def __init__(self, base_uri='https://treestatus.mozilla.org/', opener=None):
+        self._base_uri = base_uri
+
+        if opener is None:
+            opener = urllib2.build_opener()
+
+        self._opener = opener
+
+    def _request(self, path):
+        request = urllib2.Request('%s%s' % (self._base_uri, path), None)
+        response = self._opener.open(request)
+        return json.load(response)
+
+    def all(self):
+        """Obtain the status of all trees.
+
+        Returns a dict of tree names to TreeStatus instances.
+        """
+
+        o = self._request('?format=json')
+        trees = {}
+        for k, v in o.items():
+            trees[k] = TreeStatus(v)
+
+        return trees
+
+    def tree_status(self, tree):
+        """Obtain the status of a single tree.
+
+        Returns a TreeStatus instance.
+        """
+        o = self._request('%s?format=json' % tree)
+        return TreeStatus(o)
+
+    def tree_logs(self, tree):
+        o = self._request('%s/logs?format=json' % tree)
+
+        for d in o['logs']:
+            yield TreeLog(d)
+