allocator working on command line
authorDustin J. Mitchell <dustin@mozilla.com>
Wed, 05 Jan 2011 18:59:31 -0600
changeset 1236 35710713fff28f08e5e9f9e3dea017df81fc2c34
child 1237 3ef745c1e482871c443d1aab654e70bd2792e74a
push id894
push userdmitchell@mozilla.com
push dateFri, 18 Mar 2011 22:21:21 +0000
allocator working on command line
lib/python/slavealloc/__init__.py
lib/python/slavealloc/application.py
lib/python/slavealloc/data/__init__.py
lib/python/slavealloc/data/engine.py
lib/python/slavealloc/data/model.py
lib/python/slavealloc/data/queries.py
lib/python/slavealloc/exceptions.py
lib/python/slavealloc/http.py
lib/python/slavealloc/logic/__init__.py
lib/python/slavealloc/logic/allocate.py
lib/python/slavealloc/logic/buildbottac.py
lib/python/slavealloc/scripts/__init__.py
lib/python/slavealloc/scripts/dbinit.py
lib/python/slavealloc/scripts/gettac.py
lib/python/slavealloc/scripts/main.py
lib/python/slavealloc/scripts/pools.py
lib/python/slavealloc/scripts/silos.py
lib/python/slavealloc/service.py
setup.py
slavealloc.tac
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/application.py
@@ -0,0 +1,13 @@
+from twisted.application import service, strports
+from slavealloc import service as sa_service, http
+
+class Allocator(service.MultiService):
+    def __init__(self, http_port, db_url):
+        service.MultiService.__init__(self)
+
+        self.allocator = sa_service.AllocatorService(db_url)
+        self.allocator.setServiceParent(self)
+
+        self.site = http.AllocatorSite(self.allocator)
+        self.httpservice = strports.service(http_port, self.site)
+        self.httpservice.setServiceParent(self)
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/data/engine.py
@@ -0,0 +1,14 @@
+import sqlalchemy as sa
+
+class NoDBError(Exception):
+    pass
+
+def add_data_arguments(parser):
+    parser.add_argument('-D', '--db', dest='dburl',
+            default='sqlite:///slavealloc.db', # temporary
+            help='SQLAlchemy database URL')
+
+def create_engine(args):
+    if not args.dburl:
+        raise NoDBError
+    return sa.create_engine(args.dburl)
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/data/model.py
@@ -0,0 +1,78 @@
+"""
+
+Data storage for the slave allocator
+
+"""
+
+import sqlalchemy as sa
+
+metadata = sa.MetaData()
+
+# basic definitions
+
+distros = sa.Table('distros', metadata,
+    sa.Column('distroid', sa.Integer, primary_key=True),
+    sa.Column('name', sa.Text),
+)
+
+datacenters = sa.Table('datacenters', metadata,
+    sa.Column('dcid', sa.Integer, primary_key=True),
+    sa.Column('name', sa.Text),
+)
+
+bitlengths = sa.Table('bitlengths', metadata,
+    sa.Column('bitsid', sa.Integer, primary_key=True),
+    sa.Column('name', sa.Text),
+)
+
+purposes = sa.Table('purposes', metadata,
+    sa.Column('purposeid', sa.Integer, primary_key=True),
+    sa.Column('name', sa.Text),
+)
+
+trustlevels = sa.Table('trustlevels', metadata,
+    sa.Column('trustid', sa.Integer, primary_key=True),
+    sa.Column('name', sa.Text),
+    sa.Column('order', sa.Integer), # higher is more restricted
+)
+
+environments = sa.Table('environments', metadata,
+    sa.Column('envid', sa.Integer, primary_key=True),
+    sa.Column('name', sa.Text),
+)
+
+# pools
+
+pools = sa.Table('pools', metadata,
+    sa.Column('poolid', sa.Integer, primary_key=True),
+    sa.Column('name', sa.Text),
+)
+
+# all slaves
+
+slaves = sa.Table('slaves', metadata,
+    sa.Column('slaveid', sa.Integer, primary_key=True),
+    sa.Column('name', sa.Text),
+    sa.Column('distroid', sa.Integer, sa.ForeignKey('distros.distroid')),
+    sa.Column('bitsid', sa.Integer, sa.ForeignKey('bitlengths.bitsid')),
+    sa.Column('purposeid', sa.Integer, sa.ForeignKey('purposes.purposeid')),
+    sa.Column('dcid', sa.Integer, sa.ForeignKey('datacenters.dcid')),
+    sa.Column('trustid', sa.Integer, sa.ForeignKey('trustlevels.trustid')),
+    sa.Column('envid', sa.Integer, sa.ForeignKey('environments.envid')),
+    sa.Column('poolid', sa.Integer, sa.ForeignKey('pools.poolid')),
+    sa.Column('current_masterid', sa.Integer, sa.ForeignKey('masters.masterid')),
+)
+
+# masters
+
+masters = sa.Table('masters', metadata,
+    sa.Column('masterid', sa.Integer, primary_key=True),
+    sa.Column('nickname', sa.Text),
+    sa.Column('fqdn', sa.Text),
+    sa.Column('http_port', sa.Integer),
+    sa.Column('pb_port', sa.Integer),
+    sa.Column('dcid', sa.Integer, sa.ForeignKey('datacenters.dcid')),
+    sa.Column('poolid', sa.Integer, sa.ForeignKey('pools.poolid')),
+)
+
+# TODO: think about what kinds of indices are best: aggregate? individual?
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/data/queries.py
@@ -0,0 +1,87 @@
+import sqlalchemy as sa
+from slavealloc.data import model
+
+denormalized_slaves = sa.select([
+            model.slaves.c.name.label('name'),
+            model.distros.c.name.label('distro'),
+            model.bitlengths.c.name.label('bitlength'),
+            model.purposes.c.name.label('purpose'),
+            model.datacenters.c.name.label('datacenter'),
+            model.trustlevels.c.name.label('trustlevel'),
+            model.environments.c.name.label('environment'),
+            model.pools.c.name.label('pool'),
+        ], whereclause=(
+            (model.distros.c.distroid == model.slaves.c.distroid) &
+            (model.bitlengths.c.bitsid == model.slaves.c.bitsid) &
+            (model.purposes.c.purposeid == model.slaves.c.purposeid) &
+            (model.datacenters.c.dcid == model.slaves.c.dcid) &
+            (model.trustlevels.c.trustid == model.slaves.c.trustid) &
+            (model.environments.c.envid == model.slaves.c.envid) &
+            (model.pools.c.poolid == model.slaves.c.poolid)
+        ))
+
+denormalized_masters = sa.select([
+            model.masters.c.nickname,
+            model.datacenters.c.name.label('datacenter'),
+            model.pools.c.name.label('pool'),
+        ], whereclause=(
+            (model.datacenters.c.dcid == model.masters.c.dcid) &
+            (model.pools.c.poolid == model.masters.c.poolid)
+        ))
+
+# this is one query, but it's easier to build it in pieces.  Use bind parameter
+# 'slavename' to specify the new slave.  There will be exactly one result row,
+# unless there are no masters for this slave.  That result row is from the
+# masters table
+def best_master():
+    # we need two copies of slaves: one to find this slave's info, and one
+    # enumerating its peers
+    me = model.slaves.alias('me')
+    peers = model.slaves.alias('peers')
+
+    # all of the peers of this slave, *not* including the slave itself
+    me_and_peers = me.join(peers, onclause=(
+        # include only slaves matching our pool
+        (me.c.poolid == peers.c.poolid) &
+        # .. and matching our silo
+        (me.c.distroid == peers.c.distroid) &
+        (me.c.bitsid == peers.c.bitsid) &
+        (me.c.purposeid == peers.c.purposeid) &
+        (me.c.dcid == peers.c.dcid) &
+        (me.c.trustid == peers.c.trustid) &
+        (me.c.envid == peers.c.envid) &
+        # .. but do not include the slave itself
+        (me.c.slaveid != peers.c.slaveid)
+    ))
+
+    # find the peers' masters
+    peers_masters = me_and_peers.join(model.masters, onclause=(
+        (peers.c.current_masterid == model.masters.c.masterid)
+    ))
+
+    # query all masterids with nonzero counts
+    nonzero_mastercounts = sa.select(
+         [ model.masters.c.masterid, sa.func.count().label('slavecount') ],
+         from_obj=peers_masters,
+         whereclause=(me.c.name == sa.bindparam('slavename')),
+         group_by=[model.masters.c.masterid],
+    ).alias('nonzero_mastercounts')
+
+    # and join that as a subquery to get matching masters with no slaves
+    pool_masters = model.slaves.join(model.masters,
+            onclause=(model.slaves.c.poolid == model.masters.c.poolid))
+    joined_masters = pool_masters.outerjoin(nonzero_mastercounts,
+            onclause=(model.masters.c.masterid == nonzero_mastercounts.c.masterid))
+
+    # the slave counts in joined_masters are None where we'd like 0, so fix that
+    numeric_slavecount = sa.case([(nonzero_mastercounts.c.slavecount == None, 0)],
+                            else_=nonzero_mastercounts.c.slavecount)
+    numeric_slavecount = numeric_slavecount.label('numeric_slavecount')
+
+    best_master = sa.select([ model.masters ],
+            from_obj=joined_masters,
+            whereclause=(model.slaves.c.name == sa.bindparam('slavename')),
+            order_by=[numeric_slavecount,model.masters.c.masterid],
+            limit=1)
+    return best_master
+best_master = best_master()
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/exceptions.py
@@ -0,0 +1,2 @@
+class NoAllocationError(Exception):
+    "base class for errors that should result in a 404"
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/http.py
@@ -0,0 +1,50 @@
+from twisted.internet import defer
+from twisted.python import log
+from twisted.web import resource, server, error
+
+class TacResource(resource.Resource):
+    "dynamically created resource for a particular slave's buildbot.tac"
+    isLeaf = True
+
+    def __init__(self, slave_name):
+        resource.Resource.__init__(self)
+        self.slave_name = slave_name
+
+    def render_GET(self, request):
+        allocator = request.site.allocator
+        d = defer.succeed(None)
+
+        d.addCallback(lambda _ : allocator.getBuildbotTac(self.slave_name))
+
+        def handle_success(buildbot_tac):
+            request.setHeader('content-type', 'text/plain')
+            request.write(buildbot_tac)
+            request.finish()
+
+        def handle_error(f):
+            log.err(f, "while handling request for '%s'" % self.slave_name)
+            request.setResponseCode(500)
+            request.setHeader('content-type', 'text/plain')
+            request.write('error processing request: %s' % f.getErrorMessage())
+            request.finish()
+
+        d.addCallbacks(handle_success, handle_error)
+
+        # render_GET does not know how to wait for a Deferred, so we return
+        # NOT_DONE_YET which has a similar effect
+        return server.NOT_DONE_YET
+
+
+class RootResource(resource.Resource):
+    "root (/) resource for the HTTP service"
+    addSlash = True
+    isLeaf = False
+
+    def getChild(self, name, request):
+        return TacResource(name)
+
+
+class AllocatorSite(server.Site):
+    def __init__(self, allocator):
+        server.Site.__init__(self, RootResource())
+        self.allocator = allocator
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/logic/allocate.py
@@ -0,0 +1,23 @@
+from slavealloc import exceptions
+from slavealloc.data import queries, model
+
+def get_allocation(eng, slavename):
+    """
+    Return the C{masters} row for the master to which C{slavename}
+    should be assigned.
+    """
+    q = queries.best_master
+    q.bind = eng
+    allocation = q.execute(slavename=slavename).fetchone()
+    if not allocation:
+        raise exceptions.NoAllocationError
+    return allocation
+
+def allocate(eng, slavename, allocation):
+    """
+    Allocate C{slavename} to the master given by C{allocation} (as returned by
+    get_allocation).
+    """
+    q = model.slaves.update(whereclause=(model.slaves.c.name == slavename),
+                        values=dict(current_masterid=allocation.masterid))
+    eng.execute(q)
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/logic/buildbottac.py
@@ -0,0 +1,44 @@
+import time
+import socket
+
+tac_template = """\
+# AUTOMATICALLY GENERATED - DO NOT MODIFY
+# generated: %(gendate)s on %(genhost)s
+from twisted.application import service
+from buildbot.slave.bot import BuildSlave
+from twisted.python.logfile import LogFile
+from twisted.python.log import ILogObserver, FileLogObserver
+
+maxdelay = 300
+buildmaster_host = %(buildmaster_host)r
+passwd = %(passwd)r
+maxRotatedFiles = None
+basedir = %(basedir)r
+umask = 002
+slavename = %(slavename)r
+usepty = 1
+rotateLength = 1000000
+port = %(port)r
+keepalive = None
+
+application = service.Application('buildslave')
+logfile = LogFile.fromFullPath("twistd.log", rotateLength=rotateLength,
+                             maxRotatedFiles=maxRotatedFiles)
+application.setComponent(ILogObserver, FileLogObserver(logfile).emit)
+s = BuildSlave(buildmaster_host, port, slavename, passwd, basedir,
+               keepalive, usepty, umask=umask, maxdelay=maxdelay)
+s.setServiceParent(application)
+"""
+
+def make_buildbot_tac(engine, slavename, allocation):
+    info = dict()
+
+    info['gendate'] = time.ctime()
+    info['genhost'] = socket.getfqdn()
+    info['buildmaster_host'] = allocation.fqdn
+    info['port'] = allocation.pb_port
+    info['slavename'] = slavename
+    info['basedir'] = 'TODO' # TODO!!
+    info['passwd'] = 'TODO' # TODO!!
+
+    return tac_template % info
new file mode 100644
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/scripts/dbinit.py
@@ -0,0 +1,76 @@
+import sys
+import csv
+from slavealloc.data import engine, model
+
+def setup_argparse(subparsers):
+    subparser = subparsers.add_parser('dbinit', help='initialize a fresh database')
+
+    subparser.add_argument('--slave-data', dest='slave_data',
+            help="""csv of slave data to import (columns: name, distro,
+            bitlength, purpose, size, datacenter, trustlevel, and
+            environment)""")
+
+    subparser.add_argument('--master-data', dest='master_data',
+            help="""csv of master data to import (columns: nickname, fqdn,
+            http_port, pb_port, and pool)""")
+
+    return subparser
+
+def process_args(subparser, args):
+    if not args.master_data or not args.slave_data:
+        subparser.error("--master-data and --slave-data are both required")
+
+def main(args):
+    eng = engine.create_engine(args)
+    model.metadata.bind = eng
+    model.metadata.drop_all()
+    model.metadata.create_all()
+
+    rdr = csv.DictReader(open(args.slave_data))
+    slaves = list(rdr)
+
+    rdr = csv.DictReader(open(args.master_data))
+    masters = list(rdr)
+
+    def normalize(table, idcolumn, values):
+        values = list(enumerate(list(set(values)))) # remove duplicates, add ids
+        table.insert().execute([ { 'name' : n, idcolumn : i }
+                                 for i, n in values ])
+        return dict((n,i) for i, n in values)
+
+    distros = normalize(model.distros, 'distroid',
+            [ r['distro'] for r in slaves ])
+    bitlengths = normalize(model.bitlengths, 'bitsid',
+            [ r['bitlength'] for r in slaves ])
+    purposes = normalize(model.purposes, 'purposeid',
+            [ r['purpose'] for r in slaves ])
+    datacenters = normalize(model.datacenters, 'dcid',
+            [ r['datacenter'] for r in slaves ] +
+            [ r['datacenter'] for r in masters ])
+    trustlevels = normalize(model.trustlevels, 'trustid',
+            [ r['trustlevel'] for r in slaves ])
+    environments = normalize(model.environments, 'envid',
+            [ r['environment'] for r in slaves ])
+    pools = normalize(model.pools, 'poolid',
+            [ r['pool'] for r in masters ])
+
+    model.masters.insert().execute([
+        dict(nickname=row['nickname'],
+             fqdn=row['fqdn'],
+             http_port=int(row['http_port']),
+             pb_port=int(row['pb_port']),
+             dcid=datacenters[row['datacenter']],
+             poolid=pools[row['pool']])
+        for row in masters ])
+
+    model.slaves.insert().execute([
+        dict(name=row['name'],
+             distroid=distros[row['distro']],
+             bitsid=bitlengths[row['bitlength']],
+             purposeid=purposes[row['purpose']],
+             dcid=datacenters[row['datacenter']],
+             trustid=trustlevels[row['trustlevel']],
+             envid=environments[row['environment']],
+             poolid=pools[row['pool']],
+             current_masterid=None)
+        for row in slaves ])
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/scripts/gettac.py
@@ -0,0 +1,40 @@
+import sys
+from slavealloc import exceptions
+from slavealloc.data import engine
+from slavealloc.logic import allocate, buildbottac
+
+def setup_argparse(subparsers):
+    subparser = subparsers.add_parser('gettac', help='get a tac file for a slave')
+    subparser.add_argument('slave', nargs='*',
+            help="slave hostnames to allocate for (no domain)")
+    subparser.add_argument('-n', '--noop', dest='noop',
+            default=False, action='store_true',
+            help="don't actually allocate")
+    subparser.add_argument('-q', '--quiet', dest='quiet',
+            default=False, action='store_true',
+            help="don't actually output the tac file; just the allocation made")
+    return subparser
+
+def process_args(subparser, args):
+    if not args.slave:
+        subparser.error("at least one slave name is required")
+    if '.' in ''.join(args.slave):
+        subparser.error("slave name must not contain '.'; give the unqualified hostname")
+
+def main(args):
+    eng = engine.create_engine(args)
+
+    for slave in args.slave:
+        try:
+            allocation = allocate.get_allocation(eng, slave)
+        except exceptions.NoAllocationError:
+            print >>sys.stderr, "No buildbot.tac available (404 from slave allocator)"
+            sys.exit(1)
+
+        if not args.quiet:
+            print buildbottac.make_buildbot_tac(eng, slave, allocation)
+
+        if not args.noop:
+            allocate.allocate(eng, slave, allocation)
+        print >>sys.stderr, "Allocated '%s' to '%s' (%s:%s)" % (slave,
+                allocation.nickname, allocation.fqdn, allocation.pb_port)
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/scripts/main.py
@@ -0,0 +1,39 @@
+import sys
+import argparse
+import textwrap
+
+from slavealloc.data import engine
+
+# subcommands
+from slavealloc.scripts import silos, dbinit, pools, gettac
+subcommands = [ silos, dbinit, pools, gettac ]
+
+def parse_options():
+    parser = argparse.ArgumentParser(description="Runs slavealloc subcommands")
+    parser.set_defaults(_module=None)
+
+    engine.add_data_arguments(parser)
+
+    subparsers = parser.add_subparsers(title='subcommands')
+
+    for module in subcommands:
+        subparser = module.setup_argparse(subparsers)
+        subparser.set_defaults(module=module, subparser=subparser)
+
+    args = parser.parse_args()
+
+    if not args.module:
+        parser.error("No subcommand specified")
+
+    args.module.process_args(args.subparser, args)
+
+    return args.module.main, args
+
+def main():
+    func, args = parse_options()
+    try:
+        func(args)
+    except engine.NoDBError:
+        print >>sys.stderr, "No database specified (use --db)"
+        sys.exit(1)
+
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/scripts/pools.py
@@ -0,0 +1,62 @@
+import sys
+import csv
+import collections
+import sqlalchemy as sa
+from slavealloc.data import engine, model, queries
+
+pools_keys = 'pool nmasters nslaves masters'.split()
+
+def setup_argparse(subparsers):
+    subparser = subparsers.add_parser('pools', help='show master pools')
+    subparser.add_argument('-c', '--columns', dest='columns',
+            help='comma-separated list of columns to show',
+            default=','.join(pools_keys))
+    return subparser
+
+def process_args(subparser, args):
+    args.columns = [ c.strip() for c in args.columns.split(',') ]
+
+    unrecognized_columns = set(args.columns) - set(pools_keys)
+    if unrecognized_columns:
+        subparser.error("unrecognized columns: %s" % (" ".join(list(unrecognized_columns)),))
+
+def main(args):
+    eng = engine.create_engine(args)
+
+    pools = collections.defaultdict(lambda:
+            dict(masters=[], nmasters=0, nslaves=0))
+
+    col_titles = args.columns
+
+    if 'nmasters' in col_titles or 'masters' in col_titles:
+        q = queries.denormalized_masters
+        q.bind = eng
+        for master in q.execute():
+            p = pools[master.pool]
+            p['nmasters'] += 1
+            p['masters'].append(master.nickname)
+
+    if 'nslaves' in col_titles:
+        q = queries.denormalized_slaves
+        q.bind = eng
+        for slave in q.execute():
+            p = pools[slave.pool]
+            p['nslaves'] += 1
+
+    datagrid = [col_titles]
+    for name, pooldict in sorted(pools.items()):
+        row = []
+        for col in col_titles:
+            if col == 'pool':
+                row.append(name)
+            elif col in ('nmasters', 'nslaves'):
+                row.append(str(pooldict[col]))
+            elif col == 'masters':
+                row.append(' '.join(pooldict[col]))
+        datagrid.append(row)
+
+    fmtmsg = " ".join([
+        "%%-%ds" % max(len(r[i]) for r in datagrid)
+        for i in range(len(col_titles)) ])
+    for row in datagrid:
+        print fmtmsg % tuple(row)
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/scripts/silos.py
@@ -0,0 +1,82 @@
+import sys
+import csv
+import collections
+import sqlalchemy as sa
+from slavealloc.data import engine, model, queries
+
+silo_keys = 'environment purpose distro bitlength datacenter trustlevel'.split()
+
+def setup_argparse(subparsers):
+    subparser = subparsers.add_parser('silos', help='show slave silos with counts')
+    subparser.add_argument('-c', '--columns', dest='columns',
+            help='comma-separated list of columns to show',
+            default=','.join(silo_keys))
+    subparser.add_argument('-s', '--sort', dest='sort',
+            help='comma-separated list of columns to sort on; defaluts to COLUMNS. ' +
+                 'note that \'count\' is a valid sort key, too',
+            default=None)
+    subparser.add_argument('--csv', dest='csv', default=False, action='store_true',
+            help='output in CSV format')
+    return subparser
+
+def process_args(subparser, args):
+    args.columns = [ c.strip() for c in args.columns.split(',') ]
+
+    unrecognized_columns = set(args.columns) - set(silo_keys)
+    if unrecognized_columns:
+        subparser.error("unrecognized columns: %s" % (" ".join(list(unrecognized_columns)),))
+
+    if args.sort is None:
+        args.sort = args.columns[:]
+    else:
+        args.sort = [ c.strip() for c in args.sort.split(',') ]
+
+    unrecognized_keys = set(args.sort) - set(args.columns) - set(['count'])
+    if unrecognized_keys:
+        subparser.error("sort keys not available: %s" % (" ".join(list(unrecognized_keys)),))
+
+def main(args):
+    silos = collections.defaultdict(lambda : 0)
+
+    eng = engine.create_engine(args)
+
+    # get the denormalized slave data
+    q = queries.denormalized_slaves
+    q.bind = eng
+    slaves = [ r for r in q.execute() ]
+
+    # count the slaves into silos, using a key composed of the desired
+    # columns.
+    for slave in slaves:
+        k = tuple(slave[c] for c in args.columns)
+        silos[k] += 1
+
+    # add the count to the end of 'columns' and 'silos'
+    col_titles = args.columns + ['count']
+    rows = [ k + (count,) for k, count in silos.iteritems() ]
+
+    # sort the rows
+    sortidxes = [ col_titles.index(col) for col in args.sort ]
+    def keyfunc(row):
+        return [ row[i] for i in sortidxes ]
+    rows.sort(key=keyfunc)
+
+    # shorten up some long names
+    for long, short in [ ('datacenter', 'dc'), ('bitlength', 'bits') ]:
+        if long in col_titles:
+            col_titles[args.columns.index(long)] = short
+
+    if args.csv:
+        writer = csv.writer(sys.stdout)
+        writer.writerow(col_titles)
+        for row in rows:
+            writer.writerow(row)
+    else:
+        # calculate an appropriate format for the observed lengths
+        lengths = [ max([ len(col_titles[i]) ] + [ len(str(row[i])) for row in rows ])
+                    for i in xrange(len(col_titles)) ]
+        fmtmsg = " ".join(["%%-%ds" % l for l in lengths])
+
+        print fmtmsg % tuple(col_titles)
+        for row in rows:
+            print fmtmsg % row
new file mode 100644
--- /dev/null
+++ b/lib/python/slavealloc/service.py
@@ -0,0 +1,19 @@
+from twisted.python import log
+from twisted.internet import defer
+from twisted.application import service, strports
+
+class AllocatorService(service.Service):
+    
+    def __init__(self, db_url):
+        self.db_url = db_url
+
+    def startService(self):
+        log.msg("starting AllocatorService with db_url='%s'" % self.db_url)
+        service.Service.startService(self)
+
+    def stopService(self):
+        log.msg("stopping AllocatorService")
+        return service.Service.stopService(self)
+
+    def getBuildbotTac(self, slave_name):
+        return defer.succeed("fake buildbot.tac")
new file mode 100755
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,28 @@
+#! /usr/bin/env python
+
+from setuptools import setup, find_packages
+
+setup(
+    name="slavealloc",
+    version="1.0",
+    description="Mozilla RelEng Slave Allocator",
+    author = "Release Engineers",
+    author_email = "release@mozilla.com",
+
+    # python packages are under lib/python
+    packages = find_packages(),
+
+    test_suite = 'slavealloc',
+
+    install_requires = [
+        'sqlalchemy',
+        'argparse',
+        'twisted',
+    ],
+
+    entry_points = {
+        'console_scripts': [
+            'slavealloc = slavealloc.scripts.main:main'
+        ],
+    }
+)
new file mode 100644
--- /dev/null
+++ b/slavealloc.tac
@@ -0,0 +1,6 @@
+from twisted.application import service
+from slavealloc.application import Allocator
+
+application = service.Application("slavealloc")
+allocator = Allocator(http_port='tcp:8010', db_url='sqlite:////tmp/test.db')
+allocator.setServiceParent(application)