Import Buildbot-v0.8.4-pre-150-g7175a0d source upstream BUILDBOT_0_8_4_PRE_150_G715A0D
authorDustin J. Mitchell <dustin@mozilla.com>
Tue, 08 Feb 2011 18:28:49 -0600
branchupstream
changeset 141 037a0c5ab007607e7d368f2e40128566dc062862
parent 140 e3ac2bc4e0b5baa5aa7f6bd4869d07f3025c60e0
child 142 276c0e45556944193edb3a25f3e5209328e37456
push id59
push userdmitchell@mozilla.com
push dateTue, 15 Feb 2011 21:49:58 +0000
Import Buildbot-v0.8.4-pre-150-g7175a0d source
.coveragerc
MAINTAINERS.txt
common/coveragerc
common/gcode-upload.sh
master/MANIFEST.in
master/NEWS
master/buildbot/buildslave.py
master/buildbot/changes/base.py
master/buildbot/changes/bonsaipoller.py
master/buildbot/changes/changes.py
master/buildbot/changes/filter.py
master/buildbot/changes/gerritchangesource.py
master/buildbot/changes/gitpoller.py
master/buildbot/changes/hgbuildbot.py
master/buildbot/changes/mail.py
master/buildbot/changes/maildir.py
master/buildbot/changes/manager.py
master/buildbot/changes/p4poller.py
master/buildbot/changes/pb.py
master/buildbot/changes/svnpoller.py
master/buildbot/clients/gtkPanes.py
master/buildbot/clients/sendchange.py
master/buildbot/db/base.py
master/buildbot/db/buildsets.py
master/buildbot/db/changes.py
master/buildbot/db/connector.py
master/buildbot/db/dbspec.py
master/buildbot/db/enginestrategy.py
master/buildbot/db/migrate/README
master/buildbot/db/migrate/migrate.cfg
master/buildbot/db/migrate/versions/001_initial.py
master/buildbot/db/migrate/versions/002_add_proj_repo.py
master/buildbot/db/migrate/versions/003_scheduler_class_name.py
master/buildbot/db/migrate/versions/004_add_autoincrement.py
master/buildbot/db/migrate/versions/005_add_indexes.py
master/buildbot/db/migrate/versions/006_drop_last_access.py
master/buildbot/db/migrate/versions/__init__.py
master/buildbot/db/model.py
master/buildbot/db/pool.py
master/buildbot/db/schedulers.py
master/buildbot/db/schema/__init__.py
master/buildbot/db/schema/base.py
master/buildbot/db/schema/manager.py
master/buildbot/db/schema/tables.sql
master/buildbot/db/schema/v1.py
master/buildbot/db/schema/v2.py
master/buildbot/db/schema/v3.py
master/buildbot/db/schema/v4.py
master/buildbot/db/schema/v5.py
master/buildbot/db/schema/v6.py
master/buildbot/db/sourcestamps.py
master/buildbot/db/util.py
master/buildbot/interfaces.py
master/buildbot/master.py
master/buildbot/process/builder.py
master/buildbot/process/buildstep.py
master/buildbot/process/process_twisted.py
master/buildbot/process/properties.py
master/buildbot/schedulers/base.py
master/buildbot/schedulers/filter.py
master/buildbot/schedulers/trysched.py
master/buildbot/scripts/checkconfig.py
master/buildbot/scripts/runner.py
master/buildbot/scripts/sample.cfg
master/buildbot/scripts/startup.py
master/buildbot/status/builder.py
master/buildbot/status/mail.py
master/buildbot/status/web/change_hook.py
master/buildbot/status/web/changes.py
master/buildbot/status/web/console.py
master/buildbot/status/web/files/default.css
master/buildbot/status/web/hooks/base.py
master/buildbot/status/web/hooks/github.py
master/buildbot/status/web/status_json.py
master/buildbot/status/words.py
master/buildbot/steps/master.py
master/buildbot/steps/python.py
master/buildbot/steps/source.py
master/buildbot/steps/transfer.py
master/buildbot/steps/vstudio.py
master/buildbot/test/fake/fakedb.py
master/buildbot/test/fake/web.py
master/buildbot/test/integration/README.txt
master/buildbot/test/integration/__init__.py
master/buildbot/test/integration/citools-README.txt
master/buildbot/test/integration/citools.tgz
master/buildbot/test/integration/master-0-7-5-README.txt
master/buildbot/test/integration/master-0-7-5.tgz
master/buildbot/test/integration/test_upgrade.py
master/buildbot/test/regressions/test_change_properties.py
master/buildbot/test/regressions/test_import_unicode_changes.py
master/buildbot/test/regressions/test_import_weird_changes.py
master/buildbot/test/test_extra_coverage.py
master/buildbot/test/unit/test_changes_base.py
master/buildbot/test/unit/test_changes_bonsaipoller.py
master/buildbot/test/unit/test_changes_filter.py
master/buildbot/test/unit/test_changes_gerritchangesource.py
master/buildbot/test/unit/test_changes_gitpoller.py
master/buildbot/test/unit/test_changes_mail.py
master/buildbot/test/unit/test_changes_mail_CVSMaildirSource.py
master/buildbot/test/unit/test_changes_manager.py
master/buildbot/test/unit/test_changes_p4poller.py
master/buildbot/test/unit/test_changes_pb.py
master/buildbot/test/unit/test_changes_svnpoller.py
master/buildbot/test/unit/test_contrib_buildbot_cvs_mail.py
master/buildbot/test/unit/test_db_buildsets.py
master/buildbot/test/unit/test_db_changes.py
master/buildbot/test/unit/test_db_connector.py
master/buildbot/test/unit/test_db_dbspec.py
master/buildbot/test/unit/test_db_enginestrategy.py
master/buildbot/test/unit/test_db_model.py
master/buildbot/test/unit/test_db_pool.py
master/buildbot/test/unit/test_db_schedulers.py
master/buildbot/test/unit/test_db_schema_master.py
master/buildbot/test/unit/test_db_sourcestamps.py
master/buildbot/test/unit/test_db_util.py
master/buildbot/test/unit/test_master.py
master/buildbot/test/unit/test_oldpaths.py
master/buildbot/test/unit/test_persistent_queue.py
master/buildbot/test/unit/test_process_properties.py
master/buildbot/test/unit/test_schedulers_filter.py
master/buildbot/test/unit/test_status_builder.py
master/buildbot/test/unit/test_status_web_change_hook.py
master/buildbot/test/unit/test_status_web_change_hooks_github.py
master/buildbot/test/unit/test_util.py
master/buildbot/test/unit/test_util_maildir.py
master/buildbot/test/unit/test_util_netstrings.py
master/buildbot/test/unit/test_util_subscriptions.py
master/buildbot/test/util/change_import.py
master/buildbot/test/util/changesource.py
master/buildbot/test/util/compat.py
master/buildbot/test/util/connector_component.py
master/buildbot/test/util/db.py
master/buildbot/test/util/dirs.py
master/buildbot/test/util/gpo.py
master/buildbot/test/util/threads.py
master/buildbot/util/__init__.py
master/buildbot/util/loop.py
master/buildbot/util/maildir.py
master/buildbot/util/misc.py
master/buildbot/util/netstrings.py
master/buildbot/util/subscription.py
master/contrib/bzr_buildbot.py
master/contrib/git_buildbot.py
master/docs/cfg-buildslaves.texinfo
master/docs/cfg-buildsteps.texinfo
master/docs/cfg-changesources.texinfo
master/docs/cfg-global.texinfo
master/docs/cfg-schedulers.texinfo
master/docs/cfg-statustargets.texinfo
master/docs/concepts.texinfo
master/docs/cust-changesources.texinfo
master/docs/developer.texinfo
master/docs/installation.texinfo
master/setup.py
slave/NEWS
slave/buildslave/commands/base.py
slave/buildslave/commands/git.py
slave/buildslave/commands/shell.py
slave/buildslave/commands/utils.py
slave/buildslave/runprocess.py
slave/buildslave/test/fake/runprocess.py
slave/buildslave/test/test_extra_coverage.py
slave/buildslave/test/unit/runprocess-scripts.py
slave/buildslave/test/unit/test_runprocess.py
slave/buildslave/test/util/compat.py
slave/setup.py
old mode 100644
new mode 120000
--- a/.coveragerc
+++ b/.coveragerc
@@ -1,31 +1,1 @@
-[report]
-# Regexes for lines to exclude from consideration
-exclude_lines =
-    # Have to re-enable the standard pragma
-    pragma: no cover
-
-    # Don't complain about missing debug-only code:
-    def __repr__
-    if self\.debug
-
-    # Don't complain if tests don't hit defensive assertion code:
-    raise AssertionError
-    raise NotImplementedError
-
-    # Don't complain if non-runnable code isn't run:
-    if 0:
-    if __name__ == .__main__.:
-    if runtime.platformType  == 'win32'
-
-    # 'pass' generally means 'this won't be called'
-    ^ *pass *$
-
-include =
-    master/*
-    slave/*
-
-omit =
-    # omit all of our tests
-    */test/*
-    # templates cause coverage errors
-    */templates/*
+common/coveragerc
\ No newline at end of file
new file mode 100644
--- /dev/null
+++ b/MAINTAINERS.txt
@@ -0,0 +1,204 @@
+Buildbot is too complex for any one person to understand all of it.  This file
+lists who's responsible for which parts of Buildbot.  Some of these are easily
+separated into a few source files, while others are general projects that
+cover the entire Buildbot codebase.
+
+  See http://trac.buildbot.net/wiki/ComponentMaintenancePolicy for more.
+
+Each component contains indented lines identified as follows:
+
+D: Component description (optional)
+
+U: URL for more information
+
+M: Component maintainer, in the form
+      FullName <address@domain.com>  - STATUS
+   where STATUS is optional, and can be one of
+      Hacker - knows the code and can write fixes
+      Tester - can test patches and advise committers to merge
+      User - can help to triage and explain bugs
+      Retired - former maintainer, now out of the business
+
+S: Component status.  One of
+    Supported: This will be a part of Buildbot forever
+    Maintained: Someone is actively looking after this component
+    Patched: Someone is watching bugs for this component and can merge
+        fixes
+    Orphaned: This component is not actively maintained and may be removed
+        from Buildbot if it accumulates too many bugs
+    Last-Rites: This component is slated to be removed in the given
+        release, unless a maintainer steps forward
+
+---
+
+== Version Control ==
+
+Perforce VC
+    S: Orphaned
+    U: http://trac.buildbot.net/wiki/p4
+
+Subversion VC
+    S: Orphaned
+    U: http://trac.buildbot.net/wiki/svn
+
+CVS VC
+    S: Orphaned
+    U: http://trac.buildbot.net/wiki/cvs
+
+Bazaar VC
+    S: Orphaned
+    U: http://trac.buildbot.net/wiki/bzr
+
+Git VC
+    S: Maintained
+    M: Amber Yust <ayust@yelp.com> - Hacker
+    U: http://trac.buildbot.net/wiki/git
+
+BitKeeper VC
+    S: Patched
+    M: Amar Takhar <verm@snickers.org> - Hacker
+    U: http://trac.buildbot.net/wiki/bk
+
+Darcs VC
+    S: Orphaned
+    U: http://trac.buildbot.net/wiki/darcs
+
+Mercurial VC
+    S: Orphaned
+    U: http://trac.buildbot.net/wiki/hg
+
+Repo VC
+    S: Maintained
+    M: Chris Soyars <ctsoyars@gmail.com> - Tester
+    M: Piotr Sikora <piotr.sikora@frickle.com> - Hacker
+    M: Pierre Tardy <tardyp@gmail.com> - Hacker
+    U: http://trac.buildbot.net/wiki/repovc
+
+
+== Change Sources ==
+
+Gerrit
+    S: Maintained
+    M: Piotr Sikora <piotr.sikora@frickle.com> - Hacker
+    M: Pierre Tardy <tardyp@gmail.com> - Hacker
+    U: http://trac.buildbot.net/wiki/gerrit
+
+
+== Buildbot Plugins ==
+
+Debug Client
+    S: Orphaned
+
+Status Client
+    S: Patched
+    M: Mark Wielaard <mark@klomp.org>
+
+Debug Client
+    S: Orphaned
+
+Web Status
+    S: Supported
+    M: Marcus Lindblom <macke@yar.nu> - Hacker
+    U: http://trac.buildbot.net/wiki/web
+
+IRC Status
+    S: Maintained
+    M: Amber Yust <ayust@yelp.com> - Hacker
+    U: http://trac.buildbot.net/wiki/irc
+
+MailNotifier
+    S: Orphaned
+    U: http://trac.buildbot.net/wiki/mail
+
+
+== Buildbot Core Features ==
+
+Virtualization
+    D: Use of virtual, start-on-demand slaves
+    S: Maintained
+    M: John Carr <john.carr@isotoma.com> - Hacker
+    M: Tom Wardill <tom@howrandom.net> - Hacker
+    U: http://trac.buildbot.net/wiki/virtualization
+
+Buildbot Try
+    S: Orphaned
+    U: http://trac.buildbot.net/wiki/try
+
+Documentation
+    S: Supported
+    U: http://buildbot.net/buildbot/docs
+    U: http://trac.buildbot.net/wiki/docs
+
+== Packaging & System Compatibility ==
+
+MacPorts
+    S: Maintained
+    M: William Siegrist <wsiegrist@apple.com>
+    U: http://trac.macports.org/browser/trunk/dports/devel/buildbot/Portfile
+
+Fink
+    S: Maintained
+    M: Charles Lepple <clepple+fink@ghz.cc>
+    U: http://pdb.finkproject.org/pdb/browse.php?name=buildbot
+
+FreeBSD
+    S: Maintained
+    U: http://www.freebsd.org/cgi/cvsweb.cgi/ports/devel/buildbot/
+
+OpenBSD
+    S: Maintained
+    U: http://www.openbsd.org/cgi-bin/cvsweb/ports/devel/py-buildbot/
+
+Windows
+    S: Patched
+    M: Robert Stackhouse <robertstackhouse@gmail.com>
+    U: http://trac.buildbot.net/wiki/windows
+
+RedHat / Fedora
+    S: Maintained
+    M: Gareth Armstrong <gareth.armstrong@hp.com>
+    M: Gianluca Sforna <giallu@gmail.com>
+    M: Steve Milner <smilner@redhat.com>
+    U: https://admin.fedoraproject.org/pkgdb/acls/name/buildbot
+
+Debian / Ubuntu
+    S: Orphaned
+    U: http://packages.ubuntu.com/maverick/buildbot
+
+OpenCSW
+    S: Orphaned
+    U: http://www.opencsw.org/packages/CSWbuildbot/
+
+Gentoo
+    S: Maintained
+    M: Dustin J. Mitchell <dustin@v.igoros.us>
+    U: http://packages.gentoo.org/package/dev-util/buildbot
+
+
+== Other Contact Information ==
+
+Security
+    D: Contacts members of this team directly with any security concerns
+    M: Dustin J. Mitchell <dustin@v.igoros.us>
+    M: Amber Yust <ayust@yelp.com>
+    M: Steve Milner <smilner@redhat.com>
+    U: http://trac.buildbot.net/wiki/SecurityPolicy
+
+Metabuildbot Slave Donors
+    D: Maintainers of buildslaves for the Metabuildbot
+    M: Steve Milner <smilner@redhat.com>
+    M: Dustin J. Mitchell <dustin@v.igoros.us>
+    M: Mozilla Release Engineering <release@mozilla.com>
+    M: Marc-Antoine Ruel <maruel@chromium.org>
+    M: Dustin Sallings <dustin@spy.net>
+    U: http://buildbot.buildbot.net
+
+Committers
+    D: People who can commit changes to the main Buildbot repository
+    M: Dustin J. Mitchell <dustin@v.igoros.us>
+    M: Amber Yust <ayust@yelp.com>
+    M: Ben Hearsum <bhearsum@mozilla.com>
+    M: Brian Warner <warner@lothar.com>
+    M: Chris AtLee <catlee@mozilla.com>
+    M: Marc-Antoine Ruel <maruel@chromium.org>
+    M: Marcus Lindblom <macke@yar.nu>
new file mode 100644
--- /dev/null
+++ b/common/coveragerc
@@ -0,0 +1,34 @@
+[report]
+# Regexes for lines to exclude from consideration
+exclude_lines =
+    # Have to re-enable the standard pragma
+    pragma: no cover
+
+    # Don't complain about missing debug-only code:
+    def __repr__
+    if self\.debug
+
+    # Don't complain if tests don't hit defensive assertion code:
+    raise AssertionError
+    raise NotImplementedError
+
+    # Don't complain if non-runnable code isn't run:
+    if 0:
+    if __name__ == .__main__.:
+    if runtime.platformType  == 'win32'
+
+    # 'pass' generally means 'this won't be called'
+    ^ *pass *$
+
+    # conditionals on twisted versions aren't coverable
+    if twisted.version
+
+include =
+    master/*
+    slave/*
+
+omit =
+    # omit all of our tests
+    */test/*
+    # templates cause coverage errors
+    */templates/*
--- a/common/gcode-upload.sh
+++ b/common/gcode-upload.sh
@@ -48,16 +48,16 @@ i=0
 for file in {buildbot,buildbot-slave}-$VERSION.{tar.gz,zip}{,.asc}; do
     if test $i = 0; then
         i=1
         continue
     fi
     labels=`findlabels "$file"`
     file=`findfile "$file"`
     echo "Uploading $file with labels $labels"
-    python common/googlecode_upload.py \
-        -w $PASSWORD \
-        -u $USERNAME \
-        -p buildbot \
-        -s "$file" \
-        --labels=Featured \
-        "$file"
+        python common/googlecode_upload.py \
+            -w $PASSWORD \
+            -u $USERNAME \
+            -p buildbot \
+            -s `basename $file` \
+            --labels=$labels \
+            "$file"
 done
--- a/master/MANIFEST.in
+++ b/master/MANIFEST.in
@@ -3,18 +3,19 @@ include MANIFEST.in README NEWS CREDITS 
 include docs/examples/*.cfg
 include docs/*.texinfo
 include docs/*.png docs/images/*.png docs/images/*.svg docs/images/*.txt docs/images/*.ai
 include docs/images/icon.blend docs/images/Makefile
 include docs/Makefile
 include docs/version.py
 include docs/buildbot.1
 
-include buildbot/db/schema/tables.sql
 include buildbot/scripts/sample.cfg
 include buildbot/status/web/files/*
 include buildbot/status/web/templates/*.html buildbot/status/web/templates/*.xml
 include buildbot/clients/debug.glade
 include buildbot/buildbot.png
 
+include buildbot/db/migrate/README
+
 include contrib/* contrib/windows/* contrib/os-x/* contrib/css/*
 include contrib/trac/* contrib/trac/bbwatcher/* contrib/trac/bbwatcher/templates/*
 include contrib/init-scripts/*
--- a/master/NEWS
+++ b/master/NEWS
@@ -1,24 +1,51 @@
 Major User visible changes in Buildbot.             -*- outline -*-
    see the git log for a detailed list of changes:
    http://github.com/buildbot/buildbot/commits/master
 
-* Buildbot 0.8.3p1
-
-** Critical Fixes for GitPoller
-
-*** correctly initialize a new repository with 'git init' and 'git fetch',
-albiet using blocking calls (fixes #1742)
-
-*** correctly synchronize processing of each change in a large batch of changes
-(fixes #1745)
+
+* Next Version
+
+** Deprecations, Removals, and Non-Compatible Changes
+
+*** MasterShellCommand and all of the transfer steps now default to
+haltOnFailure=True and flunkOnFailure=True
+
+*** GitPoller's 'workdir' parameter should always be supplied; using the
+default (/tmp/gitpoller_work) is deprecated and will not be supported in future
+versions.
+
+*** ChangeFilter should now be imported from `buildbot.changes.filter'; the old
+import path will still work.
+
+** SQLAlchemy & SQLAlchemy-Migrate
+
+Buildbot now uses SQLAlchemy as a database abstraction layer.  This will give
+us greater inter-database compatibility and a more stable and reliable basis
+for this core component of the framework.  SQLAlchemy-Migrate is used to manage
+changes to the database schema from version to version.
+
+** Less garish color scheme
+
+The default color scheme for Buildbot has been modified to make it slightly
+less, well, neon. Note: This will not affect already-created masters, as
+their default.css file has already been created. If you currently use the
+default and want to get the new version, just overwrite public_html/default.css
+with the copy in this version.
 
 * Buildbot 0.8.3 (December 19, 2010)
 
+** Deprecations and Removals
+
+*** Change sources can no longer call change-related methods on self.parent.
+Instead, use self.master methods, e.g., self.master.addChange.
+
+* Next Release
+
 ** PBChangeSource now supports authentication
 
 PBChangeSource now supports the `user` and `passwd` arguments.  Users with a
 publicly exposed PB port should use these parameters to limit sendchange
 access.
 
 Previous versions of Buildbot should never be configured with a PBChangeSource
 and a publicly accessible slave port, as that arrangement  allows anyone to
--- a/master/buildbot/buildslave.py
+++ b/master/buildbot/buildslave.py
@@ -274,19 +274,19 @@ class AbstractBuildSlave(pb.Avatar, serv
             return d1
         d.addCallback(_get_info)
 
         def _get_version(res):
             d1 = bot.callRemote("getVersion")
             def _got_version(version):
                 state["version"] = version
             def _version_unavailable(why):
+                why.trap(pb.NoSuchMethod)
                 # probably an old slave
-                log.msg("BuildSlave.version_unavailable")
-                log.err(why)
+                state["version"] = '(unknown)'
             d1.addCallbacks(_got_version, _version_unavailable)
         d.addCallback(_get_version)
 
         def _get_commands(res):
             d1 = bot.callRemote("getCommands")
             def _got_commands(commands):
                 state["slave_commands"] = commands
             def _commands_unavailable(why):
@@ -516,19 +516,19 @@ class AbstractBuildSlave(pb.Avatar, serv
                     if why.check(pb.PBConnectionLost):
                         log.msg("Lost connection to %s" % self.slavename)
                     else:
                         log.err("Unexpected error when trying to shutdown %s" % self.slavename)
                 d.addErrback(_errback)
                 return d
             log.err("Couldn't find remote builder to shut down slave")
             return defer.succeed(None)
-        #wfd = defer.waitForDeferred(old_way())
-        #yield wfd
-        #wfd.getResult()
+        wfd = defer.waitForDeferred(old_way())
+        yield wfd
+        wfd.getResult()
 
     def maybeShutdown(self):
         """Shut down this slave if it has been asked to shut down gracefully,
         and has no active builders."""
         if not self.slave_status.getGraceful():
             return
         active_builders = [sb for sb in self.slavebuilders.values()
                            if sb.isBusy()]
--- a/master/buildbot/changes/base.py
+++ b/master/buildbot/changes/base.py
@@ -10,53 +10,59 @@
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc., 51
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
 
 from zope.interface import implements
 from twisted.application import service
-from twisted.internet import defer, task
+from twisted.internet import defer, task, reactor
 from twisted.python import log
 
 from buildbot.interfaces import IChangeSource
 from buildbot import util
 
 class ChangeSource(service.Service, util.ComparableMixin):
     implements(IChangeSource)
 
+    master = None
+    "if C{self.running} is true, then C{cs.master} points to the buildmaster."
+
     def describe(self):
-        return "no description"
+        pass
 
 class PollingChangeSource(ChangeSource):
     """
     Utility subclass for ChangeSources that use some kind of periodic polling
     operation.  Subclasses should define C{poll} and set C{self.pollInterval}.
     The rest is taken care of.
     """
 
     pollInterval = 60
     "time (in seconds) between calls to C{poll}"
 
     _loop = None
-    volatile = ['_loop'] # prevents Twisted from pickling this value
 
     def poll(self):
         """
         Perform the polling operation, and return a deferred that will fire
         when the operation is complete.  Failures will be logged, but the
         method will be called again after C{pollInterval} seconds.
         """
 
     def startService(self):
         ChangeSource.startService(self)
         def do_poll():
             d = defer.maybeDeferred(self.poll)
-            d.addErrback(log.err)
+            d.addErrback(log.err, 'while polling for changes')
             return d
-        self._loop = task.LoopingCall(do_poll)
-        self._loop.start(self.pollInterval)
+
+        # delay starting the loop until the reactor is running
+        def start_loop():
+            self._loop = task.LoopingCall(do_poll)
+            self._loop.start(self.pollInterval)
+        reactor.callWhenRunning(start_loop)
 
     def stopService(self):
-        self._loop.stop()
+        if self._loop:
+            self._loop.stop()
         return ChangeSource.stopService(self)
-
--- a/master/buildbot/changes/bonsaipoller.py
+++ b/master/buildbot/changes/bonsaipoller.py
@@ -12,19 +12,20 @@
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
 
 import time
 from xml.dom import minidom
 
 from twisted.python import log
+from twisted.internet import defer
 from twisted.web import client
 
-from buildbot.changes import base, changes
+from buildbot.changes import base
 
 class InvalidResultError(Exception):
     def __init__(self, value="InvalidResultError"):
         self.value = value
     def __str__(self):
         return repr(self.value)
 
 class EmptyResult(Exception):
@@ -199,18 +200,16 @@ class BonsaiParser:
     def _getRevision(self):
         return self.currentFileNode.getAttribute("rev")
 
 
 class BonsaiPoller(base.PollingChangeSource):
     compare_attrs = ["bonsaiURL", "pollInterval", "tree",
                      "module", "branch", "cvsroot"]
 
-    parent = None # filled in when we're added
-
     def __init__(self, bonsaiURL, module, branch, tree="default",
                  cvsroot="/cvsroot", pollInterval=30, project=''):
         self.bonsaiURL = bonsaiURL
         self.module = module
         self.branch = branch
         self.tree = tree
         self.cvsroot = cvsroot
         self.repository = module != 'all' and module or ''
@@ -248,28 +247,31 @@ class BonsaiPoller(base.PollingChangeSou
     def _get_changes(self):
         url = self._make_url()
         log.msg("Polling Bonsai tree at %s" % url)
 
         self.lastPoll = time.time()
         # get the page, in XML format
         return client.getPage(url, timeout=self.pollInterval)
 
+    @defer.deferredGenerator
     def _process_changes(self, query):
         try:
             bp = BonsaiParser(query)
             result = bp.getData()
         except InvalidResultError, e:
             log.msg("Could not process Bonsai query: " + e.value)
             return
         except EmptyResult:
             return
 
         for cinode in result.nodes:
             files = [file.filename + ' (revision '+file.revision+')'
                      for file in cinode.files]
-            c = changes.Change(who = cinode.who,
+            self.lastChange = self.lastPoll
+            w = defer.waitForDeferred(
+                    self.master.addChange(who = cinode.who,
                                files = files,
                                comments = cinode.log,
                                when = cinode.date,
-                               branch = self.branch)
-            self.parent.addChange(c)
-            self.lastChange = self.lastPoll
+                               branch = self.branch))
+            yield w
+            w.getResult()
--- a/master/buildbot/changes/changes.py
+++ b/master/buildbot/changes/changes.py
@@ -19,40 +19,19 @@ from cPickle import dump
 from zope.interface import implements
 from twisted.python import log, runtime
 from twisted.web import html
 
 from buildbot import interfaces, util
 from buildbot.process.properties import Properties
 
 class Change:
-    """I represent a single change to the source tree. This may involve
-    several files, but they are all changed by the same person, and there is
-    a change comment for the group as a whole.
-
-    If the version control system supports sequential repository- (or
-    branch-) wide change numbers (like SVN, P4, and Bzr), then revision=
-    should be set to that number. The highest such number will be used at
-    checkout time to get the correct set of files.
-
-    If it does not (like CVS), when= should be set to the timestamp (seconds
-    since epoch, as returned by time.time()) when the change was made. when=
-    will be filled in for you (to the current time) if you omit it, which is
-    suitable for ChangeSources which have no way of getting more accurate
-    timestamps.
-
-    The revision= and branch= values must be ASCII bytestrings, since they
-    will eventually be used in a ShellCommand and passed to os.exec(), which
-    requires bytestrings. These values will also be stored in a database,
-    possibly as unicode, so they must be safely convertable back and forth.
-    This restriction may be relaxed in the future.
-
-    Changes should be submitted to ChangeMaster.addChange() in
-    chronologically increasing order. Out-of-order changes will probably
-    cause the web status displays to be corrupted."""
+    """I represent a single change to the source tree. This may involve several
+    files, but they are all changed by the same person, and there is a change
+    comment for the group as a whole."""
 
     implements(interfaces.IStatusEvent)
 
     number = None
 
     branch = None
     category = None
     revision = None # used to create a source-stamp
@@ -97,16 +76,22 @@ class Change:
     def __setstate__(self, dict):
         self.__dict__ = dict
         # Older Changes won't have a 'properties' attribute in them
         if not hasattr(self, 'properties'):
             self.properties = Properties()
         if not hasattr(self, 'revlink'):
             self.revlink = ""
 
+    def __str__(self):
+        return (u"Change(who=%r, files=%r, comments=%r, revision=%r, " +
+                u"when=%r, category=%r, project=%r, repository=%r)") % (
+                self.who, self.files, self.comments, self.revision,
+                self.when, self.category, self.project, self.repository)
+
     def asText(self):
         data = ""
         data += self.getFileContents()
         if self.repository:
             data += "On: %s\n" % self.repository
         if self.project:
             data += "For: %s\n" % self.project
         data += "At: %s\n" % self.getTime()
@@ -179,33 +164,28 @@ class Change:
 
     def getProperties(self):
         data = ""
         for prop in self.properties.asList():
             data += "  %s: %s" % (prop[0], prop[1])
         return data
 
 
-class ChangeMaster:
+class ChangeMaster: # pragma: no cover
     # this is a stub, retained to allow the "buildbot upgrade-master" tool to
     # read old changes.pck pickle files and convert their contents into the
     # new database format. This is only instantiated by that tool, or by
     # test_db.py which tests that tool. The functionality that previously
     # lived here has been moved into buildbot.changes.manager.ChangeManager
 
     def __init__(self):
         self.changes = []
         # self.basedir must be filled in by the parent
         self.nextNumber = 1
 
-    def addChange(self, change):
-        change.number = self.nextNumber
-        self.nextNumber += 1
-        self.changes.append(change)
-
     def saveYourself(self):
         filename = os.path.join(self.basedir, "changes.pck")
         tmpfilename = filename + ".tmp"
         try:
             dump(self, open(tmpfilename, "wb"))
             if runtime.platformType  == 'win32':
                 # windows cannot rename a file on top of an existing one
                 if os.path.exists(filename):
@@ -214,31 +194,47 @@ class ChangeMaster:
         except Exception:
             log.msg("unable to save changes")
             log.err()
 
     # This method is used by contrib/fix_changes_pickle_encoding.py to recode all
     # bytestrings in an old changes.pck into unicode strings
     def recode_changes(self, old_encoding, quiet=False):
         """Processes the list of changes, with the change attributes re-encoded
-        as UTF-8 bytestrings"""
+        unicode objects"""
         nconvert = 0
         for c in self.changes:
             # give revision special handling, in case it is an integer
             if isinstance(c.revision, int):
                 c.revision = unicode(c.revision)
 
             for attr in ("who", "comments", "revlink", "category", "branch", "revision"):
                 a = getattr(c, attr)
                 if isinstance(a, str):
                     try:
                         setattr(c, attr, a.decode(old_encoding))
                         nconvert += 1
                     except UnicodeDecodeError:
                         raise UnicodeError("Error decoding %s of change #%s as %s:\n%r" %
                                         (attr, c.number, old_encoding, a))
+
+            # filenames are a special case, but in general they'll have the same encoding
+            # as everything else on a system.  If not, well, hack this script to do your
+            # import!
+            newfiles = []
+            for filename in util.flatten(c.files):
+                if isinstance(filename, str):
+                    try:
+                        filename = filename.decode(old_encoding)
+                        nconvert += 1
+                    except UnicodeDecodeError:
+                        raise UnicodeError("Error decoding filename '%s' of change #%s as %s:\n%r" %
+                                        (filename.decode('ascii', 'replace'),
+                                         c.number, old_encoding, a))
+                newfiles.append(filename)
+            c.files = newfiles
         if not quiet:
             print "converted %d strings" % nconvert
 
-class OldChangeMaster(ChangeMaster):
+class OldChangeMaster(ChangeMaster): # pragma: no cover
     # this is a reminder that the ChangeMaster class is old
     pass
 # vim: set ts=4 sts=4 sw=4 et:
new file mode 100644
--- /dev/null
+++ b/master/buildbot/changes/filter.py
@@ -0,0 +1,117 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import re, types
+
+from buildbot.util import ComparableMixin, NotABranch
+
+class ChangeFilter(ComparableMixin):
+
+    # NOTE: If users use a filter_fn, we have no way to determine whether it has
+    # changed at reconfig, so the scheduler will always be restarted.  That's as
+    # good as Python can do.
+    compare_attrs = ('filter_fn', 'checks')
+
+    def __init__(self,
+            # gets a Change object, returns boolean
+            filter_fn=None,
+            # change attribute comparisons: exact match to PROJECT, member of
+            # list PROJECTS, regular expression match to PROJECT_RE, or
+            # PROJECT_FN returns True when called with the project; repository,
+            # branch, and so on are similar.  Note that the regular expressions
+            # are anchored to the first character of the string.  For convenience,
+            # a list can also be specified to the singular option (e.g,. PROJETS
+            project=None, project_re=None, project_fn=None,
+            repository=None, repository_re=None, repository_fn=None,
+            branch=NotABranch, branch_re=None, branch_fn=None,
+            category=None, category_re=None, category_fn=None):
+        def mklist(x):
+            if x is not None and type(x) is not types.ListType:
+                return [ x ]
+            return x
+        def mklist_br(x): # branch needs to be handled specially
+            if x is NotABranch:
+                return None
+            if type(x) is not types.ListType:
+                return [ x ]
+            return x
+        def mkre(r):
+            if r is not None and not hasattr(r, 'match'):
+                r = re.compile(r)
+            return r
+
+        self.filter_fn = filter_fn
+        self.checks = [
+                (mklist(project), mkre(project_re), project_fn, "project"),
+                (mklist(repository), mkre(repository_re), repository_fn, "repository"),
+                (mklist_br(branch), mkre(branch_re), branch_fn, "branch"),
+                (mklist(category), mkre(category_re), category_fn, "category"),
+            ]
+
+    def filter_change(self, change):
+        if self.filter_fn is not None and not self.filter_fn(change):
+            return False
+        for (filt_list, filt_re, filt_fn, chg_attr) in self.checks:
+            chg_val = getattr(change, chg_attr, '')
+            if filt_list is not None and chg_val not in filt_list:
+                return False
+            if filt_re is not None and (chg_val is None or not filt_re.match(chg_val)):
+                return False
+            if filt_fn is not None and not filt_fn(chg_val):
+                return False
+        return True
+
+    def __repr__(self):
+        checks = []
+        for (filt_list, filt_re, filt_fn, chg_attr) in self.checks:
+            if filt_list is not None and len(filt_list) == 1:
+                checks.append('%s == %s' % (chg_attr, filt_list[0]))
+            elif filt_list is not None:
+                checks.append('%s in %r' % (chg_attr, filt_list))
+            if filt_re is not None :
+                checks.append('%s ~/%s/' % (chg_attr, filt_re))
+            if filt_fn is not None :
+                checks.append('%s(%s)' % (filt_fn.__name__, chg_attr))
+
+        return "<%s on %s>" % (self.__class__.__name__, ' and '.join(checks))
+
+    @staticmethod
+    def fromSchedulerConstructorArgs(change_filter=None,
+            branch=NotABranch, categories=None):
+
+        """
+        Static method to create a filter based on constructor args
+        change_filter, branch, and categories; use default values @code{None},
+        @code{NotABranch}, and @code{None}, respectively.  These arguments are
+        interpreted as documented for the
+        L{buildbot.schedulers.basic.Scheduler} class.
+
+        @returns: L{ChangeFilter} instance or None for not filtering
+        """
+
+        # use a change_filter, if given one
+        if change_filter:
+            if (branch is not NotABranch or categories is not None):
+                raise RuntimeError("cannot specify both change_filter and "
+                                   "branch or categories")
+            return change_filter
+        elif branch is not NotABranch or categories:
+            # build a change filter from the deprecated category and branch args
+            cfargs = {}
+            if branch is not NotABranch: cfargs['branch'] = branch
+            if categories: cfargs['category'] = categories
+            return ChangeFilter(**cfargs)
+        else:
+            return None
--- a/master/buildbot/changes/gerritchangesource.py
+++ b/master/buildbot/changes/gerritchangesource.py
@@ -10,30 +10,29 @@
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc., 51
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
 
 from twisted.internet import reactor
 
-from buildbot.changes import base, changes
+from buildbot.changes import base
 from buildbot.util import json
 from buildbot import util
 from twisted.python import log
+from twisted.internet import defer
 from twisted.internet.protocol import ProcessProtocol
 
 class GerritChangeSource(base.ChangeSource):
     """This source will maintain a connection to gerrit ssh server
     that will provide us gerrit events in json format."""
 
     compare_attrs = ["gerritserver", "gerritport"]
 
-    parent = None # filled in when we're added
-
     STREAM_GOOD_CONNECTION_TIME = 120
     "(seconds) connections longer than this are considered good, and reset the backoff timer"
 
     STREAM_BACKOFF_MIN = 0.5
     "(seconds) minimum, but nonzero, time to wait before retrying a failed connection"
 
     STREAM_BACKOFF_EXPONENT = 1.5
     "multiplier used to increase the backoff from MIN to MAX on repeated failures"
@@ -61,75 +60,85 @@ class GerritChangeSource(base.ChangeSour
         self.process = None
         self.streamProcessTimeout = self.STREAM_BACKOFF_MIN
 
     class LocalPP(ProcessProtocol):
         def __init__(self, change_source):
             self.change_source = change_source
             self.data = ""
 
+        @defer.deferredGenerator
         def outReceived(self, data):
             """Do line buffering."""
             self.data += data
             lines = self.data.split("\n")
             self.data = lines.pop(-1) # last line is either empty or incomplete
             for line in lines:
                 log.msg("gerrit: %s" % (line,))
-                self.change_source.lineReceived(line)
+                d = self.change_source.lineReceived(line)
+                wfd = defer.waitForDeferred(d)
+                yield wfd
+                wfd.getResult()
 
         def errReceived(self, data):
             log.msg("gerrit stderr: %s" % (data,))
 
         def processEnded(self, status_object):
             self.change_source.streamProcessStopped()
 
     def lineReceived(self, line):
         try:
             event = json.loads(line)
         except ValueError:
             log.msg("bad json line: %s" % (line,))
-            return
+            return defer.succeed(None)
 
         if type(event) == type({}) and "type" in event and event["type"] in ["patchset-created", "ref-updated"]:
             # flatten the event dictionary, for easy access with WithProperties
             def flatten(event, base, d):
                 for k, v in d.items():
                     if type(v) == dict:
                         flatten(event, base + "." + k, v)
                     else: # already there
                         event[base + "." + k] = v
 
             properties = {}
             flatten(properties, "event", event)
 
             if event["type"] == "patchset-created":
                 change = event["change"]
-                c = changes.Change(who="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]),
-                                   project=change["project"],
-                                   branch=change["branch"],
-                                   revision=event["patchSet"]["revision"],
-                                   revlink=change["url"],
-                                   comments=change["subject"],
-                                   files=["unknown"],
-                                   category=event["type"],
-                                   properties=properties)
+
+                chdict = dict(
+                        who="%s <%s>" % (change["owner"]["name"], change["owner"]["email"]),
+                        project=change["project"],
+                        branch=change["branch"],
+                        revision=event["patchSet"]["revision"],
+                        revlink=change["url"],
+                        comments=change["subject"],
+                        files=["unknown"],
+                        category=event["type"],
+                        properties=properties)
             elif event["type"] == "ref-updated":
                 ref = event["refUpdate"]
-                c = changes.Change(who="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"]),
-                                   project=ref["project"],
-                                   branch=ref["refName"],
-                                   revision=ref["newRev"],
-                                   comments="Gerrit: patchset(s) merged.",
-                                   files=["unknown"],
-                                   category=event["type"],
-                                   properties=properties)
+                chdict = dict(
+                        who="%s <%s>" % (event["submitter"]["name"], event["submitter"]["email"]),
+                        project=ref["project"],
+                        branch=ref["refName"],
+                        revision=ref["newRev"],
+                        comments="Gerrit: patchset(s) merged.",
+                        files=["unknown"],
+                        category=event["type"],
+                        properties=properties)
             else:
-                return # this shouldn't happen anyway
+                return defer.succeed(None) # this shouldn't happen anyway
 
-            self.parent.addChange(c)
+            d = self.master.addChange(**chdict)
+            # eat failures..
+            d.addErrback(log.err, 'error adding change from GerritChangeSource')
+            return d
 
     def streamProcessStopped(self):
         self.process = None
 
         # if the service is stopped, don't try to restart
         if not self.parent:
             log.msg("service is not running; not reconnecting")
             return
--- a/master/buildbot/changes/gitpoller.py
+++ b/master/buildbot/changes/gitpoller.py
@@ -11,241 +11,319 @@
 # this program; if not, write to the Free Software Foundation, Inc., 51
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
 
 import time
 import tempfile
 import os
-import subprocess
-
 from twisted.python import log
 from twisted.internet import defer, utils
 
-from buildbot.changes import base, changes
+from buildbot.util import deferredLocked
+from buildbot.changes import base
 
 class GitPoller(base.PollingChangeSource):
     """This source will poll a remote git repo for changes and submit
     them to the change master."""
     
     compare_attrs = ["repourl", "branch", "workdir",
                      "pollInterval", "gitbin", "usetimestamps",
                      "category", "project"]
                      
     def __init__(self, repourl, branch='master', 
                  workdir=None, pollInterval=10*60, 
                  gitbin='git', usetimestamps=True,
                  category=None, project=None,
-                 pollinterval=-2):
+                 pollinterval=-2, fetch_refspec=None):
         # for backward compatibility; the parameter used to be spelled with 'i'
         if pollinterval != -2:
             pollInterval = pollinterval
         if project is None: project = ''
 
         self.repourl = repourl
         self.branch = branch
         self.pollInterval = pollInterval
+        self.fetch_refspec = fetch_refspec
         self.lastChange = time.time()
         self.lastPoll = time.time()
         self.gitbin = gitbin
         self.workdir = workdir
         self.usetimestamps = usetimestamps
         self.category = category
         self.project = project
         self.changeCount = 0
         self.commitInfo  = {}
+        self.initLock = defer.DeferredLock()
         
         if self.workdir == None:
             self.workdir = tempfile.gettempdir() + '/gitpoller_work'
+            log.msg("WARNING: gitpoller using deprecated temporary workdir " +
+                    "'%s'; consider setting workdir=" % self.workdir)
 
     def startService(self):
-        base.PollingChangeSource.startService(self)
-        
-        dirpath = os.path.dirname(self.workdir.rstrip(os.sep))
-        if not os.path.exists(dirpath):
-            log.msg('gitpoller: creating parent directories for workdir')
-            os.makedirs(dirpath)
-            
+        # make our workdir absolute, relative to the master's basedir
+        if not os.path.isabs(self.workdir):
+            self.workdir = os.path.join(self.master.basedir, self.workdir)
+            log.msg("gitpoller: using workdir '%s'" % self.workdir)
+
+        # initialize the repository we'll use to get changes; note that
+        # startService is not an event-driven method, so this method will
+        # instead acquire self.initLock immediately when it is called.
         if not os.path.exists(self.workdir + r'/.git'):
-            log.msg('gitpoller: initializing working dir')
-            subprocess.check_call([self.gitbin, 'init', self.workdir])
-            subprocess.check_call([self.gitbin, 'remote', 'add', 'origin', self.repourl],
-                    cwd=self.workdir)
-            subprocess.check_call([self.gitbin, 'fetch', 'origin'],
-                    cwd=self.workdir)
-            if self.branch == 'master':
-                subprocess.check_call([self.gitbin, 'reset', '--hard',
-                                       'origin/%s' % self.branch],
-                        cwd=self.workdir)
+            d = self.initRepository()
+            d.addErrback(log.err, 'while initializing GitPoller repository')
+        else:
+            log.msg("GitPoller repository already exists")
+
+        # call this *after* initRepository, so that the initLock is locked first
+        base.PollingChangeSource.startService(self)
+
+    @deferredLocked('initLock')
+    def initRepository(self):
+        d = defer.succeed(None)
+        def make_dir(_):
+            dirpath = os.path.dirname(self.workdir.rstrip(os.sep))
+            if not os.path.exists(dirpath):
+                log.msg('gitpoller: creating parent directories for workdir')
+                os.makedirs(dirpath)
+        d.addCallback(make_dir)
+
+        def git_init(_):
+            log.msg('gitpoller: initializing working dir from %s' % self.repourl)
+            d = utils.getProcessOutputAndValue(self.gitbin,
+                    ['init', self.workdir], env=dict(PATH=os.environ['PATH']))
+            d.addCallback(self._convert_nonzero_to_failure)
+            d.addErrback(self._stop_on_failure)
+            return d
+        d.addCallback(git_init)
+        
+        def git_remote_add(_):
+            d = utils.getProcessOutputAndValue(self.gitbin,
+                    ['remote', 'add', 'origin', self.repourl],
+                    path=self.workdir, env=dict(PATH=os.environ['PATH']))
+            d.addCallback(self._convert_nonzero_to_failure)
+            d.addErrback(self._stop_on_failure)
+            return d
+        d.addCallback(git_remote_add)
+        
+        def git_fetch_origin(_):
+            args = ['fetch', 'origin']
+            self._extend_with_fetch_refspec(args)
+            d = utils.getProcessOutputAndValue(self.gitbin, args,
+                    path=self.workdir, env=dict(PATH=os.environ['PATH']))
+            d.addCallback(self._convert_nonzero_to_failure)
+            d.addErrback(self._stop_on_failure)
+            return d
+        d.addCallback(git_fetch_origin)
+        
+        def set_master(_):
+            log.msg('gitpoller: checking out %s' % self.branch)
+            if self.branch == 'master': # repo is already on branch 'master', so reset
+                d = utils.getProcessOutputAndValue(self.gitbin,
+                        ['reset', '--hard', 'origin/%s' % self.branch],
+                        path=self.workdir, env=dict(PATH=os.environ['PATH']))
             else:
-                subprocess.check_call([self.gitbin, 'checkout', '-b', self.branch,
-                                       'origin/%s' % self.branch],
-                        cwd=self.workdir)
-        
+                d = utils.getProcessOutputAndValue(self.gitbin,
+                        ['checkout', '-b', self.branch, 'origin/%s' % self.branch],
+                        path=self.workdir, env=dict(PATH=os.environ['PATH']))
+            d.addCallback(self._convert_nonzero_to_failure)
+            d.addErrback(self._stop_on_failure)
+            return d
+        d.addCallback(set_master)
+        def get_rev(_):
+            d = utils.getProcessOutputAndValue(self.gitbin,
+                    ['rev-parse', self.branch],
+                    path=self.workdir, env={})
+            d.addCallback(self._convert_nonzero_to_failure)
+            d.addErrback(self._stop_on_failure)
+            d.addCallback(lambda (out, err, code) : out.strip())
+            return d
+        d.addCallback(get_rev)
+        def print_rev(rev):
+            log.msg("gitpoller: finished initializing working dir from %s at rev %s"
+                    % (self.repourl, rev))
+        d.addCallback(print_rev)
+        return d
+
     def describe(self):
         status = ""
-        if not self.parent:
+        if not self.master:
             status = "[STOPPED - check log]"
         str = 'GitPoller watching the remote git repository %s, branch: %s %s' \
                 % (self.repourl, self.branch, status)
         return str
 
+    @deferredLocked('initLock')
     def poll(self):
         d = self._get_changes()
         d.addCallback(self._process_changes)
         d.addErrback(self._process_changes_failure)
         d.addCallback(self._catch_up)
         d.addErrback(self._catch_up_failure)
         return d
 
     def _get_commit_comments(self, rev):
         args = ['log', rev, '--no-walk', r'--format=%s%n%b']
         d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        d.addCallback(self._get_commit_comments_from_output)
+        def process(git_output):
+            stripped_output = git_output.strip()
+            if len(stripped_output) == 0:
+                raise EnvironmentError('could not get commit comment for rev')
+            return stripped_output
+        d.addCallback(process)
         return d
 
-    def _get_commit_comments_from_output(self,git_output):
-        stripped_output = git_output.strip()
-        if len(stripped_output) == 0:
-            raise EnvironmentError('could not get commit comment for rev')
-        self.commitInfo['comments'] = stripped_output
-        return self.commitInfo['comments'] # for tests
-
     def _get_commit_timestamp(self, rev):
         # unix timestamp
         args = ['log', rev, '--no-walk', r'--format=%ct']
         d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        d.addCallback(self._get_commit_timestamp_from_output)
+        def process(git_output):
+            stripped_output = git_output.strip()
+            if self.usetimestamps:
+                try:
+                    stamp = float(stripped_output)
+                except Exception, e:
+                        log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % stripped_output)
+                        raise e
+                return stamp
+            else:
+                return None
+        d.addCallback(process)
         return d
 
-    def _get_commit_timestamp_from_output(self, git_output):
-        stripped_output = git_output.strip()
-        if self.usetimestamps:
-            try:
-                stamp = float(stripped_output)
-            except Exception, e:
-                    log.msg('gitpoller: caught exception converting output \'%s\' to timestamp' % stripped_output)
-                    raise e
-            self.commitInfo['timestamp'] = stamp
-        else:
-            self.commitInfo['timestamp'] = None
-        return self.commitInfo['timestamp'] # for tests
-
     def _get_commit_files(self, rev):
         args = ['log', rev, '--name-only', '--no-walk', r'--format=%n']
         d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        d.addCallback(self._get_commit_files_from_output)
+        def process(git_output):
+            fileList = git_output.split()
+            return fileList
+        d.addCallback(process)
         return d
-
-    def _get_commit_files_from_output(self, git_output):
-        fileList = git_output.split()
-        self.commitInfo['files'] = fileList
-        return self.commitInfo['files'] # for tests
             
     def _get_commit_name(self, rev):
         args = ['log', rev, '--no-walk', r'--format=%aE']
         d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        d.addCallback(self._get_commit_name_from_output)
+        def process(git_output):
+            stripped_output = git_output.strip()
+            if len(stripped_output) == 0:
+                raise EnvironmentError('could not get commit name for rev')
+            return stripped_output
+        d.addCallback(process)
         return d
 
-    def _get_commit_name_from_output(self, git_output):
-        stripped_output = git_output.strip()
-        if len(stripped_output) == 0:
-            raise EnvironmentError('could not get commit name for rev')
-        self.commitInfo['name'] = stripped_output
-        return self.commitInfo['name'] # for tests
-
     def _get_changes(self):
         log.msg('gitpoller: polling git repo at %s' % self.repourl)
 
         self.lastPoll = time.time()
         
-        # get a deferred object that performs the git fetch
+        # get a deferred object that performs the fetch
+        args = ['fetch', 'origin']
+        self._extend_with_fetch_refspec(args)
 
         # This command always produces data on stderr, but we actually do not care
         # about the stderr or stdout from this command. We set errortoo=True to
         # avoid an errback from the deferred. The callback which will be added to this
         # deferred will not use the response.
-        args = ['fetch', self.repourl, self.branch]
-        d = utils.getProcessOutput(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=True )
+        d = utils.getProcessOutput(self.gitbin, args,
+                    path=self.workdir,
+                    env=dict(PATH=os.environ['PATH']), errortoo=True )
 
         return d
 
+    @defer.deferredGenerator
     def _process_changes(self, unused_output):
         # get the change list
-        revListArgs = ['log', 'HEAD..FETCH_HEAD', r'--format=%H']
-        d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir, env=dict(PATH=os.environ['PATH']), errortoo=False )
-        d.addCallback(self._process_changes_in_output)
-        return d
-    
-    @defer.deferredGenerator
-    def _process_changes_in_output(self, git_output):
+        revListArgs = ['log', '%s..origin/%s' % (self.branch, self.branch), r'--format=%H']
         self.changeCount = 0
+        d = utils.getProcessOutput(self.gitbin, revListArgs, path=self.workdir,
+                                   env=dict(PATH=os.environ['PATH']), errortoo=False )
+        wfd = defer.waitForDeferred(d)
+        yield wfd
+        results = wfd.getResult()
         
         # process oldest change first
-        revList = git_output.split()
-        if revList:
-            revList.reverse()
-            self.changeCount = len(revList)
+        revList = results.split()
+        if not revList:
+            return
+
+        revList.reverse()
+        self.changeCount = len(revList)
             
-        log.msg('gitpoller: processing %d changes: %s in "%s"' % (self.changeCount, revList, self.workdir) )
+        log.msg('gitpoller: processing %d changes: %s in "%s"'
+                % (self.changeCount, revList, self.workdir) )
 
         for rev in revList:
-            self.commitInfo = {}
+            dl = defer.DeferredList([
+                self._get_commit_timestamp(rev),
+                self._get_commit_name(rev),
+                self._get_commit_files(rev),
+                self._get_commit_comments(rev),
+            ], consumeErrors=True)
 
-            deferreds = [
-                                self._get_commit_timestamp(rev),
-                                self._get_commit_name(rev),
-                                self._get_commit_files(rev),
-                                self._get_commit_comments(rev),
-                        ]
-            dl = defer.DeferredList(deferreds)
-            dl.addCallback(self._add_change,rev)        
-
-            # wait for that deferred to finish before starting the next
             wfd = defer.waitForDeferred(dl)
             yield wfd
-            wfd.getResult()
-
+            results = wfd.getResult()
 
-    def _add_change(self, results, rev):
-        log.msg('gitpoller: _add_change results: "%s", rev: "%s" in "%s"' % (results, rev, self.workdir))
+            # check for failures
+            failures = [ r[1] for r in results if not r[0] ]
+            if failures:
+                # just fail on the first error; they're probably all related!
+                raise failures[0]
 
-        c = changes.Change(who=self.commitInfo['name'],
-                               revision=rev,
-                               files=self.commitInfo['files'],
-                               comments=self.commitInfo['comments'],
-                               when=self.commitInfo['timestamp'],
-                               branch=self.branch,
-                               category=self.category,
-                               project=self.project,
-                               repository=self.repourl)
-        log.msg('gitpoller: change "%s" in "%s"' % (c, self.workdir))
-        self.parent.addChange(c)
-        self.lastChange = self.lastPoll
-            
+            timestamp, name, files, comments = [ r[1] for r in results ]
+            d = self.master.addChange(
+                   who=name,
+                   revision=rev,
+                   files=files,
+                   comments=comments,
+                   when=timestamp,
+                   branch=self.branch,
+                   category=self.category,
+                   project=self.project,
+                   repository=self.repourl)
+            wfd = defer.waitForDeferred(d)
+            yield wfd
+            results = wfd.getResult()
 
     def _process_changes_failure(self, f):
         log.msg('gitpoller: repo poll failed')
         log.err(f)
         # eat the failure to continue along the defered chain - we still want to catch up
         return None
         
     def _catch_up(self, res):
         if self.changeCount == 0:
             log.msg('gitpoller: no changes, no catch_up')
             return
-        log.msg('gitpoller: catching up to FETCH_HEAD')
-        args = ['reset', '--hard', 'FETCH_HEAD']
+        log.msg('gitpoller: catching up tracking branch')
+        args = ['reset', '--hard', 'origin/%s' % (self.branch,)]
         d = utils.getProcessOutputAndValue(self.gitbin, args, path=self.workdir, env=dict(PATH=os.environ['PATH']))
-        def convert_nonzero_to_failure(res):
-            (stdout, stderr, code) = res
-            if code != 0:
-                raise EnvironmentError('catch up failed with exit code: %d' % code)
-        d.addCallback(convert_nonzero_to_failure)
+        d.addCallback(self._convert_nonzero_to_failure)
         return d
 
     def _catch_up_failure(self, f):
         log.err(f)
         log.msg('gitpoller: please resolve issues in local repo: %s' % self.workdir)
         # this used to stop the service, but this is (a) unfriendly to tests and (b)
         # likely to leave the error message lost in a sea of other log messages
+
+    def _convert_nonzero_to_failure(self, res):
+        "utility method to handle the result of getProcessOutputAndValue"
+        (stdout, stderr, code) = res
+        if code != 0:
+            raise EnvironmentError('command failed with exit code %d: %s' % (code, stderr))
+        return (stdout, stderr, code)
+
+    def _stop_on_failure(self, f):
+        "utility method to stop the service when a failure occurs"
+        if self.running:
+            d = defer.maybeDeferred(lambda : self.stopService())
+            d.addErrback(log.err, 'while stopping broken GitPoller service')
+        return f
+
+    def _extend_with_fetch_refspec(self, args):
+        if self.fetch_refspec:
+            if type(self.fetch_refspec) in (list,set):
+                args.extend(self.fetch_refspec)
+            else:
+                args.append(self.fetch_refspec)
--- a/master/buildbot/changes/hgbuildbot.py
+++ b/master/buildbot/changes/hgbuildbot.py
@@ -43,16 +43,20 @@
 #   fork = True|False                    # if mercurial should fork before 
 #                                        # notifying the master
 #
 #   strip = 3                            # number of path to strip for local 
 #                                        # repo path to form 'repository'
 #
 #   category = None                      # category property
 #   project = ''                         # project this repository belong to
+#
+#   auth = user:passwd                   # How to authenticate, defaults to
+#                                        # change:changepw, which is also
+#                                        # the default of PBChangeSource.
 
 import os
 
 from mercurial.node import bin, hex, nullid #@UnresolvedImport
 from mercurial.context import workingctx #@UnresolvedImport
 
 # mercurial's on-demand-importing hacks interfere with the:
 #from zope.interface import Interface
@@ -69,16 +73,17 @@ def hook(ui, repo, hooktype, node=None, 
     if master:
         branchtype = ui.config('hgbuildbot', 'branchtype')
         branch = ui.config('hgbuildbot', 'branch')
         fork = ui.configbool('hgbuildbot', 'fork', False)
         # notify also has this setting
         stripcount = int(ui.config('notify','strip') or ui.config('hgbuildbot','strip',3))
         category = ui.config('hgbuildbot', 'category', None)
         project = ui.config('hgbuildbot', 'project', '')
+        auth = ui.config('hgbuildbot', 'auth', None)
     else:
         ui.write("* You must add a [hgbuildbot] section to .hg/hgrc in "
                  "order to use buildbot hook\n")
         return
 
     if hooktype != "changegroup":
         ui.status("hgbuildbot: hooktype %s not supported.\n" % hooktype)
         return
@@ -99,17 +104,21 @@ def hook(ui, repo, hooktype, node=None, 
 
     if branch is None:
         if branchtype is not None:
             if branchtype == 'dirname':
                 branch = os.path.basename(repo.root)
             if branchtype == 'inrepo':
                 branch = workingctx(repo).branch()
 
-    s = sendchange.Sender(master)
+    if not auth:
+        auth = 'change:changepw'
+    auth = auth.split(':', 1)
+
+    s = sendchange.Sender(master, auth=auth)
     d = defer.Deferred()
     reactor.callLater(0, d.callback, None)
     # process changesets
     def _send(res, c):
         if not fork:
             ui.status("rev %s sent\n" % c['revision'])
         return s.send(c['branch'], c['revision'], c['comments'],
                       c['files'], c['username'], category=category,
@@ -139,17 +148,23 @@ def hook(ui, repo, hooktype, node=None, 
             'username': user,
             'revision': hex(node),
             'comments': desc,
             'files': files,
             'branch': branch
         }
         d.addCallback(_send, change)
 
-    d.addCallbacks(s.printSuccess, s.printFailure)
+    def _printSuccess(res):
+        ui.status(s.getSuccessString(res) + '\n')
+
+    def _printFailure(why):
+        ui.warn(s.getFailureString(why) + '\n')
+
+    d.addCallbacks(_printSuccess, _printFailure)
     d.addBoth(s.stop)
     s.run()
 
     if fork:
         os._exit(os.EX_OK)
     else:
         return
 
--- a/master/buildbot/changes/mail.py
+++ b/master/buildbot/changes/mail.py
@@ -8,62 +8,70 @@
 # details.
 #
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc., 51
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
 
-
 """
 Parse various kinds of 'CVS notify' email.
 """
 import os, re
 import time, calendar
 import datetime
 from email import message_from_file
 from email.Utils import parseaddr, parsedate_tz, mktime_tz
 from email.Iterators import body_line_iterator
 
 from zope.interface import implements
 from twisted.python import log
+from twisted.internet import defer
 from buildbot import util
 from buildbot.interfaces import IChangeSource
-from buildbot.changes import changes
-from buildbot.changes.maildir import MaildirService
+from buildbot.util.maildir import MaildirService
 
 class MaildirSource(MaildirService, util.ComparableMixin):
-    """This source will watch a maildir that is subscribed to a FreshCVS
-    change-announcement mailing list.
-    """
+    """Generic base class for Maildir-based change sources"""
     implements(IChangeSource)
 
     compare_attrs = ["basedir", "pollinterval", "prefix"]
-    name = None
 
     def __init__(self, maildir, prefix=None, category='', repository=''):
         MaildirService.__init__(self, maildir)
         self.prefix = prefix
         self.category = category
         self.repository = repository
         if prefix and not prefix.endswith("/"):
             log.msg("%s: you probably want your prefix=('%s') to end with "
                     "a slash")
 
     def describe(self):
-        return "%s mailing list in maildir %s" % (self.name, self.basedir)
+        return "%s watching maildir '%s'" % (self.__class__.__name__, self.basedir)
 
     def messageReceived(self, filename):
         path = os.path.join(self.basedir, "new", filename)
-        change = self.parse_file(open(path, "r"), self.prefix)
-        if change:
-            self.parent.addChange(change)
-        os.rename(os.path.join(self.basedir, "new", filename),
-                  os.path.join(self.basedir, "cur", filename))
+        d = defer.succeed(None)
+        def parse_file(_):
+            return self.parse_file(open(path, "r"), self.prefix)
+        d.addCallback(parse_file)
+
+        def add_change(chdict):
+            if chdict:
+                return self.master.addChange(**chdict)
+            else:
+                log.msg("no change found in maildir file '%s'" % filename)
+        d.addCallback(add_change)
+
+        def move_file(_):
+            os.rename(path, os.path.join(self.basedir, "cur", filename))
+        d.addCallback(move_file)
+
+        return d
 
     def parse_file(self, fd, prefix=None):
         m = message_from_file(fd)
         return self.parse(m, prefix)
 
 class CVSMaildirSource(MaildirSource):
     name = "CVSMaildirSource"
 
@@ -232,24 +240,29 @@ class CVSMaildirSource(MaildirSource):
         # Now get comments    
         while lines:
             line = lines.pop(0)
             comments += line
             
         comments = comments.rstrip() + "\n"
         if comments == '\n':
             comments = None
-        change = changes.Change(who, files, comments, isdir, when=when,
-                                branch=branch, revision=rev,
-                                category=category,
-                                repository=cvsroot,
-                                project=project,
-                                links=links,
-                                properties=self.properties)
-        return change
+        return dict(
+                who=who,
+                files=files,
+                comments=comments,
+                isdir=isdir,
+                when=when,
+                branch=branch,
+                revision=rev,
+                category=category,
+                repository=cvsroot,
+                project=project,
+                links=links,
+                properties=self.properties)
 
 # svn "commit-email.pl" handler.  The format is very similar to freshcvs mail;
 # here's a sample:
 
 #  From: username [at] apache.org    [slightly obfuscated to avoid spam here]
 #  To: commits [at] spamassassin.apache.org
 #  Subject: svn commit: r105955 - in spamassassin/trunk: . lib/Mail
 #  ...
@@ -279,17 +292,17 @@ class SVNCommitEmailMaildirSource(Maildi
         """
 
         # The mail is sent from the person doing the checkin. Assume that the
         # local username is enough to identify them (this assumes a one-server
         # cvs-over-rsh environment rather than the server-dirs-shared-over-NFS
         # model)
         name, addr = parseaddr(m["from"])
         if not addr:
-            return None # no From means this message isn't from FreshCVS
+            return None # no From means this message isn't from svn
         at = addr.find("@")
         if at == -1:
             who = addr # might still be useful
         else:
             who = addr[:at]
 
         # we take the time of receipt as the time of checkin. Not correct (it
         # depends upon the email latency), but it avoids the
@@ -365,17 +378,22 @@ class SVNCommitEmailMaildirSource(Maildi
                 # TODO: figure out how new directories are described, set
                 # .isdir
                 files.append(f)
 
         if not files:
             log.msg("no matching files found, ignoring commit")
             return None
 
-        return changes.Change(who, files, comments, when=when, revision=rev)
+        return dict(
+                who=who,
+                files=files,
+                comments=comments,
+                when=when,
+                revision=rev)
 
 # bzr Launchpad branch subscription mails. Sample mail:
 #
 #   From: noreply@launchpad.net
 #   Subject: [Branch ~knielsen/maria/tmp-buildbot-test] Rev 2701: test add file
 #   To: Joe <joe@acme.com>
 #   ...
 #   
@@ -492,20 +510,24 @@ class BzrLaunchpadEmailMaildirSource(Mai
             if self.defaultBranch:
                 branch = self.defaultBranch
             else:
                 if repository:
                     branch = 'lp:' + repository
                 else:
                     branch = None
 
-        #log.msg("parse(): rev=%s who=%s files=%s comments='%s' when=%s branch=%s" % (rev, who, d['files'], d['comments'], time.asctime(time.localtime(when)), branch))
         if rev and who:
-            return changes.Change(who, d['files'], d['comments'],
-                                  when=when, revision=rev, branch=branch, 
-                                  repository=repository or '')
+            return dict(
+                    who=who,
+                    files=d['files'],
+                    comments=d['comments'],
+                    when=when,
+                    revision=rev,
+                    branch=branch,
+                    repository=repository or '')
         else:
             return None
 
 def parseLaunchpadDate(datestr, tz_sign, tz_hours, tz_minutes):
     time_no_tz = calendar.timegm(time.strptime(datestr, "%Y-%m-%d %H:%M:%S"))
     tz_delta = 60 * 60 * int(tz_sign + tz_hours) + 60 * int(tz_minutes)
     return time_no_tz - tz_delta
deleted file mode 100644
--- a/master/buildbot/changes/maildir.py
+++ /dev/null
@@ -1,131 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-
-# This is a class which watches a maildir for new messages. It uses the
-# linux dirwatcher API (if available) to look for new files. The
-# .messageReceived method is invoked with the filename of the new message,
-# relative to the top of the maildir (so it will look like "new/blahblah").
-
-import os
-from twisted.python import log
-from twisted.application import service, internet
-from twisted.internet import reactor
-dnotify = None
-try:
-    import dnotify
-except:
-    # I'm not actually sure this log message gets recorded
-    log.msg("unable to import dnotify, so Maildir will use polling instead")
-
-class NoSuchMaildir(Exception):
-    pass
-
-class MaildirService(service.MultiService):
-    """I watch a maildir for new messages. I should be placed as the service
-    child of some MultiService instance. When running, I use the linux
-    dirwatcher API (if available) or poll for new files in the 'new'
-    subdirectory of my maildir path. When I discover a new message, I invoke
-    my .messageReceived() method with the short filename of the new message,
-    so the full name of the new file can be obtained with
-    os.path.join(maildir, 'new', filename). messageReceived() should be
-    overridden by a subclass to do something useful. I will not move or
-    delete the file on my own: the subclass's messageReceived() should
-    probably do that.
-    """
-    pollinterval = 10  # only used if we don't have DNotify
-
-    def __init__(self, basedir=None):
-        """Create the Maildir watcher. BASEDIR is the maildir directory (the
-        one which contains new/ and tmp/)
-        """
-        service.MultiService.__init__(self)
-        self.basedir = basedir
-        self.files = []
-        self.dnotify = None
-
-    def setBasedir(self, basedir):
-        # some users of MaildirService (scheduler.Try_Jobdir, in particular)
-        # don't know their basedir until setServiceParent, since it is
-        # relative to the buildmaster's basedir. So let them set it late. We
-        # don't actually need it until our own startService.
-        self.basedir = basedir
-
-    def startService(self):
-        service.MultiService.startService(self)
-        self.newdir = os.path.join(self.basedir, "new")
-        if not os.path.isdir(self.basedir) or not os.path.isdir(self.newdir):
-            raise NoSuchMaildir("invalid maildir '%s'" % self.basedir)
-        try:
-            if dnotify:
-                # we must hold an fd open on the directory, so we can get
-                # notified when it changes.
-                self.dnotify = dnotify.DNotify(self.newdir,
-                                               self.dnotify_callback,
-                                               [dnotify.DNotify.DN_CREATE])
-        except (IOError, OverflowError):
-            # IOError is probably linux<2.4.19, which doesn't support
-            # dnotify. OverflowError will occur on some 64-bit machines
-            # because of a python bug
-            log.msg("DNotify failed, falling back to polling")
-        if not self.dnotify:
-            t = internet.TimerService(self.pollinterval, self.poll)
-            t.setServiceParent(self)
-        self.poll()
-
-    def dnotify_callback(self):
-        log.msg("dnotify noticed something, now polling")
-
-        # give it a moment. I found that qmail had problems when the message
-        # was removed from the maildir instantly. It shouldn't, that's what
-        # maildirs are made for. I wasn't able to eyeball any reason for the
-        # problem, and safecat didn't behave the same way, but qmail reports
-        # "Temporary_error_on_maildir_delivery" (qmail-local.c:165,
-        # maildir_child() process exited with rc not in 0,2,3,4). Not sure
-        # why, and I'd have to hack qmail to investigate further, so it's
-        # easier to just wait a second before yanking the message out of new/
-
-        reactor.callLater(0.1, self.poll)
-
-
-    def stopService(self):
-        if self.dnotify:
-            self.dnotify.remove()
-            self.dnotify = None
-        return service.MultiService.stopService(self)
-
-    def poll(self):
-        assert self.basedir
-        # see what's new
-        for f in self.files:
-            if not os.path.isfile(os.path.join(self.newdir, f)):
-                self.files.remove(f)
-        newfiles = []
-        for f in os.listdir(self.newdir):
-            if not f in self.files:
-                newfiles.append(f)
-        self.files.extend(newfiles)
-        # TODO: sort by ctime, then filename, since safecat uses a rather
-        # fine-grained timestamp in the filename
-        for n in newfiles:
-            # TODO: consider catching exceptions in messageReceived
-            self.messageReceived(n)
-
-    def messageReceived(self, filename):
-        """Called when a new file is noticed. Will call
-        self.parent.messageReceived() with a path relative to maildir/new.
-        Should probably be overridden in subclasses."""
-        self.parent.messageReceived(filename)
-
--- a/master/buildbot/changes/manager.py
+++ b/master/buildbot/changes/manager.py
@@ -8,112 +8,62 @@
 # details.
 #
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc., 51
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
 
-import time
-
 from zope.interface import implements
-from twisted.python import log
 from twisted.internet import defer
 from twisted.application import service
 
 from buildbot import interfaces, util
 
 class ChangeManager(service.MultiService):
-
-    """This is the master-side service which receives file change
-    notifications from a VCS. It keeps a log of these changes, enough to
-    provide for the HTML waterfall display, and to tell
-    temporarily-disconnected bots what they missed while they were
-    offline.
-
-    Change notifications come from two different kinds of sources. The first
-    is a PB service (servicename='changemaster', perspectivename='change'),
-    which provides a remote method called 'addChange', which should be
-    called with a dict that has keys 'filename' and 'comments'.
+    """
+    This is the master-side service which receives file change notifications
+    from version-control systems.
 
-    The second is a list of objects derived from the 
-    L{buildbot.changes.base.ChangeSource} class. These are added with 
-    .addSource(), which also sets the .changemaster attribute in the source 
-    to point at the ChangeMaster. When the application begins, these will 
-    be started with .start() . At shutdown time, they will be terminated 
-    with .stop() . They must be persistable. They are expected to call 
-    self.changemaster.addChange() with Change objects.
-
-    There are several different variants of the second type of source:
-
-      - L{buildbot.changes.mail.MaildirSource} watches a maildir for CVS
-        commit mail. It uses DNotify if available, or polls every 10
-        seconds if not.  It parses incoming mail to determine what files
-        were changed.
-
+    It is a Twisted service, which has instances of
+    L{buildbot.interfaces.IChangeSource} as child services. These are added by
+    the master with C{addSource}.
     """
 
     implements(interfaces.IEventSource)
 
-    changeHorizon = None
     lastPruneChanges = None
     name = "changemanager"
 
     def __init__(self):
         service.MultiService.__init__(self)
+        self.master = None
+        "the BuildMaster"
         self._cache = util.LRUCache()
         self.lastPruneChanges = 0
-        self.changeHorizon = 0
+
+    def startService(self):
+        service.MultiService.startService(self)
+        self.master = self.parent
 
     def addSource(self, source):
         assert interfaces.IChangeSource.providedBy(source)
         assert service.IService.providedBy(source)
+        source.master = self.master
         source.setServiceParent(self)
 
     def removeSource(self, source):
         assert source in self
-        return defer.maybeDeferred(source.disownServiceParent)
-
-    def addChange(self, change):
-        """Deliver a file change event. The event should be a Change object.
-        This method will timestamp the object as it is received."""
-        msg = ("adding change, who %s, %d files, rev=%s, branch=%s, repository=%s, "
-                "comments %s, category %s, project %s" % (change.who, len(change.files),
-                                              change.revision, change.branch, change.repository,
-                                              change.comments, change.category, change.project))
-        log.msg(msg.encode('utf-8', 'replace'))
-
-        # this sets change.number, if it wasn't already set (by the
-        # migration-from-pickle code). It also fires a notification which
-        # wakes up the Schedulers.
-        self.parent.addChange(change)
-
-        self.pruneChanges(change.number)
-
-    def pruneChanges(self, last_added_changeid):
-        # this is an expensive operation, so only do it once per second, in case
-        # addChanges is called frequently
-        if not self.changeHorizon or self.lastPruneChanges > time.time() - 1:
-            return
-        self.lastPruneChanges = time.time()
-
-        ids = self.parent.db.getChangeIdsLessThanIdNow(last_added_changeid - self.changeHorizon + 1)
-        for changeid in ids:
-            log.msg("removing change with id %s" % changeid)
-            self.parent.db.removeChangeNow(changeid)
+        d = defer.maybeDeferred(source.disownServiceParent)
+        def unset_master(x):
+            source.master = None
+            return x
+        d.addBoth(unset_master)
+        return d
 
     # IEventSource methods
 
-    def eventGenerator(self, branches=[], categories=[], committers=[], minTime=0):
-        return self.parent.db.changeEventGenerator(branches, categories,
-                                                   committers, minTime)
-
-    def getChangeNumberedNow(self, changeid, t=None):
-        return self.parent.db.getChangeNumberedNow(changeid, t)
-    def getChangeByNumber(self, changeid):
-        return self.parent.db.getChangeByNumber(changeid)
-    def getChangesGreaterThan(self, last_changeid, t=None):
-        return self.parent.db.getChangesGreaterThan(last_changeid, t)
-    def getChangesByNumber(self, changeids):
-        return self.parent.db.getChangesByNumber(changeids)
-    def getLatestChangeNumberNow(self, branch=None, t=None):
-        return self.parent.db.getLatestChangeNumberNow(branch=branch, t=t)
+    def eventGenerator(self, branches=[], categories=[],
+                            committers=[], minTime=0): # pragma: no cover
+        # TODO: this function must be eliminated.
+        return self.parent.db.changes.changeEventGenerator(branches,
+                                                categories, committers, minTime)
--- a/master/buildbot/changes/p4poller.py
+++ b/master/buildbot/changes/p4poller.py
@@ -16,223 +16,171 @@
 
 # Many thanks to Dave Peticolas for contributing this module
 
 import re
 import time
 import os
 
 from twisted.python import log
-from twisted.internet import defer, reactor
-from twisted.internet.utils import getProcessOutput
-from twisted.internet.task import LoopingCall
+from twisted.internet import defer, utils
 
 from buildbot import util
-from buildbot.changes import base, changes
+from buildbot.changes import base
 
 class P4PollerError(Exception):
     """Something went wrong with the poll. This is used as a distinctive
     exception type so that unit tests can detect and ignore it."""
 
 def get_simple_split(branchfile):
     """Splits the branchfile argument and assuming branch is 
        the first path component in branchfile, will return
        branch and file else None."""
 
     index = branchfile.find('/')
     if index == -1: return None, None
     branch, file = branchfile.split('/', 1)
     return branch, file
 
-class P4Source(base.ChangeSource, util.ComparableMixin):
+class P4Source(base.PollingChangeSource, util.ComparableMixin):
     """This source will poll a perforce repository for changes and submit
     them to the change master."""
 
     compare_attrs = ["p4port", "p4user", "p4passwd", "p4base",
-                     "p4bin", "pollinterval"]
+                     "p4bin", "pollInterval"]
 
     env_vars = ["P4CLIENT", "P4PORT", "P4PASSWD", "P4USER",
                 "P4CHARSET"]
 
     changes_line_re = re.compile(
             r"Change (?P<num>\d+) on \S+ by \S+@\S+ '.*'$")
     describe_header_re = re.compile(
             r"Change \d+ by (?P<who>\S+)@\S+ on (?P<when>.+)$")
     file_re = re.compile(r"^\.\.\. (?P<path>[^#]+)#\d+ [/\w]+$")
     datefmt = '%Y/%m/%d %H:%M:%S'
 
     parent = None # filled in when we're added
     last_change = None
     loop = None
-    working = False
 
     def __init__(self, p4port=None, p4user=None, p4passwd=None,
                  p4base='//', p4bin='p4',
                  split_file=lambda branchfile: (None, branchfile),
-                 pollinterval=60 * 10, histmax=None):
-        """
-        @type  p4port:       string
-        @param p4port:       p4 port definition (host:portno)
-        @type  p4user:       string
-        @param p4user:       p4 user
-        @type  p4passwd:     string
-        @param p4passwd:     p4 passwd
-        @type  p4base:       string
-        @param p4base:       p4 file specification to limit a poll to
-                             without the trailing '...' (i.e., //)
-        @type  p4bin:        string
-        @param p4bin:        path to p4 binary, defaults to just 'p4'
-        @type  split_file:   func
-        $param split_file:   splits a filename into branch and filename.
-        @type  pollinterval: int
-        @param pollinterval: interval in seconds between polls
-        @type  histmax:      int
-        @param histmax:      (obsolete) maximum number of changes to look back through.
-                             ignored; accepted for backwards compatibility.
-        """
+                 pollInterval=60 * 10, histmax=None, pollinterval=-2):
+        # for backward compatibility; the parameter used to be spelled with 'i'
+        if pollinterval != -2:
+            pollInterval = pollinterval
 
         self.p4port = p4port
         self.p4user = p4user
         self.p4passwd = p4passwd
         self.p4base = p4base
         self.p4bin = p4bin
         self.split_file = split_file
-        self.pollinterval = pollinterval
-        self.loop = LoopingCall(self.checkp4)
-
-    def startService(self):
-        base.ChangeSource.startService(self)
-
-        # Don't start the loop just yet because the reactor isn't running.
-        # Give it a chance to go and install our SIGCHLD handler before
-        # spawning processes.
-        reactor.callLater(0, self.loop.start, self.pollinterval)
-
-    def stopService(self):
-        self.loop.stop()
-        return base.ChangeSource.stopService(self)
+        self.pollInterval = pollInterval
 
     def describe(self):
         return "p4source %s %s" % (self.p4port, self.p4base)
 
-    def checkp4(self):
-        # Our return value is only used for unit testing.
-        if self.working:
-            log.msg("Skipping checkp4 because last one has not finished")
-            return defer.succeed(None)
-        else:
-            self.working = True
-            d = self._get_changes()
-            d.addCallback(self._process_changes)
-            d.addCallbacks(self._finished_ok, self._finished_failure)
-            return d
-
-    def _finished_ok(self, res):
-        assert self.working
-        self.working = False
-        return res
-
-    def _finished_failure(self, res):
-        assert self.working
-        self.working = False
-
-        # Again, the return value is only for unit testing. If there's a
-        # failure, log it so it isn't lost. Use log.err to make sure unit
-        # tests flunk if there was a problem.
-        log.err(res, 'P4 poll failed')
-        return None
+    def poll(self):
+        d = self._poll()
+        d.addErrback(log.err, 'P4 poll failed')
+        return d
 
     def _get_process_output(self, args):
         env = dict([(e, os.environ.get(e)) for e in self.env_vars if os.environ.get(e)])
-        d = getProcessOutput(self.p4bin, args, env)
+        d = utils.getProcessOutput(self.p4bin, args, env)
         return d
 
-    def _get_changes(self):
+    @defer.deferredGenerator
+    def _poll(self):
         args = []
         if self.p4port:
             args.extend(['-p', self.p4port])
         if self.p4user:
             args.extend(['-u', self.p4user])
         if self.p4passwd:
             args.extend(['-P', self.p4passwd])
         args.extend(['changes'])
         if self.last_change is not None:
             args.extend(['%s...@%d,now' % (self.p4base, self.last_change+1)])
         else:
             args.extend(['-m', '1', '%s...' % (self.p4base,)])
-        return self._get_process_output(args)
 
-    def _process_changes(self, result):
+        wfd = defer.waitForDeferred(self._get_process_output(args))
+        yield wfd
+        result = wfd.getResult()
+
         last_change = self.last_change
         changelists = []
         for line in result.split('\n'):
             line = line.strip()
             if not line: continue
             m = self.changes_line_re.match(line)
             if not m:
                 raise P4PollerError("Unexpected 'p4 changes' output: %r" % result)
             num = int(m.group('num'))
             if last_change is None:
+                # first time through, the poller just gets a "baseline" for where to
+                # start on the next poll
                 log.msg('P4Poller: starting at change %d' % num)
                 self.last_change = num
-                return []
+                return
             changelists.append(num)
         changelists.reverse() # oldest first
 
         # Retrieve each sequentially.
-        d = defer.succeed(None)
-        for c in changelists:
-            d.addCallback(self._get_describe, c)
-            d.addCallback(self._process_describe, c)
-        return d
+        for num in changelists:
+            args = []
+            if self.p4port:
+                args.extend(['-p', self.p4port])
+            if self.p4user:
+                args.extend(['-u', self.p4user])
+            if self.p4passwd:
+                args.extend(['-P', self.p4passwd])
+            args.extend(['describe', '-s', str(num)])
+            wfd = defer.waitForDeferred(self._get_process_output(args))
+            yield wfd
+            result = wfd.getResult()
 
-    def _get_describe(self, dummy, num):
-        args = []
-        if self.p4port:
-            args.extend(['-p', self.p4port])
-        if self.p4user:
-            args.extend(['-u', self.p4user])
-        if self.p4passwd:
-            args.extend(['-P', self.p4passwd])
-        args.extend(['describe', '-s', str(num)])
-        return self._get_process_output(args)
+            lines = result.split('\n')
+            # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date
+            # field. The rstrip() is intended to remove that.
+            lines[0] = lines[0].rstrip()
+            m = self.describe_header_re.match(lines[0])
+            if not m:
+                raise P4PollerError("Unexpected 'p4 describe -s' result: %r" % result)
+            who = m.group('who')
+            when = time.mktime(time.strptime(m.group('when'), self.datefmt))
+            comments = ''
+            while not lines[0].startswith('Affected files'):
+                comments += lines.pop(0) + '\n'
+            lines.pop(0) # affected files
 
-    def _process_describe(self, result, num):
-        lines = result.split('\n')
-        # SF#1555985: Wade Brainerd reports a stray ^M at the end of the date
-        # field. The rstrip() is intended to remove that.
-        lines[0] = lines[0].rstrip()
-        m = self.describe_header_re.match(lines[0])
-        if not m:
-            raise P4PollerError("Unexpected 'p4 describe -s' result: %r" % result)
-        who = m.group('who')
-        when = time.mktime(time.strptime(m.group('when'), self.datefmt))
-        comments = ''
-        while not lines[0].startswith('Affected files'):
-            comments += lines.pop(0) + '\n'
-        lines.pop(0) # affected files
+            branch_files = {} # dict for branch mapped to file(s)
+            while lines:
+                line = lines.pop(0).strip()
+                if not line: continue
+                m = self.file_re.match(line)
+                if not m:
+                    raise P4PollerError("Invalid file line: %r" % line)
+                path = m.group('path')
+                if path.startswith(self.p4base):
+                    branch, file = self.split_file(path[len(self.p4base):])
+                    if (branch == None and file == None): continue
+                    if branch_files.has_key(branch):
+                        branch_files[branch].append(file)
+                    else:
+                        branch_files[branch] = [file]
 
-        branch_files = {} # dict for branch mapped to file(s)
-        while lines:
-            line = lines.pop(0).strip()
-            if not line: continue
-            m = self.file_re.match(line)
-            if not m:
-                raise P4PollerError("Invalid file line: %r" % line)
-            path = m.group('path')
-            if path.startswith(self.p4base):
-                branch, file = self.split_file(path[len(self.p4base):])
-                if (branch == None and file == None): continue
-                if branch_files.has_key(branch):
-                    branch_files[branch].append(file)
-                else:
-                    branch_files[branch] = [file]
+            for branch in branch_files:
+                d = self.master.addChange(
+                       who=who,
+                       files=branch_files[branch],
+                       comments=comments,
+                       revision=str(num),
+                       when=when,
+                       branch=branch)
+                wfd = defer.waitForDeferred(d)
+                yield wfd
+                wfd.getResult()
 
-        for branch in branch_files:
-            c = changes.Change(who=who,
-                               files=branch_files[branch],
-                               comments=comments,
-                               revision=str(num),
-                               when=when,
-                               branch=branch)
-            self.parent.addChange(c)
-
-        self.last_change = num
+            self.last_change = num
--- a/master/buildbot/changes/pb.py
+++ b/master/buildbot/changes/pb.py
@@ -13,114 +13,74 @@
 #
 # Copyright Buildbot Team Members
 
 
 from twisted.python import log
 from twisted.internet import defer
 
 from buildbot.pbutil import NewCredPerspective
-from buildbot.changes import base, changes
+from buildbot.changes import base
 
 class ChangePerspective(NewCredPerspective):
 
-    def __init__(self, changemaster, prefix):
-        self.changemaster = changemaster
+    def __init__(self, master, prefix):
+        self.master = master
         self.prefix = prefix
 
     def attached(self, mind):
         return self
     def detached(self, mind):
         pass
 
     def perspective_addChange(self, changedict):
         log.msg("perspective_addChange called")
-        pathnames = []
+        files = []
         for path in changedict['files']:
             if self.prefix:
                 if not path.startswith(self.prefix):
                     # this file does not start with the prefix, so ignore it
                     continue
                 path = path[len(self.prefix):]
-            pathnames.append(path)
+            files.append(path)
+        changedict['files'] = files
 
-        if pathnames:
-            change = changes.Change(who=changedict['who'],
-                                    files=pathnames,
-                                    comments=changedict['comments'],
-                                    branch=changedict.get('branch'),
-                                    revision=changedict.get('revision'),
-                                    revlink=changedict.get('revlink', ''),
-                                    category=changedict.get('category'),
-                                    when=changedict.get('when'),
-                                    properties=changedict.get('properties', {}),
-                                    repository=changedict.get('repository', '') or '',
-                                    project=changedict.get('project', '') or '',
-                                    )
-            self.changemaster.addChange(change)
+        if not files:
+            log.msg("No files listed in change... bit strange, but not fatal.")
+        return self.master.addChange(**changedict)
 
 class PBChangeSource(base.ChangeSource):
     compare_attrs = ["user", "passwd", "port", "prefix", "port"]
 
     def __init__(self, user="change", passwd="changepw", port=None,
-            prefix=None, sep=None):
-        """I listen on a TCP port for Changes from 'buildbot sendchange'.
-
-        I am a ChangeSource which will accept Changes from a remote source. I
-        share a TCP listening port with the buildslaves.
-
-        The 'buildbot sendchange' command, the contrib/svn_buildbot.py tool,
-        and the contrib/bzr_buildbot.py tool know how to send changes to me.
-
-        @type prefix: string (or None)
-        @param prefix: if set, I will ignore any filenames that do not start
-                       with this string. Moreover I will remove this string
-                       from all filenames before creating the Change object
-                       and delivering it to the Schedulers. This is useful
-                       for changes coming from version control systems that
-                       represent branches as parent directories within the
-                       repository (like SVN and Perforce). Use a prefix of
-                       'trunk/' or 'project/branches/foobranch/' to only
-                       follow one branch and to get correct tree-relative
-                       filenames.
-
-        @param sep: DEPRECATED (with an axe). sep= was removed in
-                    buildbot-0.7.4 . Instead of using it, you should use
-                    prefix= with a trailing directory separator. This
-                    docstring (and the better-than-nothing error message
-                    which occurs when you use it) will be removed in 0.7.5 .
-
-        @param port: strport to use, or None to use the master's slavePortnum
-        """
-
-        # sep= was removed in 0.7.4 . This more-helpful-than-nothing error
-        # message will be removed in 0.7.5 .
-        assert sep is None, "prefix= is now a complete string, do not use sep="
+            prefix=None):
 
         self.user = user
         self.passwd = passwd
         self.port = port
         self.prefix = prefix
         self.registration = None
 
     def describe(self):
         # TODO: when the dispatcher is fixed, report the specific port
-        #d = "PB listener on port %d" % self.port
-        d = "PBChangeSource listener on all-purpose slaveport"
+        if self.port is not None:
+            portname = self.port
+        else:
+            portname = "all-purpose slaveport"
+        d = "PBChangeSource listener on " + portname
         if self.prefix is not None:
             d += " (prefix '%s')" % self.prefix
         return d
 
     def startService(self):
         base.ChangeSource.startService(self)
-        master = self.parent.parent
         port = self.port
-        if not port:
-            port = master.slavePortnum
-        self.registration = master.pbmanager.register(
+        if port is None:
+            port = self.master.slavePortnum
+        self.registration = self.master.pbmanager.register(
                 port, self.user, self.passwd,
                 self.getPerspective)
 
     def stopService(self):
         d = defer.maybeDeferred(base.ChangeSource.stopService, self)
         def unreg(_):
             if self.registration:
                 return self.registration.unregister()
--- a/master/buildbot/changes/svnpoller.py
+++ b/master/buildbot/changes/svnpoller.py
@@ -14,31 +14,24 @@
 # Copyright Buildbot Team Members
 
 
 # Based on the work of Dave Peticolas for the P4poll
 # Changed to svn (using xml.dom.minidom) by Niklaus Giger
 # Hacked beyond recognition by Brian Warner
 
 from twisted.python import log
-from twisted.internet import defer, reactor, utils
-from twisted.internet.task import LoopingCall
+from twisted.internet import defer, utils
 
 from buildbot import util
 from buildbot.changes import base
-from buildbot.changes.changes import Change
 
 import xml.dom.minidom
 import os, urllib
 
-def _assert(condition, msg):
-    if condition:
-        return True
-    raise AssertionError(msg)
-
 # these split_file_* functions are available for use as values to the
 # split_file= argument.
 def split_file_alwaystrunk(path):
     return (None, path)
 
 def split_file_branches(path):
     # turn trunk/subdir/file.c into (None, "subdir/file.c")
     # and branches/1.5.x/subdir/file.c into ("branches/1.5.x", "subdir/file.c")
@@ -46,220 +39,80 @@ def split_file_branches(path):
     if pieces[0] == 'trunk':
         return (None, '/'.join(pieces[1:]))
     elif pieces[0] == 'branches':
         return ('/'.join(pieces[0:2]), '/'.join(pieces[2:]))
     else:
         return None
 
 
-class SVNPoller(base.ChangeSource, util.ComparableMixin):
-    """This source will poll a Subversion repository for changes and submit
-    them to the change master."""
+class SVNPoller(base.PollingChangeSource, util.ComparableMixin):
+    """
+    Poll a Subversion repository for changes and submit them to the change
+    master.
+    """
 
-    compare_attrs = ["svnurl", "split_file_function",
+    compare_attrs = ["svnurl", "split_file",
                      "svnuser", "svnpasswd",
-                     "pollinterval", "histmax",
+                     "pollInterval", "histmax",
                      "svnbin", "category", "cachepath"]
 
     parent = None # filled in when we're added
     last_change = None
     loop = None
-    working = False
 
     def __init__(self, svnurl, split_file=None,
                  svnuser=None, svnpasswd=None,
-                 pollinterval=10*60, histmax=100,
+                 pollInterval=10*60, histmax=100,
                  svnbin='svn', revlinktmpl='', category=None, 
-                 project='', cachepath=None):
-        """
-        @type  svnurl: string
-        @param svnurl: the SVN URL that describes the repository and
-                       subdirectory to watch. If this ChangeSource should
-                       only pay attention to a single branch, this should
-                       point at the repository for that branch, like
-                       svn://svn.twistedmatrix.com/svn/Twisted/trunk . If it
-                       should follow multiple branches, point it at the
-                       repository directory that contains all the branches
-                       like svn://svn.twistedmatrix.com/svn/Twisted and also
-                       provide a branch-determining function.
-
-                       Each file in the repository has a SVN URL in the form
-                       (SVNURL)/(BRANCH)/(FILEPATH), where (BRANCH) could be
-                       empty or not, depending upon your branch-determining
-                       function. Only files that start with (SVNURL)/(BRANCH)
-                       will be monitored. The Change objects that are sent to
-                       the Schedulers will see (FILEPATH) for each modified
-                       file.
-
-        @type  split_file: callable or None
-        @param split_file: a function that is called with a string of the
-                           form (BRANCH)/(FILEPATH) and should return a tuple
-                           (BRANCH, FILEPATH). This function should match
-                           your repository's branch-naming policy. Each
-                           changed file has a fully-qualified URL that can be
-                           split into a prefix (which equals the value of the
-                           'svnurl' argument) and a suffix; it is this suffix
-                           which is passed to the split_file function.
-
-                           If the function returns None, the file is ignored.
-                           Use this to indicate that the file is not relevant
-                           to this buildmaster.
-                           
-                           For example, if your repository puts the trunk in
-                           trunk/... and branches are in places like
-                           branches/1.5/..., your split_file function could
-                           look like the following (this function is
-                           available as svnpoller.split_file_branches)::
-
-                            pieces = path.split('/')
-                            if pieces[0] == 'trunk':
-                                return (None, '/'.join(pieces[1:]))
-                            elif pieces[0] == 'branches':
-                                return ('/'.join(pieces[0:2]),
-                                        '/'.join(pieces[2:]))
-                            else:
-                                return None
-
-                           If instead your repository layout puts the trunk
-                           for ProjectA in trunk/ProjectA/... and the 1.5
-                           branch in branches/1.5/ProjectA/..., your
-                           split_file function could look like::
-
-                            pieces = path.split('/')
-                            if pieces[0] == 'trunk':
-                                branch = None
-                                pieces.pop(0) # remove 'trunk'
-                            elif pieces[0] == 'branches':
-                                pieces.pop(0) # remove 'branches'
-                                # grab branch name
-                                branch = 'branches/' + pieces.pop(0)
-                            else:
-                                return None # something weird
-                            productname = pieces.pop(0)
-                            if productname != 'ProjectA':
-                                return None # wrong product
-                            return (branch, '/'.join(pieces))
-
-                           The default of split_file= is None, which
-                           indicates that no splitting should be done. This
-                           is equivalent to the following function::
-
-                            return (None, path)
-
-                           If you wish, you can override the split_file
-                           method with the same sort of function instead of
-                           passing in a split_file= argument.
-
-
-        @type  svnuser:      string
-        @param svnuser:      If set, the --username option will be added to
-                             the 'svn log' command. You may need this to get
-                             access to a private repository.
-        @type  svnpasswd:    string
-        @param svnpasswd:    If set, the --password option will be added.
-
-        @type  pollinterval: int
-        @param pollinterval: interval in seconds between polls. The default
-                             is 600 seconds (10 minutes). Smaller values
-                             decrease the latency between the time a change
-                             is recorded and the time the buildbot notices
-                             it, but it also increases the system load.
-
-        @type  histmax:      int
-        @param histmax:      maximum number of changes to look back through.
-                             The default is 100. Smaller values decrease
-                             system load, but if more than histmax changes
-                             are recorded between polls, the extra ones will
-                             be silently lost.
-
-        @type  svnbin:       string
-        @param svnbin:       path to svn binary, defaults to just 'svn'. Use
-                             this if your subversion command lives in an
-                             unusual location.
-        
-        @type  revlinktmpl:  string
-        @param revlinktmpl:  A format string to use for hyperlinks to revision
-                             information. For example, setting this to
-                             "http://reposerver/websvn/revision.php?rev=%s"
-                             would create suitable links on the build pages
-                             to information in websvn on each revision.
-
-        @type  category:     string
-        @param category:     A single category associated with the changes that
-                             could be used by schedulers watch for branches of a
-                             certain name AND category.
-                             
-        @type  project       string
-        @param project       A single project that the changes are associated with
-                             the repository, added to the changes, for the use in 
-                             change filters
-
-        @type  cachepath     string
-        @param cachepath     A path to a file that can be used to store the last
-                             rev that was processed, so we can grab changes that
-                             happened while we were offline
-        """
+                 project='', cachepath=None, pollinterval=-2):
+        # for backward compatibility; the parameter used to be spelled with 'i'
+        if pollinterval != -2:
+            pollInterval = pollinterval
 
         if svnurl.endswith("/"):
             svnurl = svnurl[:-1] # strip the trailing slash
         self.svnurl = svnurl
-        self.split_file_function = split_file or split_file_alwaystrunk
+        self.split_file = split_file or split_file_alwaystrunk
         self.svnuser = svnuser
         self.svnpasswd = svnpasswd
 
         self.revlinktmpl = revlinktmpl
 
         self.environ = os.environ.copy() # include environment variables
                                          # required for ssh-agent auth
 
         self.svnbin = svnbin
-        self.pollinterval = pollinterval
+        self.pollInterval = pollInterval
         self.histmax = histmax
         self._prefix = None
-        self.overrun_counter = 0
-        self.loop = LoopingCall(self.checksvn)
         self.category = category
         self.project = project
 
         self.cachepath = cachepath
         if self.cachepath and os.path.exists(self.cachepath):
             try:
                 f = open(self.cachepath, "r")
                 self.last_change = int(f.read().strip())
                 log.msg("SVNPoller(%s) setting last_change to %s" % (self.svnurl, self.last_change))
                 f.close()
+                # try writing it, too
+                f = open(self.cachepath, "w")
+                f.write(str(self.last_change))
+                f.close()
             except:
                 self.cachepath = None
-                log.msg("SVNPoller(%s) cache file corrupt, skipping and not using" % self.svnurl)
+                log.msg(("SVNPoller(%s) cache file corrupt or unwriteable; " +
+                        "skipping and not using") % self.svnurl)
                 log.err()
 
-    def split_file(self, path):
-        # use getattr() to avoid turning this function into a bound method,
-        # which would require it to have an extra 'self' argument
-        f = getattr(self, "split_file_function")
-        return f(path)
-
-    def startService(self):
-        log.msg("SVNPoller(%s) starting" % self.svnurl)
-        base.ChangeSource.startService(self)
-        # Don't start the loop just yet because the reactor isn't running.
-        # Give it a chance to go and install our SIGCHLD handler before
-        # spawning processes.
-        reactor.callLater(0, self.loop.start, self.pollinterval)
-
-    def stopService(self):
-        log.msg("SVNPoller(%s) shutting down" % self.svnurl)
-        self.loop.stop()
-        return base.ChangeSource.stopService(self)
-
     def describe(self):
         return "SVNPoller watching %s" % self.svnurl
 
-    def checksvn(self):
+    def poll(self):
         # Our return value is only used for unit testing.
 
         # we need to figure out the repository root, so we can figure out
         # repository-relative pathnames later. Each SVNURL is in the form
         # (ROOT)/(PROJECT)/(BRANCH)/(FILEPATH), where (ROOT) is something
         # like svn://svn.twistedmatrix.com/svn/Twisted (i.e. there is a
         # physical repository at /svn/Twisted on that host), (PROJECT) is
         # something like Projects/Twisted (i.e. within the repository's
@@ -281,85 +134,78 @@ class SVNPoller(base.ChangeSource, util.
         # <root> element that tells us ROOT. We then strip this prefix from
         # self.svnurl to determine PROJECT, and then later we strip the
         # PROJECT prefix from the filenames reported by 'svn log --xml' to
         # get a (BRANCH)/(FILEPATH) that can be passed to split_file() to
         # turn into separate BRANCH and FILEPATH values.
 
         # whew.
 
-        if self.working:
-            log.msg("SVNPoller(%s) overrun: timer fired but the previous "
-                    "poll had not yet finished." % self.svnurl)
-            self.overrun_counter += 1
-            return defer.succeed(None)
-        self.working = True
-
         if self.project:
             log.msg("SVNPoller polling " + self.project)
         else:
             log.msg("SVNPoller polling")
+
+        d = defer.succeed(None)
         if not self._prefix:
-            # this sets self._prefix when it finishes. It fires with
-            # self._prefix as well, because that makes the unit tests easier
-            # to write.
-            d = self.get_root()
-            d.addCallback(self.determine_prefix)
-        else:
-            d = defer.succeed(self._prefix)
+            d.addCallback(lambda _ : self.get_prefix())
+            def set_prefix(prefix):
+                self._prefix = prefix
+            d.addCallback(set_prefix)
 
         d.addCallback(self.get_logs)
         d.addCallback(self.parse_logs)
         d.addCallback(self.get_new_logentries)
         d.addCallback(self.create_changes)
         d.addCallback(self.submit_changes)
-        d.addCallbacks(self.finished_ok, self.finished_failure)
+        d.addCallback(self.finished_ok)
+        d.addErrback(log.err, 'error in SVNPoller while polling') # eat errors
         return d
 
     def getProcessOutput(self, args):
         # this exists so we can override it during the unit tests
         d = utils.getProcessOutput(self.svnbin, args, self.environ)
         return d
 
-    def get_root(self):
+    def get_prefix(self):
         args = ["info", "--xml", "--non-interactive", self.svnurl]
         if self.svnuser:
             args.extend(["--username=%s" % self.svnuser])
         if self.svnpasswd:
             args.extend(["--password=%s" % self.svnpasswd])
         d = self.getProcessOutput(args)
+        def determine_prefix(output):
+            try:
+                doc = xml.dom.minidom.parseString(output)
+            except xml.parsers.expat.ExpatError:
+                log.msg("SVNPoller._determine_prefix_2: ExpatError in '%s'"
+                        % output)
+                raise
+            rootnodes = doc.getElementsByTagName("root")
+            if not rootnodes:
+                # this happens if the URL we gave was already the root. In this
+                # case, our prefix is empty.
+                self._prefix = ""
+                return self._prefix
+            rootnode = rootnodes[0]
+            root = "".join([c.data for c in rootnode.childNodes])
+            # root will be a unicode string
+            assert self.svnurl.startswith(root), \
+                    ("svnurl='%s' doesn't start with <root>='%s'" %
+                    (self.svnurl, root))
+            prefix = self.svnurl[len(root):]
+            if prefix.startswith("/"):
+                prefix = prefix[1:]
+            log.msg("SVNPoller: svnurl=%s, root=%s, so prefix=%s" %
+                    (self.svnurl, root, prefix))
+            return prefix
+        d.addCallback(determine_prefix)
         return d
 
-    def determine_prefix(self, output):
-        try:
-            doc = xml.dom.minidom.parseString(output)
-        except xml.parsers.expat.ExpatError:
-            log.msg("SVNPoller._determine_prefix_2: ExpatError in '%s'"
-                    % output)
-            raise
-        rootnodes = doc.getElementsByTagName("root")
-        if not rootnodes:
-            # this happens if the URL we gave was already the root. In this
-            # case, our prefix is empty.
-            self._prefix = ""
-            return self._prefix
-        rootnode = rootnodes[0]
-        root = "".join([c.data for c in rootnode.childNodes])
-        # root will be a unicode string
-        _assert(self.svnurl.startswith(root),
-                "svnurl='%s' doesn't start with <root>='%s'" %
-                (self.svnurl, root))
-        self._prefix = self.svnurl[len(root):]
-        if self._prefix.startswith("/"):
-            self._prefix = self._prefix[1:]
-        log.msg("SVNPoller: svnurl=%s, root=%s, so prefix=%s" %
-                (self.svnurl, root, self._prefix))
-        return self._prefix
-
-    def get_logs(self, ignored_prefix=None):
+    def get_logs(self, _):
         args = []
         args.extend(["log", "--xml", "--verbose", "--non-interactive"])
         if self.svnuser:
             args.extend(["--username=%s" % self.svnuser])
         if self.svnpasswd:
             args.extend(["--password=%s" % self.svnpasswd])
         args.extend(["--limit=%d" % (self.histmax), self.svnurl])
         d = self.getProcessOutput(args)
@@ -371,69 +217,60 @@ class SVNPoller(base.ChangeSource, util.
             doc = xml.dom.minidom.parseString(output)
         except xml.parsers.expat.ExpatError:
             log.msg("SVNPoller.parse_logs: ExpatError in '%s'" % output)
             raise
         logentries = doc.getElementsByTagName("logentry")
         return logentries
 
 
-    def _filter_new_logentries(self, logentries, last_change):
-        # given a list of logentries, return a tuple of (new_last_change,
-        # new_logentries), where new_logentries contains only the ones after
-        # last_change
-        if not logentries:
-            # no entries, so last_change must stay at None
-            return (None, [])
+    def get_new_logentries(self, logentries):
+        last_change = old_last_change = self.last_change
 
-        mostRecent = int(logentries[0].getAttribute("revision"))
+        # given a list of logentries, calculate new_last_change, and
+        # new_logentries, where new_logentries contains only the ones after
+        # last_change
 
-        if last_change is None:
-            # if this is the first time we've been run, ignore any changes
-            # that occurred before now. This prevents a build at every
-            # startup.
-            log.msg('svnPoller: starting at change %s' % mostRecent)
-            return (mostRecent, [])
+        new_last_change = None
+        new_logentries = []
+        if logentries:
+            new_last_change = int(logentries[0].getAttribute("revision"))
 
-        if last_change == mostRecent:
-            # an unmodified repository will hit this case
-            log.msg('svnPoller: _process_changes last %s mostRecent %s' % (
-                      last_change, mostRecent))
-            return (mostRecent, [])
+            if last_change is None:
+                # if this is the first time we've been run, ignore any changes
+                # that occurred before now. This prevents a build at every
+                # startup.
+                log.msg('svnPoller: starting at change %s' % new_last_change)
+            elif last_change == new_last_change:
+                # an unmodified repository will hit this case
+                log.msg('svnPoller: no changes')
+            else:
+                for el in logentries:
+                    if last_change == int(el.getAttribute("revision")):
+                        break
+                    new_logentries.append(el)
+                new_logentries.reverse() # return oldest first
 
-        new_logentries = []
-        for el in logentries:
-            if last_change == int(el.getAttribute("revision")):
-                break
-            new_logentries.append(el)
-        new_logentries.reverse() # return oldest first
-        return (mostRecent, new_logentries)
-
-    def get_new_logentries(self, logentries):
-        last_change = self.last_change
-        (new_last_change,
-         new_logentries) = self._filter_new_logentries(logentries,
-                                                       self.last_change)
         self.last_change = new_last_change
         log.msg('svnPoller: _process_changes %s .. %s' %
-                (last_change, new_last_change))
+                (old_last_change, new_last_change))
         return new_logentries
 
 
     def _get_text(self, element, tag_name):
         try:
             child_nodes = element.getElementsByTagName(tag_name)[0].childNodes
             text = "".join([t.data for t in child_nodes])
         except:
             text = "<unknown>"
         return text
 
     def _transform_path(self, path):
-        _assert(path.startswith(self._prefix),
-                "filepath '%s' should start with prefix '%s'" %
+        assert path.startswith(self._prefix), \
+                ("filepath '%s' should start with prefix '%s'" %
                 (path, self._prefix))
         relative_path = path[len(self._prefix):]
         if relative_path.startswith("/"):
             relative_path = relative_path[1:]
         where = self.split_file(relative_path)
         # 'where' is either None or (branch, final_path)
         return where
 
@@ -445,28 +282,23 @@ class SVNPoller(base.ChangeSource, util.
 
             revlink=''
 
             if self.revlinktmpl:
                 if revision:
                     revlink = self.revlinktmpl % urllib.quote_plus(revision)
 
             log.msg("Adding change revision %s" % (revision,))
-            # TODO: the rest of buildbot may not be ready for unicode 'who'
-            # values
             author   = self._get_text(el, "author")
             comments = self._get_text(el, "msg")
             # there is a "date" field, but it provides localtime in the
             # repository's timezone, whereas we care about buildmaster's
             # localtime (since this will get used to position the boxes on
-            # the Waterfall display, etc). So ignore the date field and use
-            # our local clock instead.
-            #when     = self._get_text(el, "date")
-            #when     = time.mktime(time.strptime("%.19s" % when,
-            #                                     "%Y-%m-%dT%H:%M:%S"))
+            # the Waterfall display, etc). So ignore the date field, and
+            # addChange will fill in with the current time
             branches = {}
             try:
                 pathlist = el.getElementsByTagName("paths")[0]
             except IndexError: # weird, we got an empty revision
                 log.msg("ignoring commit with no paths")
                 continue
 
             for p in pathlist.getElementsByTagName("path"):
@@ -495,41 +327,37 @@ class SVNPoller(base.ChangeSource, util.
             for branch in branches.keys():
                 action = branches[branch]['action']
                 files  = branches[branch]['files']
                 number_of_files_changed = len(files)
 
                 if action == u'D' and number_of_files_changed == 1 and files[0] == '':
                     log.msg("Ignoring deletion of branch '%s'" % branch)
                 else:
-                    c = Change(who=author,
-                               files=files,
-                               comments=comments,
-                               revision=revision,
-                               branch=branch,
-                               revlink=revlink,
-                               category=self.category,
-                               repository=self.svnurl,
-                               project = self.project)
-                    changes.append(c)
+                    chdict = dict(
+                            who=author,
+                            files=files,
+                            comments=comments,
+                            revision=revision,
+                            branch=branch,
+                            revlink=revlink,
+                            category=self.category,
+                            repository=self.svnurl,
+                            project = self.project)
+                    changes.append(chdict)
 
         return changes
 
+    @defer.deferredGenerator
     def submit_changes(self, changes):
-        for c in changes:
-            self.parent.addChange(c)
+        for chdict in changes:
+            wfd = defer.waitForDeferred(self.master.addChange(**chdict))
+            yield wfd
+            wfd.getResult()
 
     def finished_ok(self, res):
         if self.cachepath:
             f = open(self.cachepath, "w")
             f.write(str(self.last_change))
             f.close()
 
         log.msg("SVNPoller finished polling %s" % res)
-        assert self.working
-        self.working = False
         return res
-
-    def finished_failure(self, f):
-        log.msg("SVNPoller failed %s" % f)
-        assert self.working
-        self.working = False
-        return None # eat the failure
--- a/master/buildbot/clients/gtkPanes.py
+++ b/master/buildbot/clients/gtkPanes.py
@@ -22,17 +22,17 @@ import sys, time
 import pygtk #@UnresolvedImport
 pygtk.require("2.0")
 import gobject, gtk #@UnresolvedImport
 assert(gtk.Window) # in gtk1 it's gtk.GtkWindow
 
 from twisted.spread import pb
 
 #from buildbot.clients.base import Builder, Client
-from buildbot.clients.base import TextClient
+from buildbot.clients.base import TextClient, StatusClient
 from buildbot.util import now
 
 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE, EXCEPTION
 
 '''
 class Pane:
     def __init__(self):
         pass
@@ -492,18 +492,21 @@ class ThreeRowClient(pb.Referenceable):
     def remote_logFinished(self, buildername, build, stepname, step,
                            logname, log):
         pass
 
 
 class GtkClient(TextClient):
     ClientClass = ThreeRowClient
 
-    def __init__(self, master):
+    def __init__(self, master, events="steps", username="statusClient", passwd="clientpw"):
         self.master = master
+        self.username = username
+        self.passwd = passwd
+        self.listener = StatusClient(events)
 
         w = gtk.Window()
         self.w = w
         #w.set_size_request(64,64)
         w.connect('destroy', lambda win: gtk.main_quit())
         self.vb = gtk.VBox(False, 2)
         self.status = gtk.Label("unconnected")
         self.vb.add(self.status)
--- a/master/buildbot/clients/sendchange.py
+++ b/master/buildbot/clients/sendchange.py
@@ -40,25 +40,30 @@ class Sender:
         return d
 
     def addChange(self, remote, change):
         d = remote.callRemote('addChange', change)
         d.addCallback(lambda res: remote.broker.transport.loseConnection())
         return d
 
     def printSuccess(self, res):
+        print self.getSuccessString(res)
+
+    def getSuccessString(self, res):
         if self.num_changes > 1:
-            print "%d changes sent successfully" % self.num_changes
+            return "%d changes sent successfully" % self.num_changes
         elif self.num_changes == 1:
-            print "change sent successfully"
+            return "change sent successfully"
         else:
-            print "no changes to send"
+            return "no changes to send"
 
     def printFailure(self, why):
-        print "change(s) NOT sent, something went wrong:"
-        print why
+        print self.getFailureString(why)
+
+    def getFailureString(self, why):
+        return "change(s) NOT sent, something went wrong: " + str(why)
 
     def stop(self, res):
         reactor.stop()
         return res
 
     def run(self):
         reactor.run()
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/base.py
@@ -0,0 +1,33 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+"""
+Base classes for database handling
+"""
+
+class DBConnectorComponent(object):
+    """
+    A fixed component of the DBConnector, handling one particular aspect of the
+    database.  Instances of subclasses are assigned to attributes of the
+    DBConnector object, so that they are available at e.g., C{master.db.model}
+    or C{master.db.changes}.  This parent class takes care of the necessary
+    backlinks and other housekeeping.
+    """
+
+    connector = None
+
+    def __init__(self, connector):
+        self.db = connector
+        "backlink to the DBConnector object"
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/buildsets.py
@@ -0,0 +1,158 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+"""
+Support for buildsets in the database
+"""
+
+import time
+import sqlalchemy as sa
+from datetime import datetime
+from buildbot.util import json
+from buildbot.db import base
+
+class BuildsetsConnectorComponent(base.DBConnectorComponent):
+    """
+    A DBConnectorComponent to handle getting buildsets into and out of the
+    database.  An instance is available at C{master.db.buildsets}.
+    """
+
+    def addBuildset(self, ssid, reason, properties, builderNames,
+                   external_idstring=None):
+        """
+        Add a new Buildset to the database, along with the buildrequests for
+        each named builder, returning the resulting bsid via a Deferred.
+        Arguments should be specified by keyword.
+
+        @param ssid: id of the SourceStamp for this buildset
+        @type ssid: integer
+
+        @param reason: reason for this buildset
+        @type reason: short unicode string
+
+        @param properties: properties for this buildset
+        @type properties: L{buildbot.process.properties.Properties} instance,
+        or None
+
+        @param builderNames: builders specified by this buildset
+        @type builderNames: list of strings
+
+        @param external_idstring: external key to identify this buildset;
+        defaults to None
+        @type external_idstring: unicode string
+
+        @returns: buildset ID via a Deferred
+        """
+        def thd(conn):
+            submitted_at = datetime.now()
+            submitted_at_epoch = time.mktime(submitted_at.timetuple())
+
+            transaction = conn.begin()
+
+            # insert the buildset itself
+            r = conn.execute(self.db.model.buildsets.insert(), dict(
+                sourcestampid=ssid,
+                submitted_at=submitted_at_epoch,
+                reason=reason,
+                external_idstring=external_idstring))
+            bsid = r.inserted_primary_key[0]
+
+            # add any properties
+            if properties:
+                conn.execute(self.db.model.buildset_properties.insert(), [
+                    dict(buildsetid=bsid, property_name=k,
+                         property_value=json.dumps([v,s]))
+                    for (k,v,s) in properties.asList() ])
+
+            # and finish with a build request for each builder
+            conn.execute(self.db.model.buildrequests.insert(), [
+                dict(buildsetid=bsid, buildername=buildername,
+                     submitted_at=submitted_at_epoch)
+                for buildername in builderNames ])
+
+            transaction.commit()
+
+            return bsid
+        return self.db.pool.do(thd)
+
+    def subscribeToBuildset(self, schedulerid, buildsetid):
+        """
+        Add a row to C{scheduler_upstream_buildsets} indicating that
+        C{SCHEDULERID} is interested in buildset @C{BSID}.
+
+        @param schedulerid: downstream scheduler
+        @type schedulerid: integer
+
+        @param buildsetid: buildset id the scheduler is subscribing to
+        @type buildsetid: integer
+
+        @returns: Deferred
+        """
+        def thd(conn):
+            conn.execute(self.db.model.scheduler_upstream_buildsets.insert(),
+                    schedulerid=schedulerid,
+                    buildsetid=buildsetid,
+                    complete=0)
+        return self.db.pool.do(thd)
+
+    def unsubscribeFromBuildset(self, schedulerid, buildsetid):
+        """
+        The opposite of L{subscribeToBuildset}, this removes the subcription
+        row from the database, rather than simply marking it as inactive.
+
+        @param schedulerid: downstream scheduler
+        @type schedulerid: integer
+
+        @param buildsetid: buildset id the scheduler is subscribing to
+        @type buildsetid: integer
+
+        @returns: Deferred
+        """
+        def thd(conn):
+            tbl = self.db.model.scheduler_upstream_buildsets
+            conn.execute(tbl.delete(
+                    (tbl.c.schedulerid == schedulerid) &
+                    (tbl.c.buildsetid == buildsetid)))
+        return self.db.pool.do(thd)
+
+    def getSubscribedBuildsets(self, schedulerid):
+        """
+        Get the set of buildsets to which this scheduler is subscribed, along
+        with the buildsets' current results.  This will exclude any rows marked
+        as not active.
+
+        The return value is a list of tuples, each containing a buildset ID, a
+        sourcestamp ID, a boolean indicating that the buildset is complete, and
+        the buildset's result.
+
+        @param schedulerid: downstream scheduler
+        @type schedulerid: integer
+
+        @returns: list as described, via Deferred
+        """
+        def thd(conn):
+            bs_tbl = self.db.model.buildsets
+            upstreams_tbl = self.db.model.scheduler_upstream_buildsets
+            q = sa.select(
+                [bs_tbl.c.id, bs_tbl.c.sourcestampid,
+                 bs_tbl.c.results, bs_tbl.c.complete],
+                whereclause=(
+                    (upstreams_tbl.c.schedulerid == schedulerid) &
+                    (upstreams_tbl.c.buildsetid == bs_tbl.c.id) &
+                    (upstreams_tbl.c.active)),
+                distinct=True)
+            return [ (row.id, row.sourcestampid, row.complete, row.results)
+                     for row in conn.execute(q).fetchall() ]
+        return self.db.pool.do(thd)
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/changes.py
@@ -0,0 +1,333 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+"""
+Support for changes in the database
+"""
+
+import sys
+import Queue
+from buildbot.util import json
+import sqlalchemy as sa
+from twisted.python import log
+from buildbot.changes.changes import Change
+from buildbot.db import base
+from buildbot import util
+
+class ChangesConnectorComponent(base.DBConnectorComponent):
+    """
+    A DBConnectorComponent to handle getting changes into and out of the
+    database.  An instance is available at C{master.db.changes}.
+    """
+
+    changeHorizon = 0
+    "maximum number of changes to keep on hand, or 0 to keep all changes forever"
+    # TODO: add a threadsafe change cache
+
+    def changeEventGenerator(self, branches=[], categories=[], committers=[], minTime=0):
+        "deprecated. do not use."
+        # the technique here is to use a queue and a boolean value to
+        # communicate between the db thread and the main thread.  The queue
+        # sends completed change tuples to the main thread, while the boolean
+        # value indicates that the thread should exit.
+        #
+        # If this wacky hack seems sensible to you, you're not looking hard
+        # enough!  Once the web UI is rewritten (waterfall and console are the only
+        # consumers of the generator), this should be killed with fire.
+
+        queue = Queue.Queue(16)
+        stop_flag = [ False ]
+
+        def thd(conn):
+            try:
+                changes_tbl = self.db.model.changes
+
+                query = changes_tbl.select()
+                if branches:
+                    query = query.where(changes_tbl.c.branch.in_(branches))
+                if categories:
+                    query = query.where(changes_tbl.c.category.in_(categories))
+                if committers:
+                    query = query.where(changes_tbl.c.author.in_(committers))
+                if minTime:
+                    query = query.where(changes_tbl.c.when_timestamp > minTime)
+                query = query.order_by(sa.desc(changes_tbl.c.changeid))
+                change_rows = conn.execute(query)
+                for ch_row in change_rows:
+                    chdict = self._chdict_from_change_row_thd(conn, ch_row)
+                    # bail out if we've been asked to stop
+                    if stop_flag[0]:
+                        break
+                    queue.put(chdict)
+                queue.put(None)
+            except:
+                # push exceptions onto the queue and return
+                queue.put(sys.exc_info())
+        d = self.db.pool.do(thd)
+
+        # note that we don't actually look at the results of this deferred.  If
+        # an error occurs in the thread, it is handled by returning a tuple
+        # instead.  Still, we might as well handle any exceptions that get
+        # raised into failures
+        d.addErrback(log.err)
+
+        try:
+            while True:
+                chdict = queue.get()
+                if chdict is None:
+                    # we've seen all of the changes
+                    break
+                if isinstance(chdict, tuple):
+                    # exception in thread; raise it here
+                    raise chdict[0], chdict[1], chdict[2]
+                else:
+                    yield self._change_from_chdict(chdict)
+        # we'll get GeneratorExit when the generator is garbage-collected before it
+        # has finished, so signal to the thread that its work is finished.
+        # TODO: GeneratorExit is not supported in Python-2.4, which means this method
+        # won't work there.  Which is OK.  This method needs to die, quickly.
+        except GeneratorExit:
+            stop_flag[0] = False
+            # .. and drain the queue
+            while not queue.empty():
+                queue.get()
+
+    def addChange(self, who, files, comments, isdir=0, links=None,
+                 revision=None, when=None, branch=None, category=None,
+                 revlink='', properties={}, repository='', project=''):
+        """Add the a Change with the given attributes to the database; returns
+        a Change instance via a deferred.
+
+        @param who: the author of this change
+        @type branch: unicode string
+
+        @param files: a list of filenames that were changed
+        @type branch: list of unicode strings
+
+        @param comments: user comments on the change
+        @type branch: unicode string
+
+        @param isdir: deprecated
+
+        @param links: a list of links related to this change, e.g., to web viewers
+        or review pages
+        @type links: list of unicode strings
+
+        @param revision: the revision identifier for this change
+        @type revision: unicode string
+
+        @param when: when this change occurs; defaults to now, and cannot be later
+        than now
+        @type when: integer (UNIX epoch time)
+
+        @param branch: the branch on which this change took place
+        @type branch: unicode string
+
+        @param category: category for this change (arbitrary use by Buildbot users)
+        @type category: unicode string
+
+        @param revlink: link to a web view of this revision
+        @type revlink: unicode string
+
+        @param properties: properties to set on this change
+        @type properties: dictionary with string keys and simple values (JSON-able)
+
+        @param repository: the repository in which this change took place
+        @type repository: unicode string
+
+        @param project: the project this change is a part of
+        @type project: unicode string
+
+        @returns: a L{buildbot.changes.changes.Change} instance via Deferred
+        """
+        # first create the change, although with no 'number'
+        change = Change(who=who, files=files, comments=comments, isdir=isdir,
+                links=links, revision=revision, when=when, branch=branch,
+                category=category, revlink=revlink, properties=properties,
+                repository=repository, project=project)
+
+        # then add it to the database and update its '.number'
+        def thd(conn):
+            assert change.number is None
+            ins = self.db.model.changes.insert()
+            r = conn.execute(ins, dict(
+                author=change.who,
+                comments=change.comments,
+                is_dir=change.isdir,
+                branch=change.branch,
+                revision=change.revision,
+                revlink=change.revlink,
+                when_timestamp=change.when,
+                category=change.category,
+                repository=change.repository,
+                project=change.project))
+            change.number = r.inserted_primary_key[0]
+            if change.links:
+                ins = self.db.model.change_links.insert()
+                conn.execute(ins, [
+                    dict(changeid=change.number, link=l)
+                        for l in change.links
+                    ])
+            if change.files:
+                ins = self.db.model.change_files.insert()
+                conn.execute(ins, [
+                    dict(changeid=change.number, filename=f)
+                        for f in change.files
+                    ])
+            if change.properties:
+                ins = self.db.model.change_properties.insert()
+                conn.execute(ins, [
+                    dict(changeid=change.number, property_name=k, property_value=json.dumps(v))
+                        for k,v,s in change.properties.asList()
+                    ])
+            return change
+        d = self.db.pool.do(thd)
+        # prune changes, if necessary
+        d.addCallback(lambda _ : self._prune_changes(change.number))
+        # return the change
+        d.addCallback(lambda _ : change)
+        return d
+
+    def getChangeInstance(self, changeid):
+        """
+        Get a L{buildbot.changes.changes.Change} instance for the given changeid,
+        or None if no such change exists.
+
+        @param changeid: the id of the change instance to fetch
+
+        @returns: Change instance via Deferred
+        """
+        assert changeid >= 0
+        def thd(conn):
+            # get the row from the 'changes' table
+            changes_tbl = self.db.model.changes
+            q = changes_tbl.select(whereclause=(changes_tbl.c.changeid == changeid))
+            rp = conn.execute(q)
+            row = rp.fetchone()
+            if not row:
+                return None
+            # and fetch the ancillary data (links, files, properties)
+            return self._chdict_from_change_row_thd(conn, row)
+        d = self.db.pool.do(thd)
+
+        def make_change(chdict):
+            if not chdict:
+                return None
+            return self._change_from_chdict(chdict)
+        d.addCallback(make_change)
+        return d
+
+    def getLatestChangeid(self):
+        """
+        Get the most-recently-assigned changeid, or None if there are no
+        changes at all.
+
+        @returns: changeid via Deferred
+        """
+        def thd(conn):
+            changes_tbl = self.db.model.changes
+            q = sa.select([ changes_tbl.c.changeid ],
+                    order_by=sa.desc(changes_tbl.c.changeid),
+                    limit=1)
+            return conn.scalar(q)
+        d = self.db.pool.do(thd)
+        return d
+
+    def setChangeHorizon(self, changeHorizon): # TODO: remove
+        "this method should go away"
+        self.changeHorizon = changeHorizon
+
+    # cache management
+
+    def _flush_cache(self):
+        pass # TODO
+
+    # utility methods
+
+    _last_prune = 0
+    def _prune_changes(self, last_added_changeid):
+        # this is an expensive operation, so only do it once per minute, in case
+        # addChange is called frequently
+        if not self.changeHorizon or self._last_prune > util.now() - 60:
+            return
+        self._last_prune = util.now()
+        log.msg("pruning changes")
+
+        def thd(conn):
+            changes_tbl = self.db.model.changes
+            current_horizon = last_added_changeid - self.changeHorizon
+
+            # create a subquery giving the changes to delete
+            ids_to_delete_query = sa.select([changes_tbl.c.changeid],
+                                    whereclause=changes_tbl.c.changeid <= current_horizon)
+
+            # and delete from all relevant tables, *ending* with the changes table
+            for table_name in ('scheduler_changes', 'sourcestamp_changes', 'change_files',
+                               'change_links', 'change_properties', 'changes'):
+                table = self.db.model.metadata.tables[table_name]
+                conn.execute(
+                    table.delete(table.c.changeid.in_(ids_to_delete_query)))
+        return self.db.pool.do(thd)
+
+    def _chdict_from_change_row_thd(self, conn, ch_row):
+        # This method must be run in a db.pool thread, and returns a chdict
+        # (which can be used to construct a Change object), given a row from
+        # the 'changes' table
+        change_links_tbl = self.db.model.change_links
+        change_files_tbl = self.db.model.change_files
+        change_properties_tbl = self.db.model.change_properties
+
+        chdict = dict(
+                number=ch_row.changeid,
+                who=ch_row.author,
+                files=[], # see below
+                comments=ch_row.comments,
+                isdir=ch_row.is_dir,
+                links=[], # see below
+                revision=ch_row.revision,
+                when=ch_row.when_timestamp,
+                branch=ch_row.branch,
+                category=ch_row.category,
+                revlink=ch_row.revlink,
+                properties={}, # see below
+                repository=ch_row.repository,
+                project=ch_row.project)
+
+        query = change_links_tbl.select(
+                whereclause=(change_links_tbl.c.changeid == ch_row.changeid))
+        rows = conn.execute(query)
+        for r in rows:
+            chdict['links'].append(r.link)
+
+        query = change_files_tbl.select(
+                whereclause=(change_files_tbl.c.changeid == ch_row.changeid))
+        rows = conn.execute(query)
+        for r in rows:
+            chdict['files'].append(r.filename)
+
+        query = change_properties_tbl.select(
+                whereclause=(change_properties_tbl.c.changeid == ch_row.changeid))
+        rows = conn.execute(query)
+        for r in rows:
+            chdict['properties'][r.property_name] = json.loads(r.property_value)
+
+        return chdict
+
+    def _change_from_chdict(self, chdict):
+        # create a Change object, given a chdict
+        changeid = chdict.pop('number')
+        c = Change(**chdict)
+        c.number = changeid
+        return c
--- a/master/buildbot/db/connector.py
+++ b/master/buildbot/db/connector.py
@@ -8,164 +8,183 @@
 # details.
 #
 # You should have received a copy of the GNU General Public License along with
 # this program; if not, write to the Free Software Foundation, Inc., 51
 # Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 #
 # Copyright Buildbot Team Members
 
-import sys, collections, base64
+import collections, base64
 
 from twisted.python import log, threadable
-from twisted.internet import defer
-from twisted.enterprise import adbapi
+from buildbot.db import enginestrategy
+
 from buildbot import util
 from buildbot.util import collections as bbcollections
-from buildbot.changes.changes import Change
 from buildbot.sourcestamp import SourceStamp
 from buildbot.buildrequest import BuildRequest
 from buildbot.process.properties import Properties
 from buildbot.status.builder import SUCCESS, WARNINGS, FAILURE
 from buildbot.util.eventual import eventually
 from buildbot.util import json
-
-# Don't auto-resubmit queries that encounter a broken connection: let them
-# fail. Use the "notification doorbell" thing to provide the retry. Set
-# cp_reconnect=True, so that a connection failure will prepare the
-# ConnectionPool to reconnect next time.
-
-class MyTransaction(adbapi.Transaction):
-    def execute(self, *args, **kwargs):
-        #print "Q", args, kwargs
-        return self._cursor.execute(*args, **kwargs)
-    def fetchall(self):
-        rc = self._cursor.fetchall()
-        #print " F", rc
-        return rc
+from buildbot.db import pool, model, changes, schedulers, sourcestamps, buildsets
 
 def _one_or_else(res, default=None, process_f=lambda x: x):
     if not res:
         return default
     return process_f(res[0][0])
 
 def str_or_none(s):
     if s is None:
         return None
     return str(s)
 
 class Token: # used for _start_operation/_end_operation
     pass
 
-class DBConnector(util.ComparableMixin):
-    # this will refuse to create the database: use 'create-master' for that
-    compare_attrs = ["args", "kwargs"]
+from twisted.enterprise import adbapi
+class TempAdbapiPool(adbapi.ConnectionPool):
+    def __init__(self, engine):
+        # this wants a module name, so give it one..
+        adbapi.ConnectionPool.__init__(self, "buildbot.db.connector")
+        self._engine = engine
+
+    def connect(self):
+        return self._engine.raw_connection()
+
+    def stop(self):
+        pass
+
+class DBConnector(object):
+    """
+    The connection between Buildbot and its backend database.  This is
+    generally accessible as master.db, but is also used during upgrades.
+
+    Most of the interesting operations available via the connector are
+    implemented in connector components, available as attributes of this
+    object, and listed below.
+    """
+
     synchronized = ["notify", "_end_operation"]
     MAX_QUERY_TIMES = 1000
 
-    def __init__(self, spec):
-        # typical args = (dbmodule, dbname, username, password)
-        self._query_times = collections.deque()
-        self._spec = spec
-
-        # this is for synchronous calls: runQueryNow, runInteractionNow
-        self._dbapi = spec.get_dbapi()
-        self._nonpool = None
-        self._nonpool_lastused = None
-        self._nonpool_max_idle = spec.get_maxidle()
+    def __init__(self, db_url, basedir):
+        self.basedir = basedir
+        "basedir for this master - used for upgrades"
 
-        # pass queries in with "?" placeholders. If the backend uses a
-        # different style, we'll replace them.
-        self.paramstyle = self._dbapi.paramstyle
+        self._engine = enginestrategy.create_engine(db_url, basedir=self.basedir)
+        self.pool = pool.DBThreadPool(self._engine)
+        "thread pool (L{buildbot.db.pool.DBThreadPool}) for this db"
 
-        self._pool = spec.get_async_connection_pool()
-        self._pool.transactionFactory = MyTransaction
-        # the pool must be started before it can be used. The real
-        # buildmaster process will do this at reactor start. CLI tools (like
-        # "buildbot upgrade-master") must do it manually. Unit tests are run
-        # in an environment in which it is already started.
+        self._oldpool = TempAdbapiPool(self._engine)
 
-        self._change_cache = util.LRUCache()
+        self._query_times = collections.deque()
+
+        self._change_cache = util.LRUCache() # TODO: remove
         self._sourcestamp_cache = util.LRUCache()
         self._active_operations = set() # protected by synchronized=
         self._pending_notifications = []
         self._subscribers = bbcollections.defaultdict(set)
 
         self._pending_operation_count = 0
 
         self._started = False
 
-    def _getCurrentTime(self):
+        # set up components
+        self.model = model.Model(self)
+        "L{buildbot.db.model.Model} instance"
+
+        self.changes = changes.ChangesConnectorComponent(self)
+        "L{buildbot.db.changes.ChangesConnectorComponent} instance"
+
+        self.schedulers = schedulers.SchedulersConnectorComponent(self)
+        "L{buildbot.db.schedulers.ChangesConnectorComponent} instance"
+
+        self.sourcestamps = sourcestamps.SourceStampsConnectorComponent(self)
+        "L{buildbot.db.sourcestamps.SourceStampsConnectorComponent} instance"
+
+        self.buildsets = buildsets.BuildsetsConnectorComponent(self)
+        "L{buildbot.db.sourcestamps.BuildsetsConnectorComponent} instance"
+
+
+    def _getCurrentTime(self): # TODO: remove
         # this is a seam for use in testing
         return util.now()
 
-    def start(self):
+    def start(self): # TODO: remove
         # this only *needs* to be called in reactorless environments (which
         # should be eliminated anyway).  but it doesn't hurt anyway
-        self._pool.start()
+        self._oldpool.start()
         self._started = True
 
-    def stop(self):
+    def stop(self): # TODO: remove
         """Call this when you're done with me"""
 
-        # Close our synchronous connection if we've got one
-        if self._nonpool:
-            self._nonpool.close()
-            self._nonpool = None
-            self._nonpool_lastused = None
-
         if not self._started:
             return
-        self._pool.close()
+        self._oldpool.stop()
         self._started = False
-        del self._pool
+        del self._oldpool
 
-    def quoteq(self, query):
+    def quoteq(self, query, returning=None): # TODO: remove
         """
         Given a query that contains qmark-style placeholders, like::
          INSERT INTO foo (col1, col2) VALUES (?,?)
         replace the '?' with '%s' if the backend uses format-style
         placeholders, like::
          INSERT INTO foo (col1, col2) VALUES (%s,%s)
+
+        While there, append "RETURNING x" for backends that don't provide
+        last row id (PostgreSQL and probably Oracle).
         """
-        if self.paramstyle == "format":
-            return query.replace("?","%s")
-        assert self.paramstyle == "qmark"
+        # PostgreSQL:
+        # * doesn't return last row id, so we must append "RETURNING x"
+        #   to queries where we want it and we must fetch it later,
+        # * doesn't accept "?" in queries.
+        if self._engine.dialect.name in ('postgres', 'postgresql'):
+            if returning:
+                query += " RETURNING %s" % returning
+            return query.replace("?", "%s")
+
+        # default
         return query
 
-    def parmlist(self, count):
+    def lastrowid(self, t): # TODO: remove
+        # PostgreSQL:
+        # * fetch last row id from previously issued "RETURNING x" query.
+        if self._engine.dialect.name in ('postgres', 'postgresql'):
+            row = t.fetchone()
+            if row:
+                 return row[0]
+            return -1
+
+        # default
+        return t.lastrowid
+
+    def parmlist(self, count): # TODO: remove
         """
         When passing long lists of values to e.g., an INSERT query, it is
         tedious to pass long strings of ? placeholders.  This function will
         create a parenthesis-enclosed list of COUNT placeholders.  Note that
         the placeholders have already had quoteq() applied.
         """
         p = self.quoteq("?")
         return "(" + ",".join([p]*count) + ")"
 
-    def get_version(self):
-        """Returns None for an empty database, or a number (probably 1) for
-        the database's version"""
-        try:
-            res = self.runQueryNow("SELECT version FROM version")
-        except (self._dbapi.OperationalError, self._dbapi.ProgrammingError):
-            # this means the version table is missing: the db is empty
-            return None
-        assert len(res) == 1
-        return res[0][0]
-
-    def runQueryNow(self, *args, **kwargs):
+    def runQueryNow(self, *args, **kwargs): # TODO: remove
         # synchronous+blocking version of runQuery()
         assert self._started
         return self.runInteractionNow(self._runQuery, *args, **kwargs)
 
     def _runQuery(self, c, *args, **kwargs):
         c.execute(*args, **kwargs)
         return c.fetchall()
 
+    # TODO: remove
     def _start_operation(self):
         t = Token()
         self._active_operations.add(t)
         return t
     def _end_operation(self, t):
         # this is always invoked from the main thread, but is wrapped by
         # synchronized= and threadable.synchronous(), since it touches
         # self._pending_notifications, which is also touched by
@@ -174,291 +193,89 @@ class DBConnector(util.ComparableMixin):
         if self._active_operations:
             return
         for (category, args) in self._pending_notifications:
             # in the distributed system, this will be a
             # transport.write(" ".join([category] + [str(a) for a in args]))
             eventually(self.send_notification, category, args)
         self._pending_notifications = []
 
-    def runInteractionNow(self, interaction, *args, **kwargs):
+    def runInteractionNow(self, interaction, *args, **kwargs): # TODO: remove
         # synchronous+blocking version of runInteraction()
         assert self._started
         start = self._getCurrentTime()
         t = self._start_operation()
         try:
             return self._runInteractionNow(interaction, *args, **kwargs)
         finally:
             self._end_operation(t)
             self._add_query_time(start)
 
-    def get_sync_connection(self):
-        # This is a wrapper around spec.get_sync_connection that maintains a
-        # single connection to the database for synchronous usage.  It will get
-        # a new connection if the existing one has been idle for more than
-        # max_idle seconds.
-        if self._nonpool_max_idle is not None:
-            now = util.now()
-            if self._nonpool_lastused and self._nonpool_lastused + self._nonpool_max_idle < now:
-                self._nonpool = None
+    def get_sync_connection(self): # TODO: remove
+        # TODO: SYNC CONNECTIONS MUST DIE
+        return self._engine.raw_connection()
 
-        if not self._nonpool:
-            self._nonpool = self._spec.get_sync_connection()
-
-        self._nonpool_lastused = util.now()
-        return self._nonpool
-
-    def _runInteractionNow(self, interaction, *args, **kwargs):
+    def _runInteractionNow(self, interaction, *args, **kwargs): # TODO: remove
         conn = self.get_sync_connection()
         c = conn.cursor()
-        try:
-            result = interaction(c, *args, **kwargs)
-            c.close()
-            conn.commit()
-            return result
-        except:
-            excType, excValue, excTraceback = sys.exc_info()
-            try:
-                conn.rollback()
-                c2 = conn.cursor()
-                c2.execute(self._pool.good_sql)
-                c2.close()
-                conn.commit()
-            except:
-                log.msg("rollback failed, will reconnect next query")
-                log.err()
-                # and the connection is probably dead: clear the reference,
-                # so we'll establish a new connection next time
-                self._nonpool = None
-            raise excType, excValue, excTraceback
+        result = interaction(c, *args, **kwargs)
+        c.close()
+        conn.commit()
+        return result
 
-    def notify(self, category, *args):
+    def notify(self, category, *args): # TODO: remove
         # this is wrapped by synchronized= and threadable.synchronous(),
         # since it will be invoked from runInteraction threads
         self._pending_notifications.append( (category,args) )
 
-    def send_notification(self, category, args):
+    def send_notification(self, category, args): # TODO: remove
         # in the distributed system, this will be invoked by lineReceived()
         #print "SEND", category, args
         for observer in self._subscribers[category]:
             eventually(observer, category, *args)
 
-    def subscribe_to(self, category, observer):
+    def subscribe_to(self, category, observer): # TODO: remove
         self._subscribers[category].add(observer)
 
-    def runQuery(self, *args, **kwargs):
+    def runQuery(self, *args, **kwargs): # TODO: remove
         assert self._started
         self._pending_operation_count += 1
-        d = self._pool.runQuery(*args, **kwargs)
+        d = self._oldpool.runQuery(*args, **kwargs)
         return d
 
-    def _runQuery_done(self, res, start, t):
-        self._end_operation(t)
-        self._add_query_time(start)
-        self._pending_operation_count -= 1
-        return res
-
-    def _add_query_time(self, start):
-        elapsed = self._getCurrentTime() - start
-        self._query_times.append(elapsed)
-        if len(self._query_times) > self.MAX_QUERY_TIMES:
-            self._query_times.popleft()
-
-    def runInteraction(self, *args, **kwargs):
-        assert self._started
-        self._pending_operation_count += 1
-        start = self._getCurrentTime()
-        t = self._start_operation()
-        d = self._pool.runInteraction(*args, **kwargs)
-        d.addBoth(self._runInteraction_done, start, t)
-        return d
-    def _runInteraction_done(self, res, start, t):
+    def _runQuery_done(self, res, start, t): # TODO: remove
         self._end_operation(t)
         self._add_query_time(start)
         self._pending_operation_count -= 1
         return res
 
-    # ChangeManager methods
-
-    def addChangeToDatabase(self, change):
-        self.runInteractionNow(self._txn_addChangeToDatabase, change)
-        self._change_cache.add(change.number, change)
-
-    def _txn_addChangeToDatabase(self, t, change):
-        q = self.quoteq("INSERT INTO changes"
-                        " (author,"
-                        "  comments, is_dir,"
-                        "  branch, revision, revlink,"
-                        "  when_timestamp, category,"
-                        "  repository, project)"
-                        " VALUES (?, ?,?, ?,?,?, ?,?, ?,?)")
-        # TODO: map None to.. empty string?
-
-        values = (change.who,
-                  change.comments, change.isdir,
-                  change.branch, change.revision, change.revlink,
-                  change.when, change.category, change.repository,
-                  change.project)
-        t.execute(q, values)
-        change.number = t.lastrowid
-
-        for link in change.links:
-            t.execute(self.quoteq("INSERT INTO change_links (changeid, link) "
-                                  "VALUES (?,?)"),
-                      (change.number, link))
-        for filename in change.files:
-            t.execute(self.quoteq("INSERT INTO change_files (changeid,filename)"
-                                  " VALUES (?,?)"),
-                      (change.number, filename))
-        for propname,propvalue in change.properties.properties.items():
-            encoded_value = json.dumps(propvalue)
-            t.execute(self.quoteq("INSERT INTO change_properties"
-                                  " (changeid, property_name, property_value)"
-                                  " VALUES (?,?,?)"),
-                      (change.number, propname, encoded_value))
-        self.notify("add-change", change.number)
-
-    def changeEventGenerator(self, branches=[], categories=[], committers=[], minTime=0):
-        q = "SELECT changeid FROM changes"
-        args = []
-        if branches or categories or committers:
-            q += " WHERE "
-            pieces = []
-            if branches:
-                pieces.append("branch IN %s" % self.parmlist(len(branches)))
-                args.extend(list(branches))
-            if categories:
-                pieces.append("category IN %s" % self.parmlist(len(categories)))
-                args.extend(list(categories))
-            if committers:
-                pieces.append("author IN %s" % self.parmlist(len(committers)))
-                args.extend(list(committers))
-            if minTime:
-                pieces.append("when_timestamp > %d" % minTime)
-            q += " AND ".join(pieces)
-        q += " ORDER BY changeid DESC"
-        rows = self.runQueryNow(q, tuple(args))
-        for (changeid,) in rows:
-            yield self.getChangeNumberedNow(changeid)
-
-    def getLatestChangeNumberNow(self, branch=None, t=None):
-        if t:
-            return self._txn_getLatestChangeNumber(branch=branch, t=t)
-        else:
-            return self.runInteractionNow(self._txn_getLatestChangeNumber)
-    def _txn_getLatestChangeNumber(self, branch, t):
-        args = None
-        if branch:
-            br_clause = "WHERE branch =? "
-            args = ( branch, )
-        q = self.quoteq("SELECT max(changeid) from changes"+ br_clause)
-        t.execute(q, args)
-        row = t.fetchone()
-        if not row:
-            return 0
-        return row[0]
+    def _add_query_time(self, start): # TODO: remove
+        elapsed = self._getCurrentTime() - start
+        self._query_times.append(elapsed)
+        if len(self._query_times) > self.MAX_QUERY_TIMES:
+            self._query_times.popleft()
 
-    def getChangeNumberedNow(self, changeid, t=None):
-        # this is a synchronous/blocking version of getChangeByNumber
-        assert changeid >= 0
-        c = self._change_cache.get(changeid)
-        if c:
-            return c
-        if t:
-            c = self._txn_getChangeNumberedNow(t, changeid)
-        else:
-            c = self.runInteractionNow(self._txn_getChangeNumberedNow, changeid)
-        self._change_cache.add(changeid, c)
-        return c
-    def _txn_getChangeNumberedNow(self, t, changeid):
-        q = self.quoteq("SELECT author, comments,"
-                        " is_dir, branch, revision, revlink,"
-                        " when_timestamp, category,"
-                        " repository, project"
-                        " FROM changes WHERE changeid = ?")
-        t.execute(q, (changeid,))
-        rows = t.fetchall()
-        if not rows:
-            return None
-        (who, comments,
-         isdir, branch, revision, revlink,
-         when, category, repository, project) = rows[0]
-        branch = str_or_none(branch)
-        revision = str_or_none(revision)
-        q = self.quoteq("SELECT link FROM change_links WHERE changeid=?")
-        t.execute(q, (changeid,))
-        rows = t.fetchall()
-        links = [row[0] for row in rows]
-        links.sort()
-
-        q = self.quoteq("SELECT filename FROM change_files WHERE changeid=?")
-        t.execute(q, (changeid,))
-        rows = t.fetchall()
-        files = [row[0] for row in rows]
-        files.sort()
+    def runInteraction(self, *args, **kwargs): # TODO: remove
+        assert self._started
+        self._pending_operation_count += 1
+        start = self._getCurrentTime()
+        t = self._start_operation()
+        d = self._oldpool.runInteraction(*args, **kwargs)
+        d.addBoth(self._runInteraction_done, start, t)
+        return d
+    def _runInteraction_done(self, res, start, t): # TODO: remove
+        self._end_operation(t)
+        self._add_query_time(start)
+        self._pending_operation_count -= 1
+        return res
 
-        p = self.get_properties_from_db("change_properties", "changeid",
-                                        changeid, t)
-        c = Change(who=who, files=files, comments=comments, isdir=isdir,
-                   links=links, revision=revision, when=when,
-                   branch=branch, category=category, revlink=revlink,
-                   repository=repository, project=project)
-        c.properties.updateFromProperties(p)
-        c.number = changeid
-        return c
-
-    def getChangeByNumber(self, changeid):
-        # return a Deferred that fires with a Change instance, or None if
-        # there is no Change with that number
-        assert changeid >= 0
-        c = self._change_cache.get(changeid)
-        if c:
-            return defer.succeed(c)
-        d1 = self.runQuery(self.quoteq("SELECT author, comments,"
-                                       " is_dir, branch, revision, revlink,"
-                                       " when_timestamp, category,"
-                                       " repository, project"
-                                       " FROM changes WHERE changeid = ?"),
-                           (changeid,))
-        d2 = self.runQuery(self.quoteq("SELECT link FROM change_links"
-                                       " WHERE changeid=?"),
-                           (changeid,))
-        d3 = self.runQuery(self.quoteq("SELECT filename FROM change_files"
-                                       " WHERE changeid=?"),
-                           (changeid,))
-        d4 = self.runInteraction(self._txn_get_properties_from_db,
-                "change_properties", "changeid", changeid)
-        d = defer.gatherResults([d1,d2,d3,d4])
-        d.addCallback(self._getChangeByNumber_query_done, changeid)
-        return d
-
-    def _getChangeByNumber_query_done(self, res, changeid):
-        (rows, link_rows, file_rows, properties) = res
-        if not rows:
-            return None
-        (who, comments,
-         isdir, branch, revision, revlink,
-         when, category, repository, project) = rows[0]
-        branch = str_or_none(branch)
-        revision = str_or_none(revision)
-        links = [row[0] for row in link_rows]
-        links.sort()
-        files = [row[0] for row in file_rows]
-        files.sort()
-
-        c = Change(who=who, files=files, comments=comments, isdir=isdir,
-                   links=links, revision=revision, when=when,
-                   branch=branch, category=category, revlink=revlink,
-                   repository=repository, project=project)
-        c.properties.updateFromProperties(properties)
-        c.number = changeid
-        self._change_cache.add(changeid, c)
-        return c
+    # old ChangeManager methods
 
     def getChangesGreaterThan(self, last_changeid, t=None):
+        # LIES LIES LIES!
         """Return a Deferred that fires with a list of all Change instances
         with numbers greater than the given value, sorted by number. This is
         useful for catching up with everything that's happened since you last
         called this function."""
         assert last_changeid >= 0
         if t:
             return self._txn_getChangesGreaterThan(t, last_changeid)
         else:
@@ -467,41 +284,26 @@ class DBConnector(util.ComparableMixin):
     def _txn_getChangesGreaterThan(self, t, last_changeid):
         q = self.quoteq("SELECT changeid FROM changes WHERE changeid > ?")
         t.execute(q, (last_changeid,))
         changes = [self.getChangeNumberedNow(changeid, t)
                    for (changeid,) in t.fetchall()]
         changes.sort(key=lambda c: c.number)
         return changes
 
-    def getChangeIdsLessThanIdNow(self, new_changeid):
-        """Return a list of all extant change id's less than the given value,
-        sorted by number."""
-        def txn(t):
-            q = self.quoteq("SELECT changeid FROM changes WHERE changeid < ?")
-            t.execute(q, (new_changeid,))
-            changes = [changeid for (changeid,) in t.fetchall()]
-            changes.sort()
-            return changes
-        return self.runInteractionNow(txn)
-
     def removeChangeNow(self, changeid):
         """Thoroughly remove a change from the database, including all dependent
         tables"""
         def txn(t):
             for table in ('changes', 'scheduler_changes', 'sourcestamp_changes',
                           'change_files', 'change_links', 'change_properties'):
                 q = self.quoteq("DELETE FROM %s WHERE changeid = ?" % table)
                 t.execute(q, (changeid,))
         return self.runInteractionNow(txn)
 
-    def getChangesByNumber(self, changeids):
-        return defer.gatherResults([self.getChangeByNumber(changeid)
-                                    for changeid in changeids])
-
     # SourceStamp-manipulating methods
 
     def getSourceStampNumberedNow(self, ssid, t=None):
         assert isinstance(ssid, (int, long))
         ss = self._sourcestamp_cache.get(ssid)
         if ss:
             return ss
         if t:
@@ -612,19 +414,19 @@ class DBConnector(util.ComparableMixin):
                 q = ("SELECT changeid FROM changes"
                      " ORDER BY changeid DESC LIMIT 1")
                 t.execute(q)
                 max_changeid = _one_or_else(t.fetchall(), 0)
                 state = scheduler.get_initial_state(max_changeid)
                 state_json = json.dumps(state)
                 q = self.quoteq("INSERT INTO schedulers"
                                 " (name, class_name, state)"
-                                "  VALUES (?,?,?)")
+                                "  VALUES (?,?,?)", "schedulerid")
                 t.execute(q, (name, class_name, state_json))
-                sid = t.lastrowid
+                sid = self.lastrowid(t)
             log.msg("scheduler '%s' got id %d" % (scheduler.name, sid))
             scheduler.schedulerid = sid
 
     def scheduler_get_state(self, schedulerid, t):
         q = self.quoteq("SELECT state FROM schedulers WHERE schedulerid=?")
         t.execute(q, (schedulerid,))
         state_json = _one_or_else(t.fetchall())
         assert state_json is not None
@@ -646,53 +448,53 @@ class DBConnector(util.ComparableMixin):
         if ss.patch:
             patchlevel = ss.patch[0]
             diff = ss.patch[1]
             subdir = None
             if len(ss.patch) > 2:
                 subdir = ss.patch[2]
             q = self.quoteq("INSERT INTO patches"
                             " (patchlevel, patch_base64, subdir)"
-                            " VALUES (?,?,?)")
+                            " VALUES (?,?,?)", "id")
             t.execute(q, (patchlevel, base64.b64encode(diff), subdir))
-            patchid = t.lastrowid
+            patchid = self.lastrowid(t)
         t.execute(self.quoteq("INSERT INTO sourcestamps"
                               " (branch, revision, patchid, project, repository)"
-                              " VALUES (?,?,?,?,?)"),
+                              " VALUES (?,?,?,?,?)", "id"),
                   (ss.branch, ss.revision, patchid, ss.project, ss.repository))
-        ss.ssid = t.lastrowid
+        ss.ssid = self.lastrowid(t)
         q2 = self.quoteq("INSERT INTO sourcestamp_changes"
                          " (sourcestampid, changeid) VALUES (?,?)")
         for c in ss.changes:
             t.execute(q2, (ss.ssid, c.number))
         return ss.ssid
 
     def create_buildset(self, ssid, reason, properties, builderNames, t,
                         external_idstring=None):
         # this creates both the BuildSet and the associated BuildRequests
         now = self._getCurrentTime()
         t.execute(self.quoteq("INSERT INTO buildsets"
                               " (external_idstring, reason,"
                               "  sourcestampid, submitted_at)"
-                              " VALUES (?,?,?,?)"),
+                              " VALUES (?,?,?,?)", "id"),
                   (external_idstring, reason, ssid, now))
-        bsid = t.lastrowid
+        bsid = self.lastrowid(t)
         for propname, propvalue in properties.properties.items():
             encoded_value = json.dumps(propvalue)
             t.execute(self.quoteq("INSERT INTO buildset_properties"
                                   " (buildsetid, property_name, property_value)"
                                   " VALUES (?,?,?)"),
                       (bsid, propname, encoded_value))
         brids = []
         for bn in builderNames:
             t.execute(self.quoteq("INSERT INTO buildrequests"
                                   " (buildsetid, buildername, submitted_at)"
-                                  " VALUES (?,?,?)"),
+                                  " VALUES (?,?,?)", "id"),
                       (bsid, bn, now))
-            brid = t.lastrowid
+            brid = self.lastrowid(t)
             brids.append(brid)
         self.notify("add-buildset", bsid)
         self.notify("add-buildrequest", *brids)
         return bsid
 
     def scheduler_classify_change(self, schedulerid, number, important, t):
         q = self.quoteq("INSERT INTO scheduler_changes"
                         " (schedulerid, changeid, important)"
@@ -835,19 +637,19 @@ class DBConnector(util.ComparableMixin):
             qargs = [now, master_name, master_incarnation] + list(batch)
             t.execute(q, qargs)
 
     def build_started(self, brid, buildnumber):
         return self.runInteractionNow(self._txn_build_started, brid, buildnumber)
     def _txn_build_started(self, t, brid, buildnumber):
         now = self._getCurrentTime()
         t.execute(self.quoteq("INSERT INTO builds (number, brid, start_time)"
-                              " VALUES (?,?,?)"),
+                              " VALUES (?,?,?)", "id"),
                   (buildnumber, brid, now))
-        bid = t.lastrowid
+        bid = self.lastrowid(t)
         self.notify("add-build", bid)
         return bid
 
     def builds_finished(self, bids):
         return self.runInteractionNow(self._txn_build_finished, bids)
     def _txn_build_finished(self, t, bids):
         now = self._getCurrentTime()
         while bids:
deleted file mode 100644
--- a/master/buildbot/db/dbspec.py
+++ /dev/null
@@ -1,263 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-import sys, os, cgi, re, time
-
-from twisted.python import log, reflect
-from twisted.enterprise import adbapi
-
-from buildbot import util
-
-class ExpiringConnectionPool(adbapi.ConnectionPool):
-    """
-    A Connection pool that expires connections after a certain amount of idle
-    time.
-    """
-    def __init__(self, dbapiName, max_idle=60, *args, **kwargs):
-        """
-        @param max_idle: reconnect connections that have been idle more than
-                         this number of seconds.
-        """
-
-        log.msg("Using expiring pool with max_idle=%i" % max_idle)
-
-        adbapi.ConnectionPool.__init__(self, dbapiName, *args, **kwargs)
-        self.max_idle = max_idle
-
-        self.connection_lastused = {}
-
-    def connect(self):
-        tid = self.threadID()
-        now = util.now()
-        lastused = self.connection_lastused.get(tid)
-        if lastused and lastused + self.max_idle < now:
-            conn = self.connections.get(tid)
-            if self.noisy:
-                log.msg("expiring old connection")
-            self.disconnect(conn)
-
-        conn = adbapi.ConnectionPool.connect(self)
-        self.connection_lastused[tid] = now
-        return conn
-
-    def disconnect(self, conn):
-        adbapi.ConnectionPool.disconnect(self, conn)
-        tid = self.threadID()
-        del self.connection_lastused[tid]
-
-class TimeoutError(Exception):
-    def __init__(self, msg):
-        Exception.__init__(self, msg)
-
-class RetryingCursor:
-    max_retry_time = 1800 # Half an hour
-    max_sleep_time = 1
-
-    def __init__(self, dbapi, cursor):
-        self.dbapi = dbapi
-        self.cursor = cursor
-
-    def sleep(self, s):
-        time.sleep(s)
-
-    def execute(self, *args, **kw):
-        start_time = util.now()
-        sleep_time = 0.1
-        while True:
-            try:
-                query_start_time = util.now()
-                result = self.cursor.execute(*args, **kw)
-                end_time = util.now()
-                if end_time - query_start_time > 2:
-                    log.msg("Long query (%is): %s" % ((end_time - query_start_time), str((args, kw))))
-                return result
-            except self.dbapi.OperationalError, e:
-                if e.args[0] == 'database is locked':
-                    # Retry
-                    log.msg("Retrying query %s" % str((args, kw)))
-                    now = util.now()
-                    if start_time + self.max_retry_time < now:
-                        raise TimeoutError("Exceeded timeout trying to do %s" % str((args, kw)))
-                    self.sleep(sleep_time)
-                    sleep_time = max(self.max_sleep_time, sleep_time * 2)
-                    continue
-                raise
-
-    def __getattr__(self, name):
-        return getattr(self.cursor, name)
-
-class RetryingConnection:
-    def __init__(self, dbapi, conn):
-        self.dbapi = dbapi
-        self.conn = conn
-
-    def cursor(self):
-        return RetryingCursor(self.dbapi, self.conn.cursor())
-
-    def __getattr__(self, name):
-        return getattr(self.conn, name)
-
-class RetryingConnectionPool(adbapi.ConnectionPool):
-    def connect(self):
-        return RetryingConnection(self.dbapi, adbapi.ConnectionPool.connect(self))
-
-class DBSpec(object):
-    """
-    A specification for the database type and other connection parameters.
-    """
-
-    # List of connkw arguments that are applicable to the connection pool only
-    pool_args = ["max_idle"]
-    def __init__(self, dbapiName, *connargs, **connkw):
-        # special-case 'sqlite3', replacing it with the available implementation
-        if dbapiName == 'sqlite3':
-            dbapiName = self._get_sqlite_dbapi_name()
-
-        self.dbapiName = dbapiName
-        self.connargs = connargs
-        self.connkw = connkw
-
-    @classmethod
-    def from_url(cls, url, basedir=None):
-        """
-        Parses a URL of the format
-          driver://[username:password@]host:port/database[?args]
-        and returns a DB object representing this URL.  Percent-
-        substitution will be performed, replacing %(basedir)s with
-        the basedir argument.
-
-        raises ValueError on an invalid URL.
-        """
-        match = re.match(r"""
-        ^(?P<driver>\w+)://
-        (
-            ((?P<user>\w+)(:(?P<passwd>\S+))?@)?
-            ((?P<host>[-A-Za-z0-9.]+)(:(?P<port>\d+))?)?/
-            (?P<database>\S+?)(\?(?P<args>.*))?
-        )?$""", url, re.X)
-        if not match:
-            raise ValueError("Malformed url")
-
-        d = match.groupdict()
-        driver = d['driver']
-        user = d['user']
-        passwd = d['passwd']
-        host = d['host']
-        port = d['port']
-        if port is not None:
-            port = int(port)
-        database = d['database']
-        args = {}
-        if d['args']:
-            for key, value in cgi.parse_qsl(d['args']):
-                args[key] = value
-
-        if driver == "sqlite":
-            # user, passwd, host, and port must all be None
-            if not user == passwd == host == port == None:
-                raise ValueError("user, passwd, host, port must all be None")
-            if not database:
-                database = ":memory:"
-            elif basedir:
-                database = database % dict(basedir=basedir)
-                database = os.path.join(basedir, database)
-            return cls("sqlite3", database, **args)
-        elif driver == "mysql":
-            args['host'] = host
-            args['db'] = database
-            if user:
-                args['user'] = user
-            if passwd:
-                args['passwd'] = passwd
-            if port:
-                args['port'] = port
-            if 'max_idle' in args:
-                args['max_idle'] = int(args['max_idle'])
-
-            return cls("MySQLdb", use_unicode=True, charset="utf8", **args)
-        else:
-            raise ValueError("Unsupported dbapi %s" % driver)
-
-    def _get_sqlite_dbapi_name(self):
-        # see which dbapi we can use and return that name; prefer
-        # pysqlite2.dbapi2 if it is available.
-        sqlite_dbapi_name = None
-        try:
-            from pysqlite2 import dbapi2 as sqlite3
-            assert sqlite3
-            sqlite_dbapi_name = "pysqlite2.dbapi2"
-        except ImportError:
-            # don't use built-in sqlite3 on 2.5 -- it has *bad* bugs
-            if sys.version_info >= (2,6):
-                import sqlite3
-                assert sqlite3
-                sqlite_dbapi_name = "sqlite3"
-            else:
-                raise
-        return sqlite_dbapi_name
-
-    def get_dbapi(self):
-        """
-        Get the dbapi module used for this connection (for things like
-        exceptions and module-global attributes
-        """
-        return reflect.namedModule(self.dbapiName)
-
-    def get_sync_connection(self):
-        """
-        Get a synchronous connection to the specified database.  This returns
-        a simple DBAPI connection object.
-        """
-        dbapi = self.get_dbapi()
-        connkw = self.connkw.copy()
-        for arg in self.pool_args:
-            if arg in connkw:
-                del connkw[arg]
-        conn = dbapi.connect(*self.connargs, **connkw)
-        if 'sqlite' in self.dbapiName:
-            conn = RetryingConnection(dbapi, conn)
-        return conn
-
-    def get_async_connection_pool(self):
-        """
-        Get an asynchronous (adbapi) connection pool for the specified
-        database.
-        """
-
-        # add some connection keywords
-        connkw = self.connkw.copy()
-        connkw["cp_reconnect"] = True
-        connkw["cp_noisy"] = True
-
-        # This disables sqlite's obsessive checks that a given connection is
-        # only used in one thread; this is justified by the Twisted ticket
-        # regarding the errors you get on connection shutdown if you do *not*
-        # add this parameter: http://twistedmatrix.com/trac/ticket/3629
-        if 'sqlite' in self.dbapiName:
-            connkw['check_same_thread'] = False
-        log.msg("creating adbapi pool: %s %s %s" % \
-                (self.dbapiName, self.connargs, connkw))
-
-        # MySQL needs support for expiring idle connections
-        if self.dbapiName == 'MySQLdb':
-            return ExpiringConnectionPool(self.dbapiName, *self.connargs, **connkw)
-        else:
-            return RetryingConnectionPool(self.dbapiName, *self.connargs, **connkw)
-
-    def get_maxidle(self):
-        default = None
-        if self.dbapiName == "MySQLdb":
-            default = 60
-        return self.connkw.get("max_idle", default)
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/enginestrategy.py
@@ -0,0 +1,148 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+"""
+A wrapper around `sqlalchemy.create_engine` that handles all of the
+special cases that Buildbot needs.  Those include:
+
+ - pool_recycle for MySQL
+ - %(basedir) substitution
+ - optimal thread pool size calculation
+
+"""
+
+import os
+import sqlalchemy
+from sqlalchemy.engine import strategies, url
+
+# from http://www.mail-archive.com/sqlalchemy@googlegroups.com/msg15079.html
+class ReconnectingListener(object):
+    def __init__(self):
+        self.retried = False
+    def checkout(self, dbapi_con, con_record, con_proxy):
+        try:
+            try:
+                dbapi_con.ping(False)
+            except TypeError:
+                dbapi_con.ping()
+        except dbapi_con.OperationalError, ex:
+            if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
+                # sqlalchemy will re-create the connection
+                raise sqlalchemy.exc.DisconnectionError()
+            raise
+
+class BuildbotEngineStrategy(strategies.ThreadLocalEngineStrategy):
+    """
+    A subclass of the ThreadLocalEngineStrategy that can effectively interact
+    with Buildbot.
+
+    This adjusts the passed-in parameters to ensure that we get the behaviors
+    Buildbot wants from particular drivers, and wraps the outgoing Engine
+    object so that its methods run in threads and return deferreds.
+    """
+
+    name = 'buildbot'
+
+    def special_case_sqlite(self, u, kwargs):
+        """For sqlite, percent-substitute %(basedir)s and use a full
+        path to the basedir.  If using a memory database, force the
+        pool size to be 1."""
+        max_conns = None
+
+        # when given a database path, stick the basedir in there
+        if u.database:
+            u.database = u.database % dict(basedir = kwargs['basedir'])
+            if not os.path.isabs(u.database[0]):
+                u.database = os.path.join(kwargs['basedir'], u.database)
+
+        # in-memory databases need exactly one connection
+        if not u.database:
+            kwargs['pool_size'] = 1
+            max_conns = 1
+
+        return u, kwargs, max_conns
+
+    def special_case_mysql(self, u, kwargs):
+        """For mysql, take max_idle out of the query arguments, and
+        use its value for pool_recycle.  Also, force use_unicode and
+        charset to be True and 'utf8', failing if they were set to
+        anything else."""
+
+        kwargs['pool_recycle'] = int(u.query.pop('max_idle', 3600))
+
+        if 'use_unicode' in u.query:
+            if u.query['use_unicode'] != "True":
+                raise TypeError("Buildbot requires use_unicode=True " +
+                                 "(and adds it automatically)")
+        else:
+            u.query['use_unicode'] = True
+
+        if 'charset' in u.query:
+            if u.query['charset'] != "utf8":
+                raise TypeError("Buildbot requires charset=utf8 " +
+                                 "(and adds it automatically)")
+        else:
+            u.query['charset'] = 'utf8'
+
+        # add the reconnecting PoolListener that will detect a
+        # disconnected connection and automatically start a new
+        # one.  This provides a measure of additional safety over
+        # the pool_recycle parameter, and is useful when e.g., the
+        # mysql server goes away
+        kwargs['listeners'] = [ ReconnectingListener() ]
+
+        return u, kwargs, None
+
+    def create(self, name_or_url, **kwargs):
+        if 'basedir' not in kwargs:
+            raise TypeError('no basedir supplied to create_engine')
+
+        max_conns = None
+
+        # apply special cases
+        u = url.make_url(name_or_url)
+        if u.drivername.startswith('sqlite'):
+            u, kwargs, max_conns = self.special_case_sqlite(u, kwargs)
+        elif u.drivername.startswith('mysql'):
+            u, kwargs, max_conns = self.special_case_sqlite(u, kwargs)
+
+        # remove the basedir as it may confuse sqlalchemy
+        basedir = kwargs.pop('basedir')
+
+        # calculate the maximum number of connections from the pool parameters,
+        # if it hasn't already been specified
+        if max_conns is None:
+            max_conns = kwargs.get('pool_size', 5) + kwargs.get('max_overflow', 10)
+
+        engine = strategies.ThreadLocalEngineStrategy.create(self,
+                                            u, **kwargs)
+
+        # annotate the engine with the optimal thread pool size; this is used
+        # by DBConnector to configure the surrounding thread pool
+        engine.optimal_thread_pool_size = max_conns
+
+        # and keep the basedir
+        engine.buildbot_basedir = basedir
+
+        return engine
+
+BuildbotEngineStrategy()
+
+# this module is really imported for the side-effects, but pyflakes will like
+# us to use something from the module -- so offer a copy of create_engine, which
+# explicitly adds the strategy argument
+def create_engine(*args, **kwargs):
+    kwargs['strategy'] = 'buildbot'
+    return sqlalchemy.create_engine(*args, **kwargs)
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/migrate/README
@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/migrate/migrate.cfg
@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=Buildbot
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/migrate/versions/001_initial.py
@@ -0,0 +1,281 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import os
+import cPickle
+from twisted.persisted import styles
+from buildbot.util import json
+import sqlalchemy as sa
+
+metadata = sa.MetaData()
+
+last_access = sa.Table('last_access', metadata,
+    sa.Column('who', sa.String(256), nullable=False),
+    sa.Column('writing', sa.Integer, nullable=False),
+    sa.Column('last_access', sa.Integer, nullable=False),
+)
+
+changes_nextid = sa.Table('changes_nextid', metadata,
+    sa.Column('next_changeid', sa.Integer),
+)
+
+changes = sa.Table('changes', metadata,
+    sa.Column('changeid', sa.Integer, autoincrement=False, primary_key=True),
+    sa.Column('author', sa.String(1024), nullable=False),
+    sa.Column('comments', sa.String(1024), nullable=False),
+    sa.Column('is_dir', sa.SmallInteger, nullable=False),
+    sa.Column('branch', sa.String(1024)),
+    sa.Column('revision', sa.String(256)),
+    sa.Column('revlink', sa.String(256)),
+    sa.Column('when_timestamp', sa.Integer, nullable=False),
+    sa.Column('category', sa.String(256)),
+)
+
+change_links = sa.Table('change_links', metadata,
+    sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
+    sa.Column('link', sa.String(1024), nullable=False),
+)
+
+change_files = sa.Table('change_files', metadata,
+    sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
+    sa.Column('filename', sa.String(1024), nullable=False),
+)
+
+change_properties = sa.Table('change_properties', metadata,
+    sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
+    sa.Column('property_name', sa.String(256), nullable=False),
+    sa.Column('property_value', sa.String(1024), nullable=False),
+)
+
+schedulers = sa.Table("schedulers", metadata,
+    sa.Column('schedulerid', sa.Integer, autoincrement=False, primary_key=True),
+    sa.Column('name', sa.String(128), nullable=False),
+    sa.Column('state', sa.String(1024), nullable=False),
+)
+
+scheduler_changes = sa.Table('scheduler_changes', metadata,
+    sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')),
+    sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')),
+    sa.Column('important', sa.SmallInteger),
+)
+
+scheduler_upstream_buildsets = sa.Table('scheduler_upstream_buildsets', metadata,
+    sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id')),
+    sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')),
+    sa.Column('active', sa.SmallInteger),
+)
+
+sourcestamps = sa.Table('sourcestamps', metadata,
+    sa.Column('id', sa.Integer, autoincrement=False, primary_key=True),
+    sa.Column('branch', sa.String(256)),
+    sa.Column('revision', sa.String(256)),
+    sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')),
+)
+
+patches = sa.Table('patches', metadata,
+    sa.Column('id', sa.Integer, autoincrement=False, primary_key=True),
+    sa.Column('patchlevel', sa.Integer, nullable=False),
+    sa.Column('patch_base64', sa.Text, nullable=False),
+    sa.Column('subdir', sa.Text),
+)
+
+sourcestamp_changes = sa.Table('sourcestamp_changes', metadata,
+    sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False),
+    sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
+)
+
+buildsets = sa.Table('buildsets', metadata,
+    sa.Column('id', sa.Integer, autoincrement=False, primary_key=True),
+    sa.Column('external_idstring', sa.String(256)),
+    sa.Column('reason', sa.String(256)),
+    sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False),
+    sa.Column('submitted_at', sa.Integer, nullable=False),
+    sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")),
+    sa.Column('complete_at', sa.Integer),
+    sa.Column('results', sa.SmallInteger),
+)
+
+buildset_properties = sa.Table('buildset_properties', metadata,
+    sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), nullable=False),
+    sa.Column('property_name', sa.String(256), nullable=False),
+    sa.Column('property_value', sa.String(1024), nullable=False),
+)
+
+buildrequests = sa.Table('buildrequests', metadata,
+    sa.Column('id', sa.Integer, autoincrement=False, primary_key=True),
+    sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), nullable=False),
+    sa.Column('buildername', sa.String(length=None), nullable=False),
+    sa.Column('priority', sa.Integer, nullable=False, server_default=sa.DefaultClause("0")),
+    sa.Column('claimed_at', sa.Integer, server_default=sa.DefaultClause("0")),
+    sa.Column('claimed_by_name', sa.String(length=None)),
+    sa.Column('claimed_by_incarnation', sa.String(length=None)),
+    sa.Column('complete', sa.Integer, server_default=sa.DefaultClause("0")),
+    sa.Column('results', sa.SmallInteger),
+    sa.Column('submitted_at', sa.Integer, nullable=False),
+    sa.Column('complete_at', sa.Integer),
+)
+
+builds = sa.Table('builds', metadata,
+    sa.Column('id', sa.Integer, autoincrement=False, primary_key=True),
+    sa.Column('number', sa.Integer, nullable=False),
+    sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False),
+    sa.Column('start_time', sa.Integer, nullable=False),
+    sa.Column('finish_time', sa.Integer),
+)
+
+def test_unicode(migrate_engine):
+    """Test that the database can handle inserting and selecting Unicode"""
+    # set up a subsidiary MetaData object to hold this temporary table
+    submeta = sa.MetaData()
+    submeta.bind = migrate_engine
+
+    test_unicode = sa.Table('test_unicode', submeta,
+        sa.Column('u', sa.Unicode),
+        sa.Column('b', sa.LargeBinary),
+    )
+    test_unicode.create()
+
+    # insert a unicode value in there
+    u = u"Frosty the \N{SNOWMAN}"
+    b='\xff\xff\x00'
+    ins = test_unicode.insert().values(u=u, b=b)
+    migrate_engine.execute(ins)
+
+    # see if the data is intact
+    row = migrate_engine.execute(sa.select([test_unicode])).fetchall()[0]
+    assert type(row['u']) is unicode
+    assert row['u'] == u
+    assert type(row['b']) is str
+    assert row['b'] == b
+
+    # drop the test table
+    test_unicode.drop()
+
+def import_changes(migrate_engine):
+    # get the basedir from the engine - see model.py if you're wondering
+    # how it got there
+    basedir = migrate_engine.buildbot_basedir
+
+    # strip None from any of these values, just in case
+    def remove_none(x):
+        if x is None: return u""
+        elif isinstance(x, str):
+            return x.decode("utf8")
+        else:
+            return x
+
+    # if we still have a changes.pck, then we need to migrate it
+    changes_pickle = os.path.join(basedir, "changes.pck")
+    if not os.path.exists(changes_pickle):
+        migrate_engine.execute(changes_nextid.insert(),
+                next_changeid=1)
+        return
+
+    #if not quiet: print "migrating changes.pck to database"
+
+    # 'source' will be an old b.c.changes.ChangeMaster instance, with a
+    # .changes attribute.  Note that we use 'r', and not 'rb', because these
+    # pickles were written using the old text pickle format, which requires
+    # newline translation
+    source = cPickle.load(open(changes_pickle,"r"))
+    styles.doUpgrade()
+
+    #if not quiet: print " (%d Change objects)" % len(source.changes)
+
+    # first, scan for changes without a number.  If we find any, then we'll
+    # renumber the changes sequentially
+    have_unnumbered = False
+    for c in source.changes:
+        if c.revision and c.number is None:
+            have_unnumbered = True
+            break
+    if have_unnumbered:
+        n = 1
+        for c in source.changes:
+            if c.revision:
+                c.number = n
+                n = n + 1
+
+    # insert the changes
+    for c in source.changes:
+        if not c.revision:
+            continue
+        try:
+            values = dict(
+                    changeid=c.number,
+                    author=c.who,
+                    comments=c.comments,
+                    is_dir=c.isdir,
+                    branch=c.branch,
+                    revision=c.revision,
+                    revlink=c.revlink,
+                    when_timestamp=c.when,
+                    category=c.category)
+            values = dict([ (k, remove_none(v)) for k, v in values.iteritems() ])
+        except UnicodeDecodeError, e:
+            raise UnicodeError("Trying to import change data as UTF-8 failed.  Please look at contrib/fix_changes_pickle_encoding.py: %s" % str(e))
+
+        migrate_engine.execute(changes.insert(), **values)
+
+        for link in c.links:
+            migrate_engine.execute(change_links.insert(),
+                    changeid=c.number, link=link)
+
+        # sometimes c.files contains nested lists -- why, I do not know!  But we deal with
+        # it all the same - see bug #915. We'll assume for now that c.files contains *either*
+        # lists of filenames or plain filenames, not both.
+        def flatten(l):
+            if l and type(l[0]) == list:
+                rv = []
+                for e in l:
+                    if type(e) == list:
+                        rv.extend(e)
+                    else:
+                        rv.append(e)
+                return rv
+            else:
+                return l
+        for filename in flatten(c.files):
+            migrate_engine.execute(change_files.insert(),
+                    changeid=c.number,
+                    filename=filename)
+
+        for propname,propvalue in c.properties.properties.items():
+            encoded_value = json.dumps(propvalue)
+            migrate_engine.execute(change_properties.insert(),
+                    changeid=c.number,
+                    property_name=propname,
+                    property_value=encoded_value)
+
+    # update next_changeid
+    max_changeid = max([ c.number for c in source.changes if c.revision ] + [ 0 ])
+    migrate_engine.execute(changes_nextid.insert(),
+            next_changeid=max_changeid+1)
+
+    #if not quiet:
+    #    print "moving changes.pck to changes.pck.old; delete it or keep it as a backup"
+    os.rename(changes_pickle, changes_pickle+".old")
+
+def upgrade(migrate_engine):
+    metadata.bind = migrate_engine
+
+    # do some tests before getting started
+    test_unicode(migrate_engine)
+
+    # create the initial schema
+    metadata.create_all()
+
+    # and import some changes
+    import_changes(migrate_engine)
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/migrate/versions/002_add_proj_repo.py
@@ -0,0 +1,30 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+
+def upgrade(migrate_engine):
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    # add project and repository columns to 'changes' an 'sourcestamps'
+    def add_cols(table):
+        repository = sa.Column('repository', sa.Text, nullable=False, server_default=sa.DefaultClause(''))
+        repository.create(table, populate_default=True)
+        project = sa.Column('project', sa.Text, nullable=False, server_default=sa.DefaultClause(''))
+        project.create(table, populate_default=True)
+
+    add_cols(sa.Table('changes', metadata, autoload=True))
+    add_cols(sa.Table('sourcestamps', metadata, autoload=True))
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/migrate/versions/003_scheduler_class_name.py
@@ -0,0 +1,29 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+
+def upgrade(migrate_engine):
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    # add an empty class_name to the schedulers table
+    schedulers = sa.Table('schedulers', metadata, autoload=True)
+    class_name = sa.Column('class_name', sa.Text, nullable=False, server_default=sa.DefaultClause(''))
+    class_name.create(schedulers, populate_default=True)
+
+    # and an index since we'll be selecting with (name= AND class=)
+    idx = sa.Index('name_and_class', schedulers.c.name, schedulers.c.class_name)
+    idx.create(migrate_engine)
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/migrate/versions/004_add_autoincrement.py
@@ -0,0 +1,86 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+
+def upgrade(migrate_engine):
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    # re-include some of the relevant tables, as they were in version 3, since
+    # sqlalchemy's reflection doesn't work very well for defaults
+
+    sa.Table("schedulers", metadata,
+        sa.Column('schedulerid', sa.Integer, autoincrement=False, primary_key=True),
+        sa.Column('name', sa.String(128), nullable=False),
+        sa.Column('state', sa.String(1024), nullable=False),
+        sa.Column('class_name', sa.Text, nullable=False, server_default=sa.DefaultClause(''))
+    )
+
+    sa.Table('changes', metadata,
+        sa.Column('changeid', sa.Integer, autoincrement=False, primary_key=True),
+        sa.Column('author', sa.String(1024), nullable=False),
+        sa.Column('comments', sa.String(1024), nullable=False),
+        sa.Column('is_dir', sa.SmallInteger, nullable=False),
+        sa.Column('branch', sa.String(1024)),
+        sa.Column('revision', sa.String(256)),
+        sa.Column('revlink', sa.String(256)),
+        sa.Column('when_timestamp', sa.Integer, nullable=False),
+        sa.Column('category', sa.String(256)),
+        sa.Column('repository', sa.Text, nullable=False, server_default=sa.DefaultClause('')),
+        sa.Column('project', sa.Text, nullable=False, server_default=sa.DefaultClause('')),
+    )
+
+    sa.Table('sourcestamps', metadata,
+        sa.Column('id', sa.Integer, autoincrement=False, primary_key=True),
+        sa.Column('branch', sa.String(256)),
+        sa.Column('revision', sa.String(256)),
+        sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')),
+        sa.Column('repository', sa.Text, nullable=False, server_default=''),
+        sa.Column('project', sa.Text, nullable=False, server_default=''),
+    )
+
+    to_autoinc = [ s.split(".") for s in
+        "schedulers.schedulerid",
+        "builds.id",
+        "changes.changeid",
+        "buildrequests.id",
+        "buildsets.id",
+        "patches.id",
+        "sourcestamps.id",
+    ]
+
+    # It seems that SQLAlchemy's ALTER TABLE doesn't work when migrating from
+    # INTEGER to PostgreSQL's SERIAL data type (which is just pseudo data type
+    # for INTEGER with SEQUENCE), so we have to work-around this with raw SQL.
+    if migrate_engine.dialect.name in ('postgres', 'postgresql'):
+        for table_name, col_name in to_autoinc:
+            migrate_engine.execute("CREATE SEQUENCE %s_%s_seq"
+                                   % (table_name, col_name))
+            migrate_engine.execute("ALTER TABLE %s ALTER COLUMN %s SET DEFAULT nextval('%s_%s_seq'::regclass)"
+                                   % (table_name, col_name, table_name, col_name))
+            migrate_engine.execute("ALTER SEQUENCE %s_%s_seq OWNED BY %s.%s"
+                                   % (table_name, col_name, table_name, col_name))
+    else:
+        for table_name, col_name in to_autoinc:
+            table = sa.Table(table_name, metadata, autoload=True)
+            col = table.c[col_name]
+            col.alter(autoincrement=True)
+
+
+    # also drop the changes_nextid table here (which really should have been a
+    # sequence..)
+    table = sa.Table('changes_nextid', metadata, autoload=True)
+    table.drop()
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/migrate/versions/005_add_indexes.py
@@ -0,0 +1,144 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+
+def upgrade(migrate_engine):
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    # note that all of the tables defined here omit the ForeignKey constraints;
+    # this just lets this code specify the tables in any order; the tables are
+    # not re-created here, so this omission causes no problems - the key
+    # constraints are still defined in the table
+
+    def add_index(table_name, col_name):
+        idx_name = "%s_%s" % (table_name, col_name)
+        idx = sa.Index(idx_name, metadata.tables[table_name].c[col_name])
+        idx.create(migrate_engine)
+
+    sa.Table('buildrequests', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+        sa.Column('buildsetid', sa.Integer, nullable=False),
+        sa.Column('buildername', sa.String(length=None), nullable=False),
+        sa.Column('priority', sa.Integer, nullable=False),
+        sa.Column('claimed_at', sa.Integer, server_default=sa.DefaultClause("0")),
+        sa.Column('claimed_by_name', sa.String(length=None)),
+        sa.Column('claimed_by_incarnation', sa.String(length=None)),
+        sa.Column('complete', sa.Integer, server_default=sa.DefaultClause("0")),
+        sa.Column('results', sa.SmallInteger),
+        sa.Column('submitted_at', sa.Integer, nullable=False),
+        sa.Column('complete_at', sa.Integer),
+    )
+    add_index("buildrequests", "buildsetid")
+    add_index("buildrequests", "buildername")
+    add_index("buildrequests", "complete")
+    add_index("buildrequests", "claimed_at")
+    add_index("buildrequests", "claimed_by_name")
+
+    sa.Table('builds', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+        sa.Column('number', sa.Integer, nullable=False),
+        sa.Column('brid', sa.Integer, nullable=False),
+        sa.Column('start_time', sa.Integer, nullable=False),
+        sa.Column('finish_time', sa.Integer),
+    )
+    add_index("builds", "number")
+    add_index("builds", "brid")
+
+    sa.Table('buildsets', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+        sa.Column('external_idstring', sa.String(256)),
+        sa.Column('reason', sa.String(256)),
+        sa.Column('sourcestampid', sa.Integer, nullable=False),
+        sa.Column('submitted_at', sa.Integer, nullable=False),
+        sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")),
+        sa.Column('complete_at', sa.Integer),
+        sa.Column('results', sa.SmallInteger),
+    )
+    add_index("buildsets", "complete")
+    add_index("buildsets", "submitted_at")
+
+    sa.Table('buildset_properties', metadata,
+        sa.Column('buildsetid', sa.Integer, nullable=False),
+        sa.Column('property_name', sa.String(256), nullable=False),
+        sa.Column('property_value', sa.String(1024), nullable=False),
+    )
+    add_index("buildset_properties", "buildsetid")
+
+    sa.Table('changes', metadata,
+        sa.Column('changeid', sa.Integer,  primary_key=True),
+        sa.Column('author', sa.String(1024), nullable=False),
+        sa.Column('comments', sa.String(1024), nullable=False),
+        sa.Column('is_dir', sa.SmallInteger, nullable=False),
+        sa.Column('branch', sa.String(1024)),
+        sa.Column('revision', sa.String(256)),
+        sa.Column('revlink', sa.String(256)),
+        sa.Column('when_timestamp', sa.Integer, nullable=False),
+        sa.Column('category', sa.String(256)),
+        sa.Column('repository', sa.Text, nullable=False, server_default=''),
+        sa.Column('project', sa.Text, nullable=False, server_default=''),
+    )
+    add_index("changes", "branch")
+    add_index("changes", "revision")
+    add_index("changes", "author")
+    add_index("changes", "category")
+    add_index("changes", "when_timestamp")
+
+    sa.Table('change_files', metadata,
+        sa.Column('changeid', sa.Integer, nullable=False),
+        sa.Column('filename', sa.String(1024), nullable=False),
+    )
+    add_index("change_files", "changeid")
+
+    sa.Table('change_links', metadata,
+        sa.Column('changeid', sa.Integer, nullable=False),
+        sa.Column('link', sa.String(1024), nullable=False),
+    )
+    add_index("change_links", "changeid")
+
+    sa.Table('change_properties', metadata,
+        sa.Column('changeid', sa.Integer, nullable=False),
+        sa.Column('property_name', sa.String(256), nullable=False),
+        sa.Column('property_value', sa.String(1024), nullable=False),
+    )
+    add_index("change_properties", "changeid")
+
+    # schedulers already has an index
+
+    sa.Table('scheduler_changes', metadata,
+        sa.Column('schedulerid', sa.Integer),
+        sa.Column('changeid', sa.Integer),
+        sa.Column('important', sa.SmallInteger),
+    )
+    add_index("scheduler_changes", "schedulerid")
+    add_index("scheduler_changes", "changeid")
+
+    sa.Table('scheduler_upstream_buildsets', metadata,
+        sa.Column('buildsetid', sa.Integer),
+        sa.Column('schedulerid', sa.Integer),
+        sa.Column('active', sa.SmallInteger),
+    )
+    add_index("scheduler_upstream_buildsets", "buildsetid")
+    add_index("scheduler_upstream_buildsets", "schedulerid")
+    add_index("scheduler_upstream_buildsets", "active")
+
+    # sourcestamps are only queried by id, no need for additional indexes
+
+    sa.Table('sourcestamp_changes', metadata,
+        sa.Column('sourcestampid', sa.Integer, nullable=False),
+        sa.Column('changeid', sa.Integer, nullable=False),
+    )
+    add_index("sourcestamp_changes", "sourcestampid")
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/migrate/versions/006_drop_last_access.py
@@ -0,0 +1,23 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import sqlalchemy as sa
+
+def upgrade(migrate_engine):
+    metadata = sa.MetaData()
+    metadata.bind = migrate_engine
+
+    table = sa.Table('last_access', metadata, autoload=True)
+    table.drop()
new file mode 100755
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/model.py
@@ -0,0 +1,425 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+"""
+Storage for the database model (schema)
+"""
+
+import sqlalchemy as sa
+import migrate
+import migrate.versioning.schema
+import migrate.versioning.repository
+import migrate.versioning.exceptions
+from twisted.python import util, log
+from buildbot.db import base
+
+class Model(base.DBConnectorComponent):
+    """
+    DBConnector component to handle the database model; an instance is available
+    at C{master.db.model}.
+
+    This class has attributes for each defined table, as well as methods to
+    handle schema migration (using sqlalchemy-migrate).  View the source to see
+    the table definitions.
+
+    Note that the Buildbot metadata is never bound to an engine, since that might
+    lead users to execute queries outside of the thread pool.
+    """
+
+    #
+    # schema
+    #
+
+    metadata = sa.MetaData()
+
+    # NOTES
+
+    # * server_defaults here are included to match those added by the migration
+    #   scripts, but they should not be depended on - all code accessing these
+    #   tables should supply default values as necessary.  The defaults are
+    #   required during migration when adding non-nullable columns to existing
+    #   tables.
+    #
+    # * dates are stored as unix timestamps (UTC-ish epoch time)
+
+    # build requests
+
+    buildrequests = sa.Table('buildrequests', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+        sa.Column('buildsetid', sa.Integer, sa.ForeignKey("buildsets.id"), nullable=False),
+        sa.Column('buildername', sa.String(length=None), nullable=False),
+        sa.Column('priority', sa.Integer, nullable=False, server_default=sa.DefaultClause("0")), # TODO: used?
+
+        # claimed_at is the time at which a master most recently asserted that
+        # it is responsible for running the build: this will be updated
+        # periodically to maintain the claim.  Note that 0 and NULL mean the
+        # same thing here (and not 1969!)
+        sa.Column('claimed_at', sa.Integer, server_default=sa.DefaultClause("0")), # TODO: timestamp
+
+        # claimed_by indicates which buildmaster has claimed this request. The
+        # 'name' contains hostname/basedir, and will be the same for subsequent
+        # runs of any given buildmaster. The 'incarnation' contains bootime/pid,
+        # and will be different for subsequent runs. This allows each buildmaster
+        # to distinguish their current claims, their old claims, and the claims
+        # of other buildmasters, to treat them each appropriately.
+        sa.Column('claimed_by_name', sa.String(length=None)),
+        sa.Column('claimed_by_incarnation', sa.String(length=None)),
+
+        # if this is zero, then the build is still pending
+        sa.Column('complete', sa.Integer, server_default=sa.DefaultClause("0")), # TODO: boolean
+
+        # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS,
+        # etc - see master/buildbot/status/builder.py
+        sa.Column('results', sa.SmallInteger),
+
+        # time the buildrequest was created
+        sa.Column('submitted_at', sa.Integer, nullable=False), # TODO: timestamp
+
+        # time the buildrequest was completed, or NULL
+        sa.Column('complete_at', sa.Integer), # TODO: timestamp
+    )
+    """A BuildRequest is a request for a particular build to be performed.
+    Each BuildRequest is a part of a BuildSet.  BuildRequests are claimed by
+    masters, to avoid multiple masters running the same build."""
+
+    # builds
+
+    builds = sa.Table('builds', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+
+        # XXX
+        # the build number is local to the builder and (maybe?) the buildmaster
+        sa.Column('number', sa.Integer, nullable=False),
+
+        sa.Column('brid', sa.Integer, sa.ForeignKey('buildrequests.id'), nullable=False),
+        sa.Column('start_time', sa.Integer, nullable=False),
+        sa.Column('finish_time', sa.Integer),
+    )
+    """This table contains basic information about each build.  Note that most data
+    about a build is still stored in on-disk pickles."""
+
+    # buildsets
+
+    buildset_properties = sa.Table('buildset_properties', metadata,
+        sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id'), nullable=False),
+        sa.Column('property_name', sa.String(256), nullable=False),
+        # JSON-encoded tuple of (value, source)
+        sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short?
+    )
+    """This table contains input properties for buildsets"""
+
+    buildsets = sa.Table('buildsets', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+
+        # a simple external identifier to track down this buildset later, e.g.,
+        # for try requests
+        sa.Column('external_idstring', sa.String(256)),
+
+        # a short string giving the reason the buildset was created
+        sa.Column('reason', sa.String(256)), # TODO: sa.Text
+        sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False),
+        sa.Column('submitted_at', sa.Integer, nullable=False), # TODO: timestamp (or redundant?)
+
+        # if this is zero, then the build set is still pending
+        sa.Column('complete', sa.SmallInteger, nullable=False, server_default=sa.DefaultClause("0")), # TODO: redundant
+        sa.Column('complete_at', sa.Integer), # TODO: timestamp (or redundant?)
+
+        # results is only valid when complete == 1; 0 = SUCCESS, 1 = WARNINGS,
+        # etc - see master/buildbot/status/builder.py
+        sa.Column('results', sa.SmallInteger), # TODO: synthesize from buildrequests
+    )
+    """This table represents BuildSets - sets of BuildRequests that share the same
+    original cause and source information."""
+
+    # changes
+
+    change_files = sa.Table('change_files', metadata,
+        sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
+        sa.Column('filename', sa.String(1024), nullable=False), # TODO: sa.Text
+    )
+    """Files touched in changes"""
+
+    change_links = sa.Table('change_links', metadata,
+        sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
+        sa.Column('link', sa.String(1024), nullable=False), # TODO: sa.Text
+    )
+    """Links (URLs) for changes"""
+
+    change_properties = sa.Table('change_properties', metadata,
+        sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
+        sa.Column('property_name', sa.String(256), nullable=False),
+        # JSON-encoded property value
+        sa.Column('property_value', sa.String(1024), nullable=False), # TODO: too short?
+    )
+    """Properties for changes"""
+
+    changes = sa.Table('changes', metadata,
+        # changeid also serves as 'change number'
+        sa.Column('changeid', sa.Integer,  primary_key=True), # TODO: rename to 'id'
+
+        # author's name (usually an email address)
+        sa.Column('author', sa.String(1024), nullable=False),
+
+        # commit comment
+        sa.Column('comments', sa.String(1024), nullable=False), # TODO: too short?
+
+        # old, CVS-related boolean
+        sa.Column('is_dir', sa.SmallInteger, nullable=False), # old, for CVS
+
+        # The branch where this change occurred.  When branch is NULL, that
+        # means the main branch (trunk, master, etc.)
+        sa.Column('branch', sa.String(1024)),
+
+        # revision identifier for this change
+        sa.Column('revision', sa.String(256)), # CVS uses NULL
+
+        # ?? (TODO)
+        sa.Column('revlink', sa.String(256)),
+
+        # this is the timestamp of the change - it is usually copied from the
+        # version-control system, and may be long in the past or even in the
+        # future!
+        sa.Column('when_timestamp', sa.Integer, nullable=False),
+
+        # an arbitrary string used for filtering changes
+        sa.Column('category', sa.String(256)),
+
+        # repository specifies, along with revision and branch, the
+        # source tree in which this change was detected.
+        sa.Column('repository', sa.Text, nullable=False, server_default=''),
+
+        # project names the project this source code represents.  It is used
+        # later to filter changes
+        sa.Column('project', sa.Text, nullable=False, server_default=''),
+    )
+    """Changes to the source code, produced by ChangeSources"""
+
+    # sourcestamps
+
+    patches = sa.Table('patches', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+
+        # number of directory levels to strip off (patch -pN)
+        sa.Column('patchlevel', sa.Integer, nullable=False),
+
+        # base64-encoded version of the patch file
+        sa.Column('patch_base64', sa.Text, nullable=False),
+
+        # subdirectory in which the patch should be applied; NULL for top-level
+        sa.Column('subdir', sa.Text),
+    )
+    """Patches for SourceStamps that were generated through the try mechanism"""
+
+    sourcestamp_changes = sa.Table('sourcestamp_changes', metadata,
+        sa.Column('sourcestampid', sa.Integer, sa.ForeignKey('sourcestamps.id'), nullable=False),
+        sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid'), nullable=False),
+    )
+    """The changes that led up to a particular source stamp."""
+    # TODO: changes should be the result of the difference of two sourcestamps!
+
+    sourcestamps = sa.Table('sourcestamps', metadata,
+        sa.Column('id', sa.Integer,  primary_key=True),
+
+        # the branch to check out.  When branch is NULL, that means
+        # the main branch (trunk, master, etc.)
+        sa.Column('branch', sa.String(256)),
+
+        # the revision to check out, or the latest if NULL
+        sa.Column('revision', sa.String(256)),
+
+        # the patch to apply to generate this source code
+        sa.Column('patchid', sa.Integer, sa.ForeignKey('patches.id')),
+
+        # the repository from which this source should be checked out
+        sa.Column('repository', sa.Text(length=None), nullable=False, server_default=''),
+
+        # the project this source code represents
+        sa.Column('project', sa.Text(length=None), nullable=False, server_default=''),
+    )
+    """A sourcestamp identifies a particular instance of the source code.
+    Ideally, this would always be absolute, but in practice source stamps can
+    also mean "latest" (when revision is NULL), which is of course a
+    time-dependent definition."""
+
+    # schedulers
+
+    scheduler_changes = sa.Table('scheduler_changes', metadata,
+        sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')),
+        sa.Column('changeid', sa.Integer, sa.ForeignKey('changes.changeid')),
+        # true if this change is important to this scheduler
+        sa.Column('important', sa.SmallInteger), # TODO: Boolean
+    )
+    """This table references "classified" changes that have not yet been "processed".
+    That is, the scheduler has looked at these changes and determined that
+    something should be done, but that hasn't happened yet.  Rows are deleted
+    from this table as soon as the scheduler is done with the change."""
+
+    scheduler_upstream_buildsets = sa.Table('scheduler_upstream_buildsets', metadata,
+        sa.Column('buildsetid', sa.Integer, sa.ForeignKey('buildsets.id')),
+        sa.Column('schedulerid', sa.Integer, sa.ForeignKey('schedulers.schedulerid')),
+        # true if this buildset is still active
+        sa.Column('active', sa.SmallInteger), # TODO: redundant
+    )
+    """This table references buildsets in which a particular scheduler is
+    interested.  On every run, a scheduler checks its upstream buildsets for
+    completion and reacts accordingly.  Records are never deleted from this
+    table, but active is set to 0 when the record is no longer necessary."""
+    # TODO: delete records eventually
+
+    schedulers = sa.Table("schedulers", metadata,
+        # unique ID for scheduler
+        sa.Column('schedulerid', sa.Integer, primary_key=True), # TODO: rename to id
+        # scheduler's name in master.cfg
+        sa.Column('name', sa.String(128), nullable=False),
+        # JSON-encoded state for this scheduler
+        sa.Column('state', sa.String(1024), nullable=False),
+        # scheduler's class name, basically representing a "type" for the state
+        sa.Column('class_name', sa.String(128), nullable=False),
+    )
+    """This table records the "state" for each scheduler.  This state is, at least,
+    the last change that was analyzed, but is stored in an opaque JSON object.
+    Note that schedulers are never deleted."""
+    # TODO: delete records eventually
+
+    # indexes
+
+    sa.Index('name_and_class', schedulers.c.name, schedulers.c.class_name)
+    sa.Index('buildrequests_buildsetid', buildrequests.c.buildsetid)
+    sa.Index('buildrequests_buildername', buildrequests.c.buildername)
+    sa.Index('buildrequests_complete', buildrequests.c.complete)
+    sa.Index('buildrequests_claimed_at', buildrequests.c.claimed_at)
+    sa.Index('buildrequests_claimed_by_name', buildrequests.c.claimed_by_name)
+    sa.Index('builds_number', builds.c.number)
+    sa.Index('builds_brid', builds.c.brid)
+    sa.Index('buildsets_complete', buildsets.c.complete)
+    sa.Index('buildsets_submitted_at', buildsets.c.submitted_at)
+    sa.Index('buildset_properties_buildsetid', buildset_properties.c.buildsetid)
+    sa.Index('changes_branch', changes.c.branch)
+    sa.Index('changes_revision', changes.c.revision)
+    sa.Index('changes_author', changes.c.author)
+    sa.Index('changes_category', changes.c.category)
+    sa.Index('changes_when_timestamp', changes.c.when_timestamp)
+    sa.Index('change_files_changeid', change_files.c.changeid)
+    sa.Index('change_links_changeid', change_links.c.changeid)
+    sa.Index('change_properties_changeid', change_properties.c.changeid)
+    sa.Index('scheduler_changes_schedulerid', scheduler_changes.c.schedulerid)
+    sa.Index('scheduler_changes_changeid', scheduler_changes.c.changeid)
+    sa.Index('scheduler_upstream_buildsets_buildsetid', scheduler_upstream_buildsets.c.buildsetid)
+    sa.Index('scheduler_upstream_buildsets_schedulerid', scheduler_upstream_buildsets.c.schedulerid)
+    sa.Index('scheduler_upstream_buildsets_active', scheduler_upstream_buildsets.c.active)
+    sa.Index('sourcestamp_changes_sourcestampid', sourcestamp_changes.c.sourcestampid)
+
+    #
+    # migration support
+    #
+
+    # this is a bit more complicated than might be expected because the first
+    # seven database versions were once implemented using a homespun migration
+    # system, and we need to support upgrading masters from that system.  The
+    # old system used a 'version' table, where SQLAlchemy-Migrate uses
+    # 'migrate_version'
+
+    repo_path = util.sibpath(__file__, "migrate")
+    "path to the SQLAlchemy-Migrate 'repository'"
+
+    def is_current(self):
+        """Returns true (via deferred) if the database's version is up to date."""
+        def thd(engine):
+            # we don't even have to look at the old version table - if there's
+            # no migrate_version, then we're not up to date.
+            repo = migrate.versioning.repository.Repository(self.repo_path)
+            repo_version = repo.latest
+            try:
+                # migrate.api doesn't let us hand in an engine
+                schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path)
+                db_version = schema.version
+            except migrate.versioning.exceptions.DatabaseNotControlledError:
+                return False
+
+            return db_version == repo_version
+        return self.db.pool.do_with_engine(thd)
+
+    def upgrade(self):
+        """Upgrade the database to the most recent schema version, returning a
+        deferred."""
+
+        # here, things are a little tricky.  If we have a 'version' table, then
+        # we need to version_control the database with the proper version
+        # number, drop 'version', and then upgrade.  If we have no 'version'
+        # table and no 'migrate_version' table, then we need to version_control
+        # the database.  Otherwise, we just need to upgrade it.
+
+        def table_exists(engine, tbl):
+            try:
+                r = engine.execute("select * from %s limit 1" % tbl)
+                r.close()
+                return True
+            except:
+                return False
+
+        # due to http://code.google.com/p/sqlalchemy-migrate/issues/detail?id=100, we cannot
+        # use the migrate.versioning.api module.  So these methods perform similar wrapping
+        # functions to what is done by the API functions, but without disposing of the engine.
+        def upgrade(engine):
+            schema = migrate.versioning.schema.ControlledSchema(engine, self.repo_path)
+            changeset = schema.changeset(None)
+            for version, change in changeset:
+                log.msg('migrating schema version %s -> %d'
+                        % (version, version + 1))
+                schema.runchange(version, change, 1)
+
+        def version_control(engine, version=None):
+            migrate.versioning.schema.ControlledSchema.create(engine, self.repo_path, version)
+
+        # the upgrade process must run in a db thread
+        def thd(engine):
+            # if the migrate_version table exists, we can just let migrate
+            # take care of this process.
+            if table_exists(engine, 'migrate_version'):
+                upgrade(engine)
+
+            # if the version table exists, then we can version_control things
+            # at that version, drop the version table, and let migrate take
+            # care of the rest.
+            elif table_exists(engine, 'version'):
+                # get the existing version
+                r = engine.execute("select version from version limit 1")
+                old_version = r.scalar()
+
+                # set up migrate at the same version
+                version_control(engine, old_version)
+
+                # drop the no-longer-required version table
+                engine.drop('version')
+
+                # and, finally, upgrade using migrate
+                upgrade(engine)
+
+            # otherwise, this db is uncontrolled, so we just version control it
+            # and update it.
+            else:
+                version_control(engine)
+                upgrade(engine)
+        return self.db.pool.do_with_engine(thd)
+
+# migrate has a bug in one of its warnings; this is fixed in version control
+# (3ba66abc4d), but not yet released. It can't hurt to fix it here, too, so we
+# get realistic tracebacks
+try:
+    import migrate.versioning.exceptions as ex1
+    import migrate.changeset.exceptions as ex2
+    ex1.MigrateDeprecationWarning = ex2.MigrateDeprecationWarning
+except ImportError:
+    pass
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/pool.py
@@ -0,0 +1,118 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+import twisted
+from sqlalchemy import engine
+from twisted.internet import reactor, threads, defer
+from twisted.python import threadpool, failure, versions
+
+class DBThreadPool(threadpool.ThreadPool):
+    """
+    A pool of threads ready and waiting to execute queries.
+
+    If the engine has an @C{optimal_thread_pool_size} attribute, then the
+    maxthreads of the thread pool will be set to that value.  This is most
+    useful for SQLite in-memory connections, where exactly one connection
+    (and thus thread) should be used.
+    """
+
+    running = False
+
+    def __init__(self, engine):
+        pool_size = 5
+        if hasattr(engine, 'optimal_thread_pool_size'):
+            pool_size = engine.optimal_thread_pool_size
+        threadpool.ThreadPool.__init__(self,
+                        minthreads=1,
+                        maxthreads=pool_size,
+                        name='DBThreadPool')
+        self.engine = engine
+        self._start_evt = reactor.callWhenRunning(self._start)
+
+    def _start(self):
+        self._start_evt = None
+        if not self.running:
+            self.start()
+            self._stop_evt = reactor.addSystemEventTrigger(
+                    'during', 'shutdown', self._stop)
+            self.running = True
+
+    def _stop(self):
+        self._stop_evt = None
+        self.stop()
+        self.engine.dispose()
+        self.running = False
+
+    def do(self, callable, *args, **kwargs):
+        """
+        Call CALLABLE in a thread, with a Connection as first argument.
+        Returns a deferred that will indicate the results of the callable.
+
+        Note: do not return any SQLAlchemy objects via this deferred!
+        """
+        def thd():
+            conn = self.engine.contextual_connect()
+            rv = callable(conn, *args, **kwargs)
+            assert not isinstance(rv, engine.ResultProxy), \
+                    "do not return ResultProxy objects!"
+            return rv
+        return threads.deferToThreadPool(reactor, self, thd)
+
+    def do_with_engine(self, callable, *args, **kwargs):
+        """
+        Like l{do}, but with an SQLAlchemy Engine as the first argument
+        """
+        def thd():
+            conn = self.engine
+            rv = callable(conn, *args, **kwargs)
+            assert not isinstance(rv, engine.ResultProxy), \
+                    "do not return ResultProxy objects!"
+            return rv
+        return threads.deferToThreadPool(reactor, self, thd)
+
+    # older implementations for twisted < 0.8.2, which does not have
+    # deferToThreadPool; this basically re-implements it, although it gets some
+    # of the synchronization wrong - the thread may still be "in use" when the
+    # deferred fires in the parent, which can lead to database accesses hopping
+    # between threads.  In practice, this should not cause any difficulty.
+    def do_081(self, callable, *args, **kwargs):
+        d = defer.Deferred()
+        def thd():
+            try:
+                conn = self.engine.contextual_connect()
+                rv = callable(conn, *args, **kwargs)
+                assert not isinstance(rv, engine.ResultProxy), \
+                        "do not return ResultProxy objects!"
+                reactor.callFromThread(d.callback, rv)
+            except:
+                reactor.callFromThread(d.errback, failure.Failure())
+        self.callInThread(thd)
+        return d
+    def do_with_engine_081(self, callable, *args, **kwargs):
+        d = defer.Deferred()
+        def thd():
+            try:
+                conn = self.engine
+                rv = callable(conn, *args, **kwargs)
+                assert not isinstance(rv, engine.ResultProxy), \
+                        "do not return ResultProxy objects!"
+                reactor.callFromThread(d.callback, rv)
+            except:
+                reactor.callFromThread(d.errback, failure.Failure())
+        self.callInThread(thd)
+        return d
+    if twisted.version < versions.Version('twisted', 8, 2, 0):
+        do = do_081
+        do_with_engine = do_with_engine_081
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/schedulers.py
@@ -0,0 +1,161 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+"""
+Support for schedulers in the database
+"""
+
+from buildbot.util import json
+import sqlalchemy as sa
+from twisted.python import log
+from buildbot.db import base
+
+class SchedulersConnectorComponent(base.DBConnectorComponent):
+    """
+    A DBConnectorComponent to handle maintaining schedulers' state in the db.
+    """
+
+    def getState(self, schedulerid):
+        """Get this scheduler's state, as a dictionary.  Returs a Deferred"""
+        def thd(conn):
+            schedulers_tbl = self.db.model.schedulers
+            q = sa.select([ schedulers_tbl.c.state ],
+                    whereclause=(schedulers_tbl.c.schedulerid == schedulerid))
+            row = conn.execute(q).fetchone()
+            if not row:
+                return {} # really shouldn't happen - the row should exist
+            try:
+                return json.loads(row.state)
+            except:
+                log.msg("JSON error loading state for scheduler #%s" % (schedulerid,))
+                return {}
+        return self.db.pool.do(thd)
+
+    def setState(self, schedulerid, state):
+        """Set this scheduler's stored state, represented as a JSON-able
+        dictionary.  Returs a Deferred.  Note that this will overwrite any
+        existing state; be careful with updates!"""
+        def thd(conn):
+            schedulers_tbl = self.db.model.schedulers
+            q = schedulers_tbl.update(
+                    whereclause=(schedulers_tbl.c.schedulerid == schedulerid))
+            conn.execute(q, state=json.dumps(state))
+        return self.db.pool.do(thd)
+
+    # TODO: maybe only the singular is needed?
+    def classifyChanges(self, schedulerid, classifications):
+        """Record a collection of classifications in the scheduler_changes
+        table. CLASSIFICATIONS is a dictionary mapping CHANGEID to IMPORTANT
+        (boolean).  Returns a Deferred."""
+        def thd(conn):
+            scheduler_changes_tbl = self.db.model.scheduler_changes
+            q = scheduler_changes_tbl.insert()
+            for changeid, important in classifications.items():
+                conn.execute(q,
+                        schedulerid=schedulerid,
+                        changeid=changeid,
+                        important=important)
+        return self.db.pool.do(thd)
+
+    def flushChangeClassifications(self, schedulerid, less_than=None):
+        """
+        Flush all scheduler_changes for L{schedulerid}, limiting to those less
+        than C{less_than} if the parameter is supplied.  Returns a Deferred.
+        """
+        def thd(conn):
+            scheduler_changes_tbl = self.db.model.scheduler_changes
+            wc = (scheduler_changes_tbl.c.schedulerid == schedulerid)
+            if less_than is not None:
+                wc = wc & (scheduler_changes_tbl.c.changeid < less_than)
+            q = scheduler_changes_tbl.delete(whereclause=wc)
+            conn.execute(q)
+        return self.db.pool.do(thd)
+
+    class Thunk: pass
+    def getChangeClassifications(self, schedulerid, branch=Thunk):
+        """
+        Return the scheduler_changes rows for this scheduler, in the form of a
+        dictionary mapping changeid to a boolean (important).  Returns a
+        Deferred.
+
+        @param schedulerid: scheduler to look up changes for
+        @type schedulerid: integer
+
+        @param branch: limit to changes with this branch
+        @type branch: string or None (for default branch)
+
+        @returns: dictionary via Deferred
+        """
+        def thd(conn):
+            scheduler_changes_tbl = self.db.model.scheduler_changes
+            changes_tbl = self.db.model.changes
+
+            wc = (scheduler_changes_tbl.c.schedulerid == schedulerid)
+            if branch is not self.Thunk:
+                wc = wc & (
+                    (scheduler_changes_tbl.c.changeid == changes_tbl.c.changeid) &
+                    (changes_tbl.c.branch == branch))
+            q = sa.select(
+                [ scheduler_changes_tbl.c.changeid, scheduler_changes_tbl.c.important ],
+                whereclause=wc)
+            return dict([ (r.changeid, [False,True][r.important]) for r in conn.execute(q) ])
+        return self.db.pool.do(thd)
+
+    def getSchedulerId(self, sched_name, sched_class):
+        """
+        Get the schedulerid for the given scheduler, creating a new schedulerid
+        if none is found.
+
+        Note that this makes no attempt to "claim" the schedulerid: schedulers
+        with the same name and class, but running in different masters, will be
+        assigned the same schedulerid - with disastrous results.
+
+        @param sched_name: the scheduler's configured name
+        @param sched_class: the class name of this scheduler
+        @returns: schedulerid, via a Deferred
+        """
+        def thd(conn):
+            # get a matching row, *or* one without a class_name (from 0.8.0)
+            schedulers_tbl = self.db.model.schedulers
+            q = schedulers_tbl.select(
+                    whereclause=(
+                        (schedulers_tbl.c.name == sched_name) &
+                        ((schedulers_tbl.c.class_name == sched_class) |
+                         (schedulers_tbl.c.class_name == ''))))
+            res = conn.execute(q)
+            row = res.fetchone()
+            res.close()
+
+            # if no existing row, then insert a new one and return it.  There
+            # is no protection against races here, but that's OK - the worst
+            # that happens is two sourcestamps with identical content; before
+            # 0.8.4 this was always the case.
+            if not row:
+                q = schedulers_tbl.insert()
+                res = conn.execute(q,
+                        name=sched_name,
+                        class_name=sched_class,
+                        state='{}')
+                return res.inserted_primary_key[0]
+
+            # upgrade the row with the class name, if necessary
+            if row.class_name == '':
+                q = schedulers_tbl.update(
+                    whereclause=(
+                        (schedulers_tbl.c.name == sched_name) &
+                        (schedulers_tbl.c.class_name == '')))
+                conn.execute(q, class_name=sched_class)
+            return row.schedulerid
+        return self.db.pool.do(thd)
deleted file mode 100644
deleted file mode 100644
--- a/master/buildbot/db/schema/base.py
+++ /dev/null
@@ -1,27 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-class Upgrader(object):
-
-    def __init__(self, dbapi, conn, basedir, quiet=False):
-        self.dbapi = dbapi
-        self.conn = conn
-        self.basedir = basedir
-        self.quiet = quiet
-
-        self.dbapiName = dbapi.__name__
-
-    def upgrade(self):
-        raise NotImplementedError
deleted file mode 100644
--- a/master/buildbot/db/schema/manager.py
+++ /dev/null
@@ -1,84 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-from twisted.python import reflect
-
-# note that schema modules are not loaded unless an upgrade is taking place
-
-CURRENT_VERSION = 6
-
-class DBSchemaManager(object):
-    """
-    This class is responsible for managing the database schema and upgrading it
-    as necessary.  This includes both the *actual* database and the old pickle
-    database, as migrations move data between the two.
-
-    Note that this class is *entirely synchronous*!  Performing any other operations
-    while changing the schema is just asking for trouble.
-    """
-    def __init__(self, spec, basedir):
-        self.spec = spec
-        self.basedir = basedir
-        self.dbapi = self.spec.get_dbapi()
-
-    def get_db_version(self, conn=None):
-        """
-        Get the current schema version for this database
-        """
-        close_conn = False
-        if not conn:
-            conn = self.spec.get_sync_connection()
-            close_conn = True
-        c = conn.cursor()
-        try:
-            try:
-                c.execute("SELECT version FROM version")
-                rows = c.fetchall()
-                assert len(rows) == 1, "%i rows in version table! (should only be 1)" % len(rows)
-                return rows[0][0]
-            except (self.dbapi.OperationalError, self.dbapi.ProgrammingError):
-                # no version table = version 0
-                return 0
-        finally:
-            if close_conn:
-                conn.close()
-
-    def get_current_version(self):
-        """
-        Get the current db version for this release of buildbot
-        """
-        return CURRENT_VERSION
-
-    def is_current(self):
-        """
-        Is this database current?
-        """
-        return self.get_db_version() == self.get_current_version()
-
-    def upgrade(self, quiet=False):
-        """
-        Upgrade this database to the current version
-        """
-        conn = self.spec.get_sync_connection()
-        try:
-            while self.get_db_version() < self.get_current_version():
-                next_version = self.get_db_version() + 1
-                next_version_module = reflect.namedModule("buildbot.db.schema.v%d" % next_version)
-                upg = next_version_module.Upgrader(self.dbapi, conn, self.basedir, quiet)
-                upg.upgrade()
-                conn.commit()
-                assert self.get_db_version() == next_version
-        finally:
-            conn.close()
deleted file mode 100644
--- a/master/buildbot/db/schema/tables.sql
+++ /dev/null
@@ -1,181 +0,0 @@
-CREATE TABLE buildrequests (
-    `id` INTEGER PRIMARY KEY AUTO_INCREMENT,
-
-    -- every BuildRequest has a BuildSet
-    -- the sourcestampid and reason live in the BuildSet
-    `buildsetid` INTEGER NOT NULL,
-
-    `buildername` VARCHAR(256) NOT NULL,
-
-    `priority` INTEGER NOT NULL default 0,
-
-    -- claimed_at is the time at which a master most recently asserted that
-    -- it is responsible for running the build: this will be updated
-    -- periodically to maintain the claim
-    `claimed_at` INTEGER default 0,
-
-    -- claimed_by indicates which buildmaster has claimed this request. The
-    -- 'name' contains hostname/basedir, and will be the same for subsequent
-    -- runs of any given buildmaster. The 'incarnation' contains bootime/pid,
-    -- and will be different for subsequent runs. This allows each buildmaster
-    -- to distinguish their current claims, their old claims, and the claims
-    -- of other buildmasters, to treat them each appropriately.
-    `claimed_by_name` VARCHAR(256) default NULL,
-    `claimed_by_incarnation` VARCHAR(256) default NULL,
-
-    `complete` INTEGER default 0, -- complete=0 means 'pending'
-
-     -- results is only valid when complete==1
-    `results` SMALLINT, -- 0=SUCCESS,1=WARNINGS,etc, from status/builder.py
-
-    `submitted_at` INTEGER NOT NULL,
-
-    `complete_at` INTEGER
-);
-CREATE TABLE builds (
-    `id` INTEGER PRIMARY KEY AUTO_INCREMENT,
-    `number` INTEGER NOT NULL, -- BuilderStatus.getBuild(number)
-    -- 'number' is scoped to both the local buildmaster and the buildername
-    `brid` INTEGER NOT NULL, -- matches buildrequests.id
-    `start_time` INTEGER NOT NULL,
-    `finish_time` INTEGER
-);
-CREATE TABLE buildset_properties (
-    `buildsetid` INTEGER NOT NULL,
-    `property_name` VARCHAR(256) NOT NULL,
-    `property_value` VARCHAR(1024) NOT NULL -- too short?
-);
-CREATE TABLE buildsets (
-    `id` INTEGER PRIMARY KEY AUTO_INCREMENT,
-    `external_idstring` VARCHAR(256),
-    `reason` VARCHAR(256),
-    `sourcestampid` INTEGER NOT NULL,
-    `submitted_at` INTEGER NOT NULL,
-    `complete` SMALLINT NOT NULL default 0,
-    `complete_at` INTEGER,
-    `results` SMALLINT -- 0=SUCCESS,2=FAILURE, from status/builder.py
-     -- results is NULL until complete==1
-);
-CREATE TABLE change_files (
-    `changeid` INTEGER NOT NULL,
-    `filename` VARCHAR(1024) NOT NULL
-);
-CREATE TABLE change_links (
-    `changeid` INTEGER NOT NULL,
-    `link` VARCHAR(1024) NOT NULL
-);
-CREATE TABLE change_properties (
-    `changeid` INTEGER NOT NULL,
-    `property_name` VARCHAR(256) NOT NULL,
-    `property_value` VARCHAR(1024) NOT NULL -- too short?
-);
-CREATE TABLE changes (
-    `changeid` INTEGER PRIMARY KEY AUTO_INCREMENT, -- also serves as 'change number'
-    `author` VARCHAR(1024) NOT NULL,
-    `comments` VARCHAR(1024) NOT NULL, -- too short?
-    `is_dir` SMALLINT NOT NULL, -- old, for CVS
-    `branch` VARCHAR(1024) NULL,
-    `revision` VARCHAR(256), -- CVS uses NULL. too short for darcs?
-    `revlink` VARCHAR(256) NULL,
-    `when_timestamp` INTEGER NOT NULL, -- copied from incoming Change
-    `category` VARCHAR(256) NULL,
-
-    -- repository specifies, along with revision and branch, the
-    -- source tree in which this change was detected.
-    `repository` text not null default '',
-
-    -- project names the project this source code represents.  It is used
-    -- later to filter changes
-    `project` text not null default ''
-);
-
-CREATE TABLE patches (
-    `id` INTEGER PRIMARY KEY AUTO_INCREMENT,
-    `patchlevel` INTEGER NOT NULL,
-    `patch_base64` TEXT NOT NULL, -- encoded bytestring
-    `subdir` TEXT -- usually NULL
-);
-CREATE TABLE sourcestamp_changes (
-    `sourcestampid` INTEGER NOT NULL,
-    `changeid` INTEGER NOT NULL
-);
-CREATE TABLE sourcestamps (
-    `id` INTEGER PRIMARY KEY AUTO_INCREMENT,
-    `branch` VARCHAR(256) default NULL,
-    `revision` VARCHAR(256) default NULL,
-    `patchid` INTEGER default NULL,
-    `repository` TEXT not null default '',
-    `project` TEXT not null default ''
-);
-
---
--- Scheduler Tables
---
-
--- This table records the "state" for each scheduler.  This state is, at least,
--- the last change that was analyzed, but is stored in an opaque JSON object.
--- Note that schedulers are never deleted.
-CREATE TABLE schedulers (
-    `schedulerid` INTEGER PRIMARY KEY AUTO_INCREMENT, -- joins to other tables
-    `name` VARCHAR(128) NOT NULL, -- the scheduler's name according to master.cfg
-    `class_name` VARCHAR(128) NOT NULL, -- the scheduler's class
-    `state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary
-);
-CREATE UNIQUE INDEX `name_and_class` ON schedulers (`name`, `class_name`);
-
-
--- This stores "classified" changes that have not yet been "processed".  That
--- is, the scheduler has looked at these changes and determined that something
--- should be done, but that hasn't happened yet.  Rows are "retired" from this
--- table as soon as the scheduler is done with the change.
-CREATE TABLE scheduler_changes (
-    `schedulerid` INTEGER,
-    `changeid` INTEGER,
-    `important` SMALLINT
-);
-
--- This stores buildsets in which a particular scheduler is interested.
--- On every run, a scheduler checks its upstream buildsets for completion
--- and reacts accordingly.  Records are never deleted from this table, but
--- active is set to 0 when the record is no longer necessary.
-CREATE TABLE scheduler_upstream_buildsets (
-    `buildsetid` INTEGER,
-    `schedulerid` INTEGER,
-    `active` SMALLINT
-);
-
---
--- Schema Information
---
-
--- database version; each upgrade script should change this
-CREATE TABLE version (
-    version INTEGER NOT NULL
-);
-
-CREATE INDEX `buildrequests_buildsetid` ON `buildrequests` (`buildsetid`);
-CREATE INDEX `buildrequests_buildername` ON `buildrequests` (`buildername` (255));
-CREATE INDEX `buildrequests_complete` ON `buildrequests` (`complete`);
-CREATE INDEX `buildrequests_claimed_at` ON `buildrequests` (`claimed_at`);
-CREATE INDEX `buildrequests_claimed_by_name` ON `buildrequests` (`claimed_by_name` (255));
-CREATE INDEX `builds_number` ON `builds` (`number`);
-CREATE INDEX `builds_brid` ON `builds` (`brid`);
-CREATE INDEX `buildsets_complete` ON `buildsets` (`complete`);
-CREATE INDEX `buildsets_submitted_at` ON `buildsets` (`submitted_at`);
-CREATE INDEX `buildset_properties_buildsetid` ON `buildset_properties` (`buildsetid`);
-CREATE INDEX `changes_branch` ON `changes` (`branch` (255));
-CREATE INDEX `changes_revision` ON `changes` (`revision` (255));
-CREATE INDEX `changes_author` ON `changes` (`author` (255));
-CREATE INDEX `changes_category` ON `changes` (`category` (255));
-CREATE INDEX `changes_when_timestamp` ON `changes` (`when_timestamp`);
-CREATE INDEX `change_files_changeid` ON `change_files` (`changeid`);
-CREATE INDEX `change_links_changeid` ON `change_links` (`changeid`);
-CREATE INDEX `change_properties_changeid` ON `change_properties` (`changeid`);
-CREATE INDEX `scheduler_changes_schedulerid` ON `scheduler_changes` (`schedulerid`);
-CREATE INDEX `scheduler_changes_changeid` ON `scheduler_changes` (`changeid`);
-CREATE INDEX `scheduler_upstream_buildsets_buildsetid` ON `scheduler_upstream_buildsets` (`buildsetid`);
-CREATE INDEX `scheduler_upstream_buildsets_schedulerid` ON `scheduler_upstream_buildsets` (`schedulerid`);
-CREATE INDEX `scheduler_upstream_buildsets_active` ON `scheduler_upstream_buildsets` (`active`);
-CREATE INDEX `sourcestamp_changes_sourcestampid` ON `sourcestamp_changes` (`sourcestampid`);
-
-INSERT INTO version VALUES(5);
deleted file mode 100644
--- a/master/buildbot/db/schema/v1.py
+++ /dev/null
@@ -1,349 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-import cPickle
-import textwrap
-import os
-import sys
-
-from twisted.persisted import styles
-
-from buildbot.db import util
-from buildbot.db.schema import base
-from buildbot.util import json
-
-# This is version 1, so it introduces a lot of new tables over version 0,
-# which had no database.
-
-TABLES = [
-    # the schema here is defined as version 1
-    textwrap.dedent("""
-        CREATE TABLE version (
-            version INTEGER NOT NULL -- contains one row, currently set to 1
-        );
-    """),
-
-    # last_access is used for logging, to record the last time that each
-    # client (or rather class of clients) touched the DB. The idea is that if
-    # something gets weird, you can check this and discover that you have an
-    # older tool (which uses a different schema) mucking things up.
-    textwrap.dedent("""
-        CREATE TABLE last_access (
-            `who` VARCHAR(256) NOT NULL, -- like 'buildbot-0.8.0'
-            `writing` INTEGER NOT NULL, -- 1 if you are writing, 0 if you are reading
-            -- PRIMARY KEY (who, writing),
-            `last_access` TIMESTAMP     -- seconds since epoch
-        );
-    """),
-
-    textwrap.dedent("""
-        CREATE TABLE changes_nextid (next_changeid INTEGER);
-    """),
-
-    textwrap.dedent("""
-        -- Changes are immutable: once added, never changed
-        CREATE TABLE changes (
-            `changeid` INTEGER PRIMARY KEY NOT NULL, -- also serves as 'change number'
-            `author` VARCHAR(1024) NOT NULL,
-            `comments` VARCHAR(1024) NOT NULL, -- too short?
-            `is_dir` SMALLINT NOT NULL, -- old, for CVS
-            `branch` VARCHAR(1024) NULL,
-            `revision` VARCHAR(256), -- CVS uses NULL. too short for darcs?
-            `revlink` VARCHAR(256) NULL,
-            `when_timestamp` INTEGER NOT NULL, -- copied from incoming Change
-            `category` VARCHAR(256) NULL
-        );
-    """),
-
-    textwrap.dedent("""
-        CREATE TABLE change_links (
-            `changeid` INTEGER NOT NULL,
-            `link` VARCHAR(1024) NOT NULL
-        );
-    """),
-
-    textwrap.dedent("""
-        CREATE TABLE change_files (
-            `changeid` INTEGER NOT NULL,
-            `filename` VARCHAR(1024) NOT NULL
-        );
-    """),
-
-    textwrap.dedent("""
-        CREATE TABLE change_properties (
-            `changeid` INTEGER NOT NULL,
-            `property_name` VARCHAR(256) NOT NULL,
-            `property_value` VARCHAR(1024) NOT NULL -- too short?
-        );
-    """),
-
-    # Scheduler tables
-    textwrap.dedent("""
-        CREATE TABLE schedulers (
-            `schedulerid` INTEGER PRIMARY KEY, -- joins to other tables
-            `name` VARCHAR(127) UNIQUE NOT NULL,
-            `state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary
-        );
-    """),
-
-    textwrap.dedent("""
-        CREATE TABLE scheduler_changes (
-            `schedulerid` INTEGER,
-            `changeid` INTEGER,
-            `important` SMALLINT
-        );
-    """),
-
-    textwrap.dedent("""
-        CREATE TABLE scheduler_upstream_buildsets (
-            `buildsetid` INTEGER,
-            `schedulerid` INTEGER,
-            `active` SMALLINT
-        );
-    """),
-
-    # SourceStamps
-    textwrap.dedent("""
-        -- SourceStamps are immutable: once added, never changed
-        CREATE TABLE sourcestamps (
-            `id` INTEGER PRIMARY KEY,
-            `branch` VARCHAR(256) default NULL,
-            `revision` VARCHAR(256) default NULL,
-            `patchid` INTEGER default NULL
-        );
-    """),
-    textwrap.dedent("""
-        CREATE TABLE patches (
-            `id` INTEGER PRIMARY KEY,
-            `patchlevel` INTEGER NOT NULL,
-            `patch_base64` TEXT NOT NULL, -- encoded bytestring
-            `subdir` TEXT -- usually NULL
-        );
-    """),
-    textwrap.dedent("""
-        CREATE TABLE sourcestamp_changes (
-            `sourcestampid` INTEGER NOT NULL,
-            `changeid` INTEGER NOT NULL
-        );
-    """),
-
-    # BuildRequests
-    textwrap.dedent("""
-        -- BuildSets are mutable. Python code may not cache them. Every
-        -- BuildRequest must have exactly one associated BuildSet.
-        CREATE TABLE buildsets (
-            `id` INTEGER PRIMARY KEY NOT NULL,
-            `external_idstring` VARCHAR(256),
-            `reason` VARCHAR(256),
-            `sourcestampid` INTEGER NOT NULL,
-            `submitted_at` INTEGER NOT NULL,
-            `complete` SMALLINT NOT NULL default 0,
-            `complete_at` INTEGER,
-            `results` SMALLINT -- 0=SUCCESS,2=FAILURE, from status/builder.py
-             -- results is NULL until complete==1
-        );
-    """),
-    textwrap.dedent("""
-        CREATE TABLE buildset_properties (
-            `buildsetid` INTEGER NOT NULL,
-            `property_name` VARCHAR(256) NOT NULL,
-            `property_value` VARCHAR(1024) NOT NULL -- too short?
-        );
-    """),
-
-    textwrap.dedent("""
-        -- the buildrequests table represents the queue of builds that need to be
-        -- done. In an idle buildbot, all requests will have complete=1.
-        -- BuildRequests are mutable. Python code may not cache them.
-        CREATE TABLE buildrequests (
-            `id` INTEGER PRIMARY KEY NOT NULL,
-
-            -- every BuildRequest has a BuildSet
-            -- the sourcestampid and reason live in the BuildSet
-            `buildsetid` INTEGER NOT NULL,
-
-            `buildername` VARCHAR(256) NOT NULL,
-
-            `priority` INTEGER NOT NULL default 0,
-
-            -- claimed_at is the time at which a master most recently asserted that
-            -- it is responsible for running the build: this will be updated
-            -- periodically to maintain the claim
-            `claimed_at` INTEGER default 0,
-
-            -- claimed_by indicates which buildmaster has claimed this request. The
-            -- 'name' contains hostname/basedir, and will be the same for subsequent
-            -- runs of any given buildmaster. The 'incarnation' contains bootime/pid,
-            -- and will be different for subsequent runs. This allows each buildmaster
-            -- to distinguish their current claims, their old claims, and the claims
-            -- of other buildmasters, to treat them each appropriately.
-            `claimed_by_name` VARCHAR(256) default NULL,
-            `claimed_by_incarnation` VARCHAR(256) default NULL,
-
-            `complete` INTEGER default 0, -- complete=0 means 'pending'
-
-             -- results is only valid when complete==1
-            `results` SMALLINT, -- 0=SUCCESS,1=WARNINGS,etc, from status/builder.py
-
-            `submitted_at` INTEGER NOT NULL,
-
-            `complete_at` INTEGER
-        );
-    """),
-
-    textwrap.dedent("""
-        -- this records which builds have been started for each request
-        CREATE TABLE builds (
-            `id` INTEGER PRIMARY KEY NOT NULL,
-            `number` INTEGER NOT NULL, -- BuilderStatus.getBuild(number)
-            -- 'number' is scoped to both the local buildmaster and the buildername
-            `brid` INTEGER NOT NULL, -- matches buildrequests.id
-            `start_time` INTEGER NOT NULL,
-            `finish_time` INTEGER
-        );
-    """),
-]
-
-class Upgrader(base.Upgrader):
-    def upgrade(self):
-        self.test_unicode()
-        self.add_tables()
-        self.migrate_changes()
-        self.set_version()
-
-    def test_unicode(self):
-        # first, create a test table
-        c = self.conn.cursor()
-        c.execute("CREATE TABLE test_unicode (`name` VARCHAR(100))")
-        q = util.sql_insert(self.dbapi, 'test_unicode', ["name"])
-        try:
-            val = u"Frosty the \N{SNOWMAN}"
-            c.execute(q, [val])
-            c.execute("SELECT * FROM test_unicode")
-            row = c.fetchall()[0]
-            if row[0] != val:
-                raise UnicodeError("Your database doesn't support unicode data; for MySQL, set the default collation to utf8_general_ci.")
-        finally:
-            pass
-            c.execute("DROP TABLE test_unicode")
-
-    def add_tables(self):
-        # first, add all of the tables
-        c = self.conn.cursor()
-        for t in TABLES:
-            try:
-                c.execute(t)
-            except:
-                print >>sys.stderr, "error executing SQL query: %s" % t
-                raise
-
-    def _addChangeToDatabase(self, change, cursor):
-        # strip None from any of these values, just in case
-        def remove_none(x):
-            if x is None: return u""
-            elif isinstance(x, str):
-                return x.decode("utf8")
-            else:
-                return x
-        try:
-            values = tuple(remove_none(x) for x in
-                             (change.number, change.who,
-                              change.comments, change.isdir,
-                              change.branch, change.revision, change.revlink,
-                              change.when, change.category))
-        except UnicodeDecodeError, e:
-            raise UnicodeError("Trying to import change data as UTF-8 failed.  Please look at contrib/fix_changes_pickle_encoding.py: %s" % str(e))
-
-        q = util.sql_insert(self.dbapi, 'changes',
-            """changeid author comments is_dir branch revision
-               revlink when_timestamp category""".split())
-        cursor.execute(q, values)
-
-        for link in change.links:
-            cursor.execute(util.sql_insert(self.dbapi, 'change_links', ('changeid', 'link')),
-                          (change.number, link))
-
-        # sometimes change.files contains nested lists -- why, I do not know!  But we deal with
-        # it all the same - see bug #915. We'll assume for now that change.files contains *either*
-        # lists of filenames or plain filenames, not both.
-        def flatten(l):
-            if l and type(l[0]) == list:
-                rv = []
-                for e in l:
-                    if type(e) == list:
-                        rv.extend(e)
-                    else:
-                        rv.append(e)
-                return rv
-            else:
-                return l
-        for filename in flatten(change.files):
-            cursor.execute(util.sql_insert(self.dbapi, 'change_files', ('changeid', 'filename')),
-                          (change.number, filename))
-        for propname,propvalue in change.properties.properties.items():
-            encoded_value = json.dumps(propvalue)
-            cursor.execute(util.sql_insert(self.dbapi, 'change_properties',
-                                  ('changeid', 'property_name', 'property_value')),
-                          (change.number, propname, encoded_value))
-
-    def migrate_changes(self):
-        # if we still have a changes.pck, then we need to migrate it
-        changes_pickle = os.path.join(self.basedir, "changes.pck")
-        if os.path.exists(changes_pickle):
-            if not self.quiet: print "migrating changes.pck to database"
-
-            # 'source' will be an old b.c.changes.ChangeMaster instance, with a
-            # .changes attribute
-            source = cPickle.load(open(changes_pickle,"rb"))
-            styles.doUpgrade()
-
-            if not self.quiet: print " (%d Change objects)" % len(source.changes)
-
-            # first, scan for changes without a number.  If we find any, then we'll
-            # renumber the changes sequentially
-            have_unnumbered = False
-            for c in source.changes:
-                if c.revision and c.number is None:
-                    have_unnumbered = True
-                    break
-            if have_unnumbered:
-                n = 1
-                for c in source.changes:
-                    if c.revision:
-                        c.number = n
-                        n = n + 1
-
-            # insert the changes
-            cursor = self.conn.cursor()
-            for c in source.changes:
-                if not c.revision:
-                    continue
-                self._addChangeToDatabase(c, cursor)
-
-            # update next_changeid
-            max_changeid = max([ c.number for c in source.changes if c.revision ] + [ 0 ])
-            cursor.execute("""INSERT into changes_nextid VALUES (%d)""" % (max_changeid+1))
-
-            if not self.quiet:
-                print "moving changes.pck to changes.pck.old; delete it or keep it as a backup"
-            os.rename(changes_pickle, changes_pickle+".old")
-        else:
-            c = self.conn.cursor()
-            c.execute("""INSERT into changes_nextid VALUES (1)""")
-
-    def set_version(self):
-        c = self.conn.cursor()
-        c.execute("""INSERT INTO version VALUES (1)""")
-
deleted file mode 100644
--- a/master/buildbot/db/schema/v2.py
+++ /dev/null
@@ -1,50 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-
-from buildbot.db.schema import base
-
-class Upgrader(base.Upgrader):
-    def upgrade(self):
-        self.add_columns()
-        self.set_version()
-
-    def add_columns(self):
-        if self.dbapiName == 'MySQLdb':
-            default_text = ""
-        else:
-            default_text = "default ''"
-
-        cursor = self.conn.cursor()
-        cursor.execute("""
-        ALTER TABLE changes
-            add column `repository` text not null %s
-        """ % default_text)
-        cursor.execute("""
-        ALTER TABLE changes
-            add column `project` text not null %s
-        """ % default_text)
-        cursor.execute("""
-        ALTER TABLE sourcestamps
-            add column `repository` text not null %s
-        """ % default_text)
-        cursor.execute("""
-        ALTER TABLE sourcestamps
-            add column `project` text not null %s
-        """ % default_text)
-
-    def set_version(self):
-        c = self.conn.cursor()
-        c.execute("""UPDATE version set version = 2 where version = 1""")
deleted file mode 100644
--- a/master/buildbot/db/schema/v3.py
+++ /dev/null
@@ -1,74 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-from buildbot.db.schema import base
-
-class Upgrader(base.Upgrader):
-    def upgrade(self):
-        self.migrate_schedulers()
-        self.set_version()
-
-    def migrate_schedulers(self):
-        cursor = self.conn.cursor()
-        # If this fails, there's no cleaning up to do
-        cursor.execute("""
-            ALTER TABLE schedulers
-                RENAME TO schedulers_old
-        """)
-
-        try:
-            cursor.execute("""
-                CREATE TABLE schedulers (
-                    `schedulerid` INTEGER PRIMARY KEY, -- joins to other tables
-                    `name` VARCHAR(127) NOT NULL, -- the scheduler's name according to master.cfg
-                    `class_name` VARCHAR(127) NOT NULL, -- the scheduler's class
-                    `state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary
-                );
-            """)
-        except:
-            # Restore the original table
-            cursor.execute("""
-                ALTER TABLE schedulers_old
-                    RENAME TO schedulers
-            """)
-            raise
-
-        try:
-            cursor.execute("""
-                CREATE UNIQUE INDEX `name_and_class` ON
-                    schedulers (`name`, `class_name`)
-            """)
-
-            cursor.execute("""
-                INSERT INTO schedulers (`schedulerid`, `name`, `state`, `class_name`)
-                    SELECT `schedulerid`, `name`, `state`, '' FROM schedulers_old
-            """)
-            cursor.execute("""
-                DROP TABLE schedulers_old
-            """)
-        except:
-            # Clean up the new table, and restore the original
-            cursor.execute("""
-                DROP TABLE schedulers
-            """)
-            cursor.execute("""
-                ALTER TABLE schedulers_old
-                    RENAME TO schedulers
-            """)
-            raise
-
-    def set_version(self):
-        c = self.conn.cursor()
-        c.execute("""UPDATE version set version = 3 where version = 2""")
deleted file mode 100644
--- a/master/buildbot/db/schema/v4.py
+++ /dev/null
@@ -1,224 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-from buildbot.db.schema import base
-
-class Upgrader(base.Upgrader):
-    def upgrade(self):
-        self.migrate_buildrequests()
-        self.migrate_builds()
-        self.migrate_buildsets()
-        self.migrate_changes()
-        self.migrate_patches()
-        self.migrate_sourcestamps()
-        self.migrate_schedulers()
-        self.set_version()
-
-    def makeAutoincColumn(self, name):
-        if self.dbapiName == 'MySQLdb':
-            return "`%s` INTEGER PRIMARY KEY AUTO_INCREMENT" % name
-        elif self.dbapiName in ('sqlite3', 'pysqlite2.dbapi2'):
-            return "`%s` INTEGER PRIMARY KEY AUTOINCREMENT" % name
-        raise ValueError("Unsupported dbapi: %s" % self.dbapiName)
-
-    def migrate_table(self, table_name, schema):
-        names = {
-            'old_name': "%s_old" % table_name,
-            'table_name': table_name,
-        }
-        cursor = self.conn.cursor()
-        # If this fails, there's no cleaning up to do
-        cursor.execute("""
-            ALTER TABLE %(table_name)s
-                RENAME TO %(old_name)s
-        """ % names)
-
-        try:
-            cursor.execute(schema)
-        except:
-            # Restore the original table
-            cursor.execute("""
-                ALTER TABLE %(old_name)s
-                    RENAME TO %(table_name)s
-            """ % names)
-            raise
-
-        try:
-            cursor.execute("""
-                INSERT INTO %(table_name)s
-                    SELECT * FROM %(old_name)s
-            """ % names)
-            cursor.execute("""
-                DROP TABLE %(old_name)s
-            """ % names)
-        except:
-            # Clean up the new table, and restore the original
-            cursor.execute("""
-                DROP TABLE %(table_name)s
-            """ % names)
-            cursor.execute("""
-                ALTER TABLE %(old_name)s
-                    RENAME TO %(table_name)s
-            """ % names)
-            raise
-
-    def set_version(self):
-        c = self.conn.cursor()
-        c.execute("""UPDATE version set version = 4 where version = 3""")
-
-    def migrate_schedulers(self):
-        schedulerid_col = self.makeAutoincColumn('schedulerid')
-        schema = """
-            CREATE TABLE schedulers (
-                %(schedulerid_col)s, -- joins to other tables
-                `name` VARCHAR(100) NOT NULL, -- the scheduler's name according to master.cfg
-                `class_name` VARCHAR(100) NOT NULL, -- the scheduler's class
-                `state` VARCHAR(1024) NOT NULL -- JSON-encoded state dictionary
-            );
-        """ % {'schedulerid_col': schedulerid_col}
-        self.migrate_table('schedulers', schema)
-
-        # Fix up indices
-        cursor = self.conn.cursor()
-        cursor.execute("""
-            CREATE UNIQUE INDEX `name_and_class` ON
-                schedulers (`name`, `class_name`)
-        """)
-
-    def migrate_builds(self):
-        buildid_col = self.makeAutoincColumn('id')
-        schema = """
-            CREATE TABLE builds (
-                %(buildid_col)s,
-                `number` INTEGER NOT NULL, -- BuilderStatus.getBuild(number)
-                -- 'number' is scoped to both the local buildmaster and the buildername
-                `brid` INTEGER NOT NULL, -- matches buildrequests.id
-                `start_time` INTEGER NOT NULL,
-                `finish_time` INTEGER
-            );
-        """ % {'buildid_col': buildid_col}
-        self.migrate_table('builds', schema)
-
-    def migrate_changes(self):
-        changeid_col = self.makeAutoincColumn('changeid')
-        schema = """
-            CREATE TABLE changes (
-                %(changeid_col)s, -- also serves as 'change number'
-                `author` VARCHAR(1024) NOT NULL,
-                `comments` VARCHAR(1024) NOT NULL, -- too short?
-                `is_dir` SMALLINT NOT NULL, -- old, for CVS
-                `branch` VARCHAR(1024) NULL,
-                `revision` VARCHAR(256), -- CVS uses NULL. too short for darcs?
-                `revlink` VARCHAR(256) NULL,
-                `when_timestamp` INTEGER NOT NULL, -- copied from incoming Change
-                `category` VARCHAR(256) NULL,
-
-                -- repository specifies, along with revision and branch, the
-                -- source tree in which this change was detected.
-                `repository` TEXT NOT NULL default '',
-
-                -- project names the project this source code represents.  It is used
-                -- later to filter changes
-                `project` TEXT NOT NULL default ''
-            );
-        """ % {'changeid_col': changeid_col}
-        self.migrate_table('changes', schema)
-
-        # Drop changes_nextid columnt
-        cursor = self.conn.cursor()
-        cursor.execute("DROP TABLE changes_nextid")
-
-    def migrate_buildrequests(self):
-        buildrequestid_col = self.makeAutoincColumn('id')
-        schema = """
-            CREATE TABLE buildrequests (
-                %(buildrequestid_col)s,
-
-                -- every BuildRequest has a BuildSet
-                -- the sourcestampid and reason live in the BuildSet
-                `buildsetid` INTEGER NOT NULL,
-
-                `buildername` VARCHAR(256) NOT NULL,
-
-                `priority` INTEGER NOT NULL default 0,
-
-                -- claimed_at is the time at which a master most recently asserted that
-                -- it is responsible for running the build: this will be updated
-                -- periodically to maintain the claim
-                `claimed_at` INTEGER default 0,
-
-                -- claimed_by indicates which buildmaster has claimed this request. The
-                -- 'name' contains hostname/basedir, and will be the same for subsequent
-                -- runs of any given buildmaster. The 'incarnation' contains bootime/pid,
-                -- and will be different for subsequent runs. This allows each buildmaster
-                -- to distinguish their current claims, their old claims, and the claims
-                -- of other buildmasters, to treat them each appropriately.
-                `claimed_by_name` VARCHAR(256) default NULL,
-                `claimed_by_incarnation` VARCHAR(256) default NULL,
-
-                `complete` INTEGER default 0, -- complete=0 means 'pending'
-
-                 -- results is only valid when complete==1
-                `results` SMALLINT, -- 0=SUCCESS,1=WARNINGS,etc, from status/builder.py
-
-                `submitted_at` INTEGER NOT NULL,
-
-                `complete_at` INTEGER
-            );
-        """ % {'buildrequestid_col': buildrequestid_col}
-        self.migrate_table('buildrequests', schema)
-
-    def migrate_buildsets(self):
-        buildsetsid_col = self.makeAutoincColumn('id')
-        schema = """
-            CREATE TABLE buildsets (
-                %(buildsetsid_col)s,
-                `external_idstring` VARCHAR(256),
-                `reason` VARCHAR(256),
-                `sourcestampid` INTEGER NOT NULL,
-                `submitted_at` INTEGER NOT NULL,
-                `complete` SMALLINT NOT NULL default 0,
-                `complete_at` INTEGER,
-                `results` SMALLINT -- 0=SUCCESS,2=FAILURE, from status/builder.py
-                 -- results is NULL until complete==1
-            );
-        """ % {'buildsetsid_col': buildsetsid_col}
-        self.migrate_table("buildsets", schema)
-
-    def migrate_patches(self):
-        patchesid_col = self.makeAutoincColumn('id')
-        schema = """
-            CREATE TABLE patches (
-                %(patchesid_col)s,
-                `patchlevel` INTEGER NOT NULL,
-                `patch_base64` TEXT NOT NULL, -- encoded bytestring
-                `subdir` TEXT -- usually NULL
-            );
-        """ % {'patchesid_col': patchesid_col}
-        self.migrate_table("patches", schema)
-
-    def migrate_sourcestamps(self):
-        sourcestampsid_col = self.makeAutoincColumn('id')
-        schema = """
-            CREATE TABLE sourcestamps (
-                %(sourcestampsid_col)s,
-                `branch` VARCHAR(256) default NULL,
-                `revision` VARCHAR(256) default NULL,
-                `patchid` INTEGER default NULL,
-                `repository` TEXT not null default '',
-                `project` TEXT not null default ''
-            );
-        """ % {'sourcestampsid_col': sourcestampsid_col}
-        self.migrate_table("sourcestamps", schema)
deleted file mode 100644
--- a/master/buildbot/db/schema/v5.py
+++ /dev/null
@@ -1,69 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-from buildbot.db.schema import base
-
-class Upgrader(base.Upgrader):
-    def upgrade(self):
-        self.add_index("buildrequests", "buildsetid")
-        self.add_index("buildrequests", "buildername", 255)
-        self.add_index("buildrequests", "complete")
-        self.add_index("buildrequests", "claimed_at")
-        self.add_index("buildrequests", "claimed_by_name", 255)
-
-        self.add_index("builds", "number")
-        self.add_index("builds", "brid")
-
-        self.add_index("buildsets", "complete")
-        self.add_index("buildsets", "submitted_at")
-
-        self.add_index("buildset_properties", "buildsetid")
-
-        self.add_index("changes", "branch", 255)
-        self.add_index("changes", "revision", 255)
-        self.add_index("changes", "author", 255)
-        self.add_index("changes", "category", 255)
-        self.add_index("changes", "when_timestamp")
-
-        self.add_index("change_files", "changeid")
-        self.add_index("change_links", "changeid")
-        self.add_index("change_properties", "changeid")
-
-        # schedulers already has an index
-
-        self.add_index("scheduler_changes", "schedulerid")
-        self.add_index("scheduler_changes", "changeid")
-
-        self.add_index("scheduler_upstream_buildsets", "buildsetid")
-        self.add_index("scheduler_upstream_buildsets", "schedulerid")
-        self.add_index("scheduler_upstream_buildsets", "active")
-
-        # sourcestamps are only queried by id, no need for additional indexes
-
-        self.add_index("sourcestamp_changes", "sourcestampid")
-
-        self.set_version()
-
-    def add_index(self, table, column, length=None):
-        lengthstr=""
-        if length is not None and self.dbapiName == 'MySQLdb':
-            lengthstr = " (%i)" % length
-        q = "CREATE INDEX `%(table)s_%(column)s` ON `%(table)s` (`%(column)s`%(lengthstr)s)"
-        cursor = self.conn.cursor()
-        cursor.execute(q % {'table': table, 'column': column, 'lengthstr': lengthstr})
-
-    def set_version(self):
-        c = self.conn.cursor()
-        c.execute("""UPDATE version set version = 5 where version = 4""")
deleted file mode 100644
--- a/master/buildbot/db/schema/v6.py
+++ /dev/null
@@ -1,23 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-from buildbot.db.schema import base
-
-class Upgrader(base.Upgrader):
-    def upgrade(self):
-        cursor = self.conn.cursor()
-        cursor.execute("DROP table last_access")
-        cursor.execute("""UPDATE version set version = 6 where version = 5""")
-
new file mode 100644
--- /dev/null
+++ b/master/buildbot/db/sourcestamps.py
@@ -0,0 +1,65 @@
+# This file is part of Buildbot.  Buildbot is free software: you can
+# redistribute it and/or modify it under the terms of the GNU General Public
+# License as published by the Free Software Foundation, version 2.
+#
+# This program is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc., 51
+# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Copyright Buildbot Team Members
+
+"""
+Support for creating and reading source stamps
+"""
+
+import base64
+from buildbot.db import base
+
+class SourceStampsConnectorComponent(base.DBConnectorComponent):
+    """
+    A DBConnectorComponent to handle source stamps in the database
+    """
+
+    def createSourceStamp(self, branch, revision, repository, project,
+                          patch_body=None, patch_level=0, patch_subdir=None,
+                          changeids=[]):
+        """
+        Create a new SourceStamp instance with the given attributes, and return
+        its sourcestamp ID, via a Deferred.
+        """
+        def thd(conn):
+            # handle inserting a patch
+            patchid = None
+            if patch_body is not None:
+                ins = self.db.model.patches.insert()
+                r = conn.execute(ins, dict(
+                    patchlevel=patch_level,
+                    patch_base64=base64.b64encode(patch_body),
+                    subdir=patch_subdir))
+                patchid = r.inserted_primary_key[0]
+
+            # insert the sourcestamp itself
+            ins = self.db.model.sourcestamps.insert()
+            r = conn.execute(ins, dict(
+                branch=branch,
+                revision=revision,
+                patchid=patchid,
+                repository=repository,
+                project=project))
+            ssid = r.inserted_primary_key[0]
+
+            # handle inserting change ids
+            if changeids:
+                ins = self.db.model.sourcestamp_changes.insert()
+                conn.execute(ins, [
+                    dict(sourcestampid=ssid, changeid=changeid)
+                    for changeid in changeids ])
+
+            # and return the new ssid
+            return ssid
+        return self.db.pool.do(thd)
deleted file mode 100644
--- a/master/buildbot/db/util.py
+++ /dev/null
@@ -1,32 +0,0 @@
-# This file is part of Buildbot.  Buildbot is free software: you can
-# redistribute it and/or modify it under the terms of the GNU General Public
-# License as published by the Free Software Foundation, version 2.
-#
-# This program is distributed in the hope that it will be useful, but WITHOUT
-# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
-# FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
-# details.
-#
-# You should have received a copy of the GNU General Public License along with
-# this program; if not, write to the Free Software Foundation, Inc., 51
-# Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-#
-# Copyright Buildbot Team Members
-
-def sql_insert(dbapi, table, columns):
-    """
-    Make an SQL insert statement for the given table and columns, using the
-    appropriate paramstyle for the dbi.  Note that this only supports positional
-    parameters.  This will need to be reworked if Buildbot supports a backend with
-    a name-based paramstyle.
-    """
-
-    if dbapi.paramstyle == 'qmark':
-        params = ",".join(("?",)*len(columns))
-    elif dbapi.paramstyle == 'numeric':
-        params = ",".join(":%d" % d for d in range(1, len(columns)+1))
-    elif dbapi.paramstyle == 'format':
-        params = ",".join(("%s",)*len(columns))
-    else:
-        raise RuntimeError("unsupported paramstyle %s" % dbapi.paramstyle)
-    return "INSERT INTO %s (%s) VALUES (%s)" % (table, ", ".join(columns), params)
--- a/master/buildbot/interfaces.py
+++ b/master/buildbot/interfaces.py
@@ -36,27 +36,30 @@ class LatentBuildSlaveFailedToSubstantia
 # other exceptions
 class BuildbotNotRunningError(Exception):
     pass
 
 class ParameterError(Exception):
     pass
 
 class IChangeSource(Interface):
-    """Object which feeds Change objects to the changemaster. When files or
-    directories are changed and the version control system provides some
-    kind of notification, this object should turn it into a Change object
-    and pass it through::
+    """
+    Service which feeds Change objects to the changemaster. When files or
+    directories are changed in version control, this object should represent
+    the changes as a change dictionary and call::
 
-      self.changemaster.addChange(change)
+      self.master.addChange(who=.., rev=.., ..)
+
+    See 'Writing Change Sources' in the manual for more information.
     """
+    master = Attribute('master',
+            'Pointer to BuildMaster, automatically set when started.')
 
     def describe():
-        """Should return a string which briefly describes this source. This
-        string will be displayed in an HTML status page."""
+        """Return a string which briefly describes this source."""
 
 class IScheduler(Interface):
     """I watch for Changes in the source tree and decide when to trigger
     Builds. I create BuildSet objects and submit them to the BuildMaster. I
     am a service, and the BuildMaster is always my parent.
 
     @ivar properties: properties to be applied to all builds started by this
     scheduler
@@ -1034,19 +1037,17 @@ class IStatusReceiver(Interface):
     def slaveConnected(slaveName):
         """The slave has connected."""
 
     def slaveDisconnected(slaveName):
         """The slave has disconnected."""
 
 class IControl(Interface):
     def addChange(change):
-        """Add a change to all builders. Each Builder will decide for
-        themselves whether the change is interesting or not, and may initiate
-        a build as a result."""
+        """Add a change to the change queue, for analysis by schedulers."""
 
     def submitBuildSet(builderNames, ss, reason, props=None, now=False):
         """Create a BuildSet, which will eventually cause a build of the
         given SourceStamp to be run on all of the named builders. This
         returns a BuildSetStatus object, which can be used to keep track of
         the builds that are performed.
 
         If now=True, and the builder has no slave attached, NoSlaveError will
--- a/master/buildbot/master.py
+++ b/master/buildbot/master.py
@@ -24,29 +24,26 @@ from twisted.python import log, componen
 from twisted.python.failure import Failure
 from twisted.internet import defer, reactor
 from twisted.spread import pb
 from twisted.application import service
 from twisted.application.internet import TimerService
 
 import buildbot
 import buildbot.pbmanager
-from buildbot.util import now, safeTranslate, eventual
+from buildbot.util import now, safeTranslate, eventual, subscription
 from buildbot.pbutil import NewCredPerspective
 from buildbot.process.builder import Builder, IDLE
 from buildbot.status.builder import Status, BuildSetStatus
-from buildbot.changes.changes import Change
 from buildbot.changes.manager import ChangeManager
 from buildbot import interfaces, locks
 from buildbot.process.properties import Properties
 from buildbot.config import BuilderConfig
 from buildbot.process.builder import BuilderControl
-from buildbot.db.dbspec import DBSpec
 from buildbot.db import connector, exceptions
-from buildbot.db.schema.manager import DBSchemaManager
 from buildbot.schedulers.manager import SchedulerManager
 from buildbot.util.loop import DelegateLoop
 
 ########################################
 
 class BotMaster(service.MultiService):
 
     """This is the master-side service which manages remote buildbot slaves.
@@ -551,25 +548,16 @@ class DebugPerspective(NewCredPerspectiv
         bpr.update(properties, "remote requestBuild")
         bc.submitBuildRequest(ss, reason, bpr)
 
     def perspective_pingBuilder(self, buildername):
         c = interfaces.IControl(self.master)
         bc = c.getBuilder(buildername)
         bc.ping()
 
-    def perspective_fakeChange(self, file, revision=None, who="fakeUser",
-                               branch=None, repository="", 
-                               project=""):
-        change = Change(who, [file], "some fake comments\n",
-                        branch=branch, revision=revision,
-                        repository=repository, project=project)
-        c = interfaces.IControl(self.master)
-        c.addChange(change)
-
     def perspective_setCurrentState(self, buildername, state):
         builder = self.botmaster.builders.get(buildername)
         if not builder: return
         if state == "offline":
             builder.statusbag.currentlyOffline()
         if state == "idle":
             builder.statusbag.currentlyIdle()
         if state == "waiting":
@@ -607,17 +595,17 @@ class BuildMaster(service.MultiService):
     manhole = None
     debugPassword = None
     projectName = "(unspecified)"
     projectURL = None
     buildbotURL = None
     change_svc = None
     properties = Properties()
 
-    def __init__(self, basedir, configFileName="master.cfg", db_spec=None):
+    def __init__(self, basedir, configFileName="master.cfg"):
         service.MultiService.__init__(self)
         self.setName("buildmaster")
         self.basedir = basedir
         self.configFileName = configFileName
 
         self.pbmanager = buildbot.pbmanager.PBManager()
         self.pbmanager.setServiceParent(self)
         "L{buildbot.pbmanager.PBManager} instance managing connections for this master"
@@ -643,27 +631,29 @@ class BuildMaster(service.MultiService):
         self.debugClientRegistration = None
 
         self.status = Status(self.botmaster, self.basedir)
         self.statusTargets = []
 
         self.db = None
         self.db_url = None
         self.db_poll_interval = _Unset
-        if db_spec:
-            self.loadDatabase(db_spec)
 
         # note that "read" here is taken in the past participal (i.e., "I read
         # the config already") rather than the imperative ("you should read the
         # config later")
         self.readConfig = False
         
         # create log_rotation object and set default parameters (used by WebStatus)
         self.log_rotation = LogRotation()
 
+        # subscription point to recieve changes; users should call subscribeToChanges
+        # to get into this subscription point
+        self._change_subscriptions = subscription.SubscriptionPoint("changes")
+
     def startService(self):
         service.MultiService.startService(self)
         if not self.readConfig:
             # TODO: consider catching exceptions during this call to
             # loadTheConfigFile and bailing (reactor.stop) if it fails,
             # since without a config file we can't do anything except reload
             # the config file, and it would be nice for the user to discover
             # this quickly.
@@ -705,412 +695,425 @@ class BuildMaster(service.MultiService):
             log.msg("error during loadConfig")
             log.err()
             log.msg("The new config file is unusable, so I'll ignore it.")
             log.msg("I will keep using the previous config file instead.")
             return # sorry unit tests
         f.close()
         return d # for unit tests
 
-    def loadConfig(self, f, check_synchronously_only=False):
+    def loadConfig(self, f, checkOnly=False):
         """Internal function to load a specific configuration file. Any
-        errors in the file will be signalled by raising an exception.
+        errors in the file will be signalled by raising a failure.  Returns
+        a deferred.
+        """
 
-        If check_synchronously_only=True, I will return (with None)
-        synchronously, after checking the config file for sanity, or raise an
-        exception. I may also emit some DeprecationWarnings.
+        # this entire operation executes in a deferred, so that any exceptions
+        # are automatically converted to a failure object.
+        d = defer.succeed(None)
 
-        If check_synchronously_only=False, I will return a Deferred that
-        fires (with None) when the configuration changes have been completed.
-        This may involve a round-trip to each buildslave that was involved."""
+        def do_load(_):
+            log.msg("configuration update started")
+
+            # execute the config file
 
-        localDict = {'basedir': os.path.expanduser(self.basedir)}
-        try:
-            exec f in localDict
-        except:
-            log.msg("error while parsing config file")
-            raise
+            localDict = {'basedir': os.path.expanduser(self.basedir)}
+            try:
+                exec f in localDict
+            except:
+                log.msg("error while parsing config file")
+                raise
 
-        try:
-            config = localDict['BuildmasterConfig']
-        except KeyError:
-            log.err("missing config dictionary")
-            log.err("config file must define BuildmasterConfig")
-            raise
+            try:
+                config = localDict['BuildmasterConfig']
+            except KeyError:
+                log.err("missing config dictionary")
+                log.err("config file must define BuildmasterConfig")
+                raise
+
+            # check for unknown keys
 
-        known_keys = ("slaves", "change_source",
-                      "schedulers", "builders", "mergeRequests",
-                      "slavePortnum", "debugPassword", "logCompressionLimit",
-                      "manhole", "status", "projectName", "projectURL",
-                      "buildbotURL", "properties", "prioritizeBuilders",
-                      "eventHorizon", "buildCacheSize", "changeCacheSize",
-                      "logHorizon", "buildHorizon", "changeHorizon",
-                      "logMaxSize", "logMaxTailSize", "logCompressionMethod",
-                      "db_url", "multiMaster", "db_poll_interval",
-                      )
-        for k in config.keys():
-            if k not in known_keys:
-                log.msg("unknown key '%s' defined in config dictionary" % k)
+            known_keys = ("slaves", "change_source",
+                          "schedulers", "builders", "mergeRequests",
+                          "slavePortnum", "debugPassword", "logCompressionLimit",
+                          "manhole", "status", "projectName", "projectURL",
+                          "buildbotURL", "properties", "prioritizeBuilders",
+                          "eventHorizon", "buildCacheSize", "changeCacheSize",
+                          "logHorizon", "buildHorizon", "changeHorizon",
+                          "logMaxSize", "logMaxTailSize", "logCompressionMethod",
+                          "db_url", "multiMaster", "db_poll_interval",
+                          )
+            for k in config.keys():
+                if k not in known_keys:
+                    log.msg("unknown key '%s' defined in config dictionary" % k)
 
-        try:
-            # required
-            schedulers = config['schedulers']
-            builders = config['builders']
-            slavePortnum = config['slavePortnum']
-            #slaves = config['slaves']
-            #change_source = config['change_source']
+            # load known keys into local vars, applying defaults
+
+            try:
+                # required
+                schedulers = config['schedulers']
+                builders = config['builders']
+                slavePortnum = config['slavePortnum']
+                #slaves = config['slaves']
+                #change_source = config['change_source']
 
-            # optional
-            db_url = config.get("db_url", "sqlite:///state.sqlite")
-            db_poll_interval = config.get("db_poll_interval", None)
-            debugPassword = config.get('debugPassword')
-            manhole = config.get('manhole')
-            status = config.get('status', [])
-            projectName = config.get('projectName')
-            projectURL = config.get('projectURL')
-            buildbotURL = config.get('buildbotURL')
-            properties = config.get('properties', {})
-            buildCacheSize = config.get('buildCacheSize', None)
-            changeCacheSize = config.get('changeCacheSize', None)
-            eventHorizon = config.get('eventHorizon', 50)
-            logHorizon = config.get('logHorizon', None)
-            buildHorizon = config.get('buildHorizon', None)
-            logCompressionLimit = config.get('logCompressionLimit', 4*1024)
-            if logCompressionLimit is not None and not \
-                    isinstance(logCompressionLimit, int):
-                raise ValueError("logCompressionLimit needs to be bool or int")
-            logCompressionMethod = config.get('logCompressionMethod', "bz2")
-            if logCompressionMethod not in ('bz2', 'gz'):
-                raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'")
-            logMaxSize = config.get('logMaxSize')
-            if logMaxSize is not None and not \
-                    isinstance(logMaxSize, int):
-                raise ValueError("logMaxSize needs to be None or int")
-            logMaxTailSize = config.get('logMaxTailSize')
-            if logMaxTailSize is not None and not \
-                    isinstance(logMaxTailSize, int):
-                raise ValueError("logMaxTailSize needs to be None or int")
-            mergeRequests = config.get('mergeRequests')
-            if mergeRequests not in (None, False) and not callable(mergeRequests):
-                raise ValueError("mergeRequests must be a callable or False")
-            prioritizeBuilders = config.get('prioritizeBuilders')
-            if prioritizeBuilders is not None and not callable(prioritizeBuilders):
-                raise ValueError("prioritizeBuilders must be callable")
-            changeHorizon = config.get("changeHorizon")
-            if changeHorizon is not None and not isinstance(changeHorizon, int):
-                raise ValueError("changeHorizon needs to be an int")
+                # optional
+                db_url = config.get("db_url", "sqlite:///state.sqlite")
+                db_poll_interval = config.get("db_poll_interval", None)
+                debugPassword = config.get('debugPassword')
+                manhole = config.get('manhole')
+                status = config.get('status', [])
+                projectName = config.get('projectName')
+                projectURL = config.get('projectURL')
+                buildbotURL = config.get('buildbotURL')
+                properties = config.get('properties', {})
+                buildCacheSize = config.get('buildCacheSize', None)
+                changeCacheSize = config.get('changeCacheSize', None)
+                eventHorizon = config.get('eventHorizon', 50)
+                logHorizon = config.get('logHorizon', None)
+                buildHorizon = config.get('buildHorizon', None)
+                logCompressionLimit = config.get('logCompressionLimit', 4*1024)
+                if logCompressionLimit is not None and not \
+                        isinstance(logCompressionLimit, int):
+                    raise ValueError("logCompressionLimit needs to be bool or int")
+                logCompressionMethod = config.get('logCompressionMethod', "bz2")
+                if logCompressionMethod not in ('bz2', 'gz'):
+                    raise ValueError("logCompressionMethod needs to be 'bz2', or 'gz'")
+                logMaxSize = config.get('logMaxSize')
+                if logMaxSize is not None and not \
+                        isinstance(logMaxSize, int):
+                    raise ValueError("logMaxSize needs to be None or int")
+                logMaxTailSize = config.get('logMaxTailSize')
+                if logMaxTailSize is not None and not \
+                        isinstance(logMaxTailSize, int):
+                    raise ValueError("logMaxTailSize needs to be None or int")
+                mergeRequests = config.get('mergeRequests')
+                if mergeRequests not in (None, False) and not callable(mergeRequests):
+                    raise ValueError("mergeRequests must be a callable or False")
+                prioritizeBuilders = config.get('prioritizeBuilders')
+                if prioritizeBuilders is not None and not callable(prioritizeBuilders):
+                    raise ValueError("prioritizeBuilders must be callable")
+                changeHorizon = config.get("changeHorizon")
+                if changeHorizon is not None and not isinstance(changeHorizon, int):
+                    raise ValueError("changeHorizon needs to be an int")
 
-            multiMaster = config.get("multiMaster", False)
+                multiMaster = config.get("multiMaster", False)
 
-        except KeyError:
-            log.msg("config dictionary is missing a required parameter")
-            log.msg("leaving old configuration in place")
-            raise
-
-        if "sources" in config:
-            m = ("c['sources'] is deprecated as of 0.7.6 and is no longer "
-                 "accepted in >= 0.8.0 . Please use c['change_source'] instead.")
-            raise KeyError(m)
+            except KeyError:
+                log.msg("config dictionary is missing a required parameter")
+                log.msg("leaving old configuration in place")
+                raise
 
-        if "bots" in config:
-            m = ("c['bots'] is deprecated as of 0.7.6 and is no longer "
-                 "accepted in >= 0.8.0 . Please use c['slaves'] instead.")
-            raise KeyError(m)
+            if "sources" in config:
+                m = ("c['sources'] is deprecated as of 0.7.6 and is no longer "
+                     "accepted in >= 0.8.0 . Please use c['change_source'] instead.")
+                raise KeyError(m)
 
-        slaves = config.get('slaves', [])
-        if "slaves" not in config:
-            log.msg("config dictionary must have a 'slaves' key")
-            log.msg("leaving old configuration in place")
-            raise KeyError("must have a 'slaves' key")
-
-        if changeHorizon is not None:
-            self.change_svc.changeHorizon = changeHorizon
+            if "bots" in config:
+                m = ("c['bots'] is deprecated as of 0.7.6 and is no longer "
+                     "accepted in >= 0.8.0 . Please use c['slaves'] instead.")
+                raise KeyError(m)
 
-        change_source = config.get('change_source', [])
-        if isinstance(change_source, (list, tuple)):
-            change_sources = change_source
-        else:
-            change_sources = [change_source]
+            slaves = config.get('slaves', [])
+            if "slaves" not in config:
+                log.msg("config dictionary must have a 'slaves' key")
+                log.msg("leaving old configuration in place")
+                raise KeyError("must have a 'slaves' key")
+
+            if changeHorizon is not None:
+                self.change_svc.changeHorizon = changeHorizon
+
+            change_source = config.get('change_source', [])
+            if isinstance(change_source, (list, tuple)):
+                change_sources = change_source
+            else:
+                change_sources = [change_source]
 
-        # do some validation first
-        for s in slaves:
-            assert interfaces.IBuildSlave.providedBy(s)
-            if s.slavename in ("debug", "change", "status"):
-                raise KeyError(
-                    "reserved name '%s' used for a bot" % s.slavename)
-        if config.has_key('interlocks'):
-            raise KeyError("c['interlocks'] is no longer accepted")
-        assert self.db_url is None or db_url == self.db_url, \
-                "Cannot change db_url after master has started"
-        assert db_poll_interval is None or isinstance(db_poll_interval, int), \
-               "db_poll_interval must be an integer: seconds between polls"
-        assert self.db_poll_interval is _Unset or db_poll_interval == self.db_poll_interval, \
-               "Cannot change db_poll_interval after master has started"
+            # do some validation first
+            for s in slaves:
+                assert interfaces.IBuildSlave.providedBy(s)
+                if s.slavename in ("debug", "change", "status"):
+                    raise KeyError(
+                        "reserved name '%s' used for a bot" % s.slavename)
+            if config.has_key('interlocks'):
+                raise KeyError("c['interlocks'] is no longer accepted")
+            assert self.db_url is None or db_url == self.db_url, \
+                    "Cannot change db_url after master has started"
+            assert db_poll_interval is None or isinstance(db_poll_interval, int), \
+                   "db_poll_interval must be an integer: seconds between polls"
+            assert self.db_poll_interval is _Unset or db_poll_interval == self.db_poll_interval, \
+                   "Cannot change db_poll_interval after master has started"
 
-        assert isinstance(change_sources, (list, tuple))
-        for s in change_sources:
-            assert interfaces.IChangeSource(s, None)
-        # this assertion catches c['schedulers'] = Scheduler(), since
-        # Schedulers are service.MultiServices and thus iterable.
-        errmsg = "c['schedulers'] must be a list of Scheduler instances"
-        assert isinstance(schedulers, (list, tuple)), errmsg
-        for s in schedulers:
-            assert interfaces.IScheduler(s, None), errmsg
-        assert isinstance(status, (list, tuple))
-        for s in status:
-            assert interfaces.IStatusReceiver(s, None)
-
-        slavenames = [s.slavename for s in slaves]
-        buildernames = []
-        dirnames = []
+            assert isinstance(change_sources, (list, tuple))
+            for s in change_sources:
+                assert interfaces.IChangeSource(s, None)
+            # this assertion catches c['schedulers'] = Scheduler(), since
+            # Schedulers are service.MultiServices and thus iterable.
+            errmsg = "c['schedulers'] must be a list of Scheduler instances"
+            assert isinstance(schedulers, (list, tuple)), errmsg
+            for s in schedulers:
+                assert interfaces.IScheduler(s, None), errmsg
+            assert isinstance(status, (list, tuple))
+            for s in status:
+                assert interfaces.IStatusReceiver(s, None)
 
-        # convert builders from objects to config dictionaries
-        builders_dicts = []
-        for b in builders:
-            if isinstance(b, BuilderConfig):
-                builders_dicts.append(b.getConfigDict())
-            elif type(b) is dict:
-                builders_dicts.append(b)
-            else:
-                raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b)
-        builders = builders_dicts
+            slavenames = [s.slavename for s in slaves]
+            buildernames = []
+            dirnames = []
 
-        for b in builders:
-            if b.has_key('slavename') and b['slavename'] not in slavenames:
-                raise ValueError("builder %s uses undefined slave %s" \
-                                 % (b['name'], b['slavename']))
-            for n in b.get('slavenames', []):
-                if n not in slavenames:
+            # convert builders from objects to config dictionaries
+            builders_dicts = []
+            for b in builders:
+                if isinstance(b, BuilderConfig):
+                    builders_dicts.append(b.getConfigDict())
+                elif type(b) is dict:
+                    builders_dicts.append(b)
+                else:
+                    raise ValueError("builder %s is not a BuilderConfig object (or a dict)" % b)
+            builders = builders_dicts
+
+            for b in builders:
+                if b.has_key('slavename') and b['slavename'] not in slavenames:
                     raise ValueError("builder %s uses undefined slave %s" \
-                                     % (b['name'], n))
-            if b['name'] in buildernames:
-                raise ValueError("duplicate builder name %s"
-                                 % b['name'])
-            buildernames.append(b['name'])
+                                     % (b['name'], b['slavename']))
+                for n in b.get('slavenames', []):
+                    if n not in slavenames:
+                        raise ValueError("builder %s uses undefined slave %s" \
+                                         % (b['name'], n))
+                if b['name'] in buildernames:
+                    raise ValueError("duplicate builder name %s"
+                                     % b['name'])
+                buildernames.append(b['name'])
 
-            # sanity check name (BuilderConfig does this too)
-            if b['name'].startswith("_"):
-                errmsg = ("builder names must not start with an "
-                          "underscore: " + b['name'])
-                log.err(errmsg)
-                raise ValueError(errmsg)
-
-            # Fix the dictionary with default values, in case this wasn't
-            # specified with a BuilderConfig object (which sets the same defaults)
-            b.setdefault('builddir', safeTranslate(b['name']))
-            b.setdefault('slavebuilddir', b['builddir'])
-            b.setdefault('buildHorizon', buildHorizon)
-            b.setdefault('logHorizon', logHorizon)
-            b.setdefault('eventHorizon', eventHorizon)
-            if b['builddir'] in dirnames:
-                raise ValueError("builder %s reuses builddir %s"
-                                 % (b['name'], b['builddir']))
-            dirnames.append(b['builddir'])
+                # sanity check name (BuilderConfig does this too)
+                if b['name'].startswith("_"):
+                    errmsg = ("builder names must not start with an "
+                              "underscore: " + b['name'])
+                    log.err(errmsg)
+                    raise ValueError(errmsg)
 
-        unscheduled_buildernames = buildernames[:]
-        schedulernames = []
-        for s in schedulers:
-            for b in s.listBuilderNames():
-                # Skip checks for builders in multimaster mode
-                if not multiMaster:
-                    assert b in buildernames, \
-                           "%s uses unknown builder %s" % (s, b)
-                if b in unscheduled_buildernames:
-                    unscheduled_buildernames.remove(b)
-
-            if s.name in schedulernames:
-                msg = ("Schedulers must have unique names, but "
-                       "'%s' was a duplicate" % (s.name,))
-                raise ValueError(msg)
-            schedulernames.append(s.name)
+                # Fix the dictionary with default values, in case this wasn't
+                # specified with a BuilderConfig object (which sets the same defaults)
+                b.setdefault('builddir', safeTranslate(b['name']))
+                b.setdefault('slavebuilddir', b['builddir'])
+                b.setdefault('buildHorizon', buildHorizon)
+                b.setdefault('logHorizon', logHorizon)
+                b.setdefault('eventHorizon', eventHorizon)
+                if b['builddir'] in dirnames:
+                    raise ValueError("builder %s reuses builddir %s"
+                                     % (b['name'], b['builddir']))
+                dirnames.append(b['builddir'])
 
-        # Skip the checks for builders in multimaster mode
-        if not multiMaster and unscheduled_buildernames:
-            log.msg("Warning: some Builders have no Schedulers to drive them:"
-                    " %s" % (unscheduled_buildernames,))
+            unscheduled_buildernames = buildernames[:]
+            schedulernames = []
+            for s in schedulers:
+                for b in s.listBuilderNames():
+                    # Skip checks for builders in multimaster mode
+                    if not multiMaster:
+                        assert b in buildernames, \
+                               "%s uses unknown builder %s" % (s, b)
+                    if b in unscheduled_buildernames:
+                        unscheduled_buildernames.remove(b)
 
-        # assert that all locks used by the Builds and their Steps are
-        # uniquely named.
-        lock_dict = {}
-        for b in builders:
-            for l in b.get('locks', []):
-                if isinstance(l, locks.LockAccess): # User specified access to the lock
-                    l = l.lockid
-                if lock_dict.has_key(l.name):
-                    if lock_dict[l.name] is not l:
-                        raise ValueError("Two different locks (%s and %s) "
-                                         "share the name %s"
-                                         % (l, lock_dict[l.name], l.name))
-                else:
-                    lock_dict[l.name] = l
-            # TODO: this will break with any BuildFactory that doesn't use a
-            # .steps list, but I think the verification step is more
-            # important.
-            for s in b['factory'].steps:
-                for l in s[1].get('locks', []):
+                if s.name in schedulernames:
+                    msg = ("Schedulers must have unique names, but "
+                           "'%s' was a duplicate" % (s.name,))
+                    raise ValueError(msg)
+                schedulernames.append(s.name)
+
+            # Skip the checks for builders in multimaster mode
+            if not multiMaster and unscheduled_buildernames:
+                log.msg("Warning: some Builders have no Schedulers to drive them:"
+                        " %s" % (unscheduled_buildernames,))
+
+            # assert that all locks used by the Builds and their Steps are
+            # uniquely named.
+            lock_dict = {}
+            for b in builders:
+                for l in b.get('locks', []):
                     if isinstance(l, locks.LockAccess): # User specified access to the lock
                         l = l.lockid
                     if lock_dict.has_key(l.name):
                         if lock_dict[l.name] is not l:
-                            raise ValueError("Two different locks (%s and %s)"
-                                             " share the name %s"
+                            raise ValueError("Two different locks (%s and %s) "
+                                             "share the name %s"
                                              % (l, lock_dict[l.name], l.name))
                     else:
                         lock_dict[l.name] = l
-
-        if not isinstance(properties, dict):
-            raise ValueError("c['properties'] must be a dictionary")
-
-        # slavePortnum supposed to be a strports specification
-        if type(slavePortnum) is int:
-            slavePortnum = "tcp:%d" % slavePortnum
+                # TODO: this will break with any BuildFactory that doesn't use a
+                # .steps list, but I think the verification step is more
+                # important.
+                for s in b['factory'].steps:
+                    for l in s[1].get('locks', []):
+                        if isinstance(l, locks.LockAccess): # User specified access to the lock
+                            l = l.lockid
+                        if lock_dict.has_key(l.name):
+                            if lock_dict[l.name] is not l:
+                                raise ValueError("Two different locks (%s and %s)"
+                                                 " share the name %s"
+                                                 % (l, lock_dict[l.name], l.name))
+                        else:
+                            lock_dict[l.name] = l
 
-        if check_synchronously_only:
-            return
-        # now we're committed to implementing the new configuration, so do
-        # it atomically
-        # TODO: actually, this is spread across a couple of Deferreds, so it
-        # really isn't atomic.
+            if not isinstance(properties, dict):
+                raise ValueError("c['properties'] must be a dictionary")
 
-        d = defer.succeed(None)
-
-        self.projectName = projectName
-        self.projectURL = projectURL
-        self.buildbotURL = buildbotURL
+            # slavePortnum supposed to be a strports specification
+            if type(slavePortnum) is int:
+                slavePortnum = "tcp:%d" % slavePortnum
 
-        self.properties = Properties()
-        self.properties.update(properties, self.configFileName)
+            ### ---- everything from here on down is done only on an actual (re)start
+            if checkOnly:
+                return
+
+            self.projectName = projectName
+            self.projectURL = projectURL
+            self.buildbotURL = buildbotURL
+
+            self.properties = Properties()
+            self.properties.update(properties, self.configFileName)
 
-        self.status.logCompressionLimit = logCompressionLimit
-        self.status.logCompressionMethod = logCompressionMethod
-        self.status.logMaxSize = logMaxSize
-        self.status.logMaxTailSize = logMaxTailSize
-        # Update any of our existing builders with the current log parameters.
-        # This is required so that the new value is picked up after a
-        # reconfig.
-        for builder in self.botmaster.builders.values():
-            builder.builder_status.setLogCompressionLimit(logCompressionLimit)
-            builder.builder_status.setLogCompressionMethod(logCompressionMethod)
-            builder.builder_status.setLogMaxSize(logMaxSize)
-            builder.builder_status.setLogMaxTailSize(logMaxTailSize)
-
-        if mergeRequests is not None:
-            self.botmaster.mergeRequests = mergeRequests
-        if prioritizeBuilders is not None:
-            self.botmaster.prioritizeBuilders = prioritizeBuilders
+            self.status.logCompressionLimit = logCompressionLimit
+            self.status.logCompressionMethod = logCompressionMethod
+            self.status.logMaxSize = logMaxSize
+            self.status.logMaxTailSize = logMaxTailSize
+            # Update any of our existing builders with the current log parameters.
+            # This is required so that the new value is picked up after a
+            # reconfig.
+            for builder in self.botmaster.builders.values():
+                builder.builder_status.setLogCompressionLimit(logCompressionLimit)
+                builder.builder_status.setLogCompressionMethod(logCompressionMethod)
+                builder.builder_status.setLogMaxSize(logMaxSize)
+                builder.builder_status.setLogMaxTailSize(logMaxTailSize)
 
-        self.buildCacheSize = buildCacheSize
-        self.changeCacheSize = changeCacheSize
-        self.eventHorizon = eventHorizon
-        self.logHorizon = logHorizon
-        self.buildHorizon = buildHorizon
-        self.slavePortnum = slavePortnum # TODO: move this to master.config.slavePortnum
+            if mergeRequests is not None:
+                self.botmaster.mergeRequests = mergeRequests
+            if prioritizeBuilders is not None:
+                self.botmaster.prioritizeBuilders = prioritizeBuilders
 
-        # Set up the database
-        d.addCallback(lambda res:
-                      self.loadConfig_Database(db_url, db_poll_interval))
+            self.buildCacheSize = buildCacheSize
+            self.changeCacheSize = changeCacheSize
+            self.eventHorizon = eventHorizon
+            self.logHorizon = logHorizon
+            self.buildHorizon = buildHorizon
+            self.slavePortnum = slavePortnum # TODO: move this to master.config.slavePortnum
 
-        # set up slaves
-        d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
+            # Set up the database
+            d.addCallback(lambda res:
+                          self.loadConfig_Database(db_url, db_poll_interval))
+
+            # set up slaves
+            d.addCallback(lambda res: self.loadConfig_Slaves(slaves))
 
-        # self.manhole
-        if manhole != self.manhole:
-            # changing
-            if self.manhole:
-                # disownServiceParent may return a Deferred
-                d.addCallback(lambda res: self.manhole.disownServiceParent())
-                def _remove(res):
-                    self.manhole = None
-                    return res
-                d.addCallback(_remove)
-            if manhole:
-                def _add(res):
-                    self.manhole = manhole
-                    manhole.setServiceParent(self)
-                d.addCallback(_add)
+            # self.manhole
+            if manhole != self.manhole:
+                # changing
+                if self.manhole:
+                    # disownServiceParent may return a Deferred
+                    d.addCallback(lambda res: self.manhole.disownServiceParent())
+                    def _remove(res):
+                        self.manhole = None
+                        return res
+                    d.addCallback(_remove)
+                if manhole:
+                    def _add(res):
+                        self.manhole = manhole
+                        manhole.setServiceParent(self)
+                    d.addCallback(_add)
 
-        # add/remove self.botmaster.builders to match builders. The
-        # botmaster will handle startup/shutdown issues.
-        d.addCallback(lambda res: self.loadConfig_Builders(builders))
+            # add/remove self.botmaster.builders to match builders. The
+            # botmaster will handle startup/shutdown issues.
+            d.addCallback(lambda res: self.loadConfig_Builders(builders))
 
-        d.addCallback(lambda res: self.loadConfig_status(status))
+            d.addCallback(lambda res: self.loadConfig_status(status))
 
-        # Schedulers are added after Builders in case they start right away
-        d.addCallback(lambda res:
-                      self.scheduler_manager.updateSchedulers(schedulers))
+            # Schedulers are added after Builders in case they start right away
+            d.addCallback(lambda res:
+                          self.scheduler_manager.updateSchedulers(schedulers))
 
-        # and Sources go after Schedulers for the same reason
-        d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
+            # and Sources go after Schedulers for the same reason
+            d.addCallback(lambda res: self.loadConfig_Sources(change_sources))
 
-        # debug client
-        d.addCallback(lambda res: self.loadConfig_DebugClient(debugPassword))
+            # debug client
+            d.addCallback(lambda res: self.loadConfig_DebugClient(debugPassword))
 
-        log.msg("configuration update started")
+        d.addCallback(do_load)
+
         def _done(res):
             self.readConfig = True
             log.msg("configuration update complete")
-        d.addCallback(_done)
-        d.addCallback(lambda res: self.botmaster.triggerNewBuildCheck())
-        d.addErrback(log.err)
+        # the remainder is only done if we are really loading the config
+        if not checkOnly:
+            d.addCallback(_done)
+            d.addCallback(lambda res: self.botmaster.triggerNewBuildCheck())
+            d.addErrback(log.err)
         return d
 
-    def loadDatabase(self, db_spec, db_poll_interval=None):
+    def loadDatabase(self, db_url, db_poll_interval=None):
         if self.db:
             return
 
+        self.db = connector.DBConnector(db_url, self.basedir)
+        if self.changeCacheSize:
+            self.db.setChangeCacheSize(self.changeCacheSize)
+        self.db.start()
+
         # make sure it's up to date
-        sm = DBSchemaManager(db_spec, self.basedir)
-        if not sm.is_current():
+        d = self.db.model.is_current()
+        def check_current(res):
+            if res:
+                return # good to go!
             raise exceptions.DatabaseNotReadyError, textwrap.dedent("""
                 The Buildmaster database needs to be upgraded before this version of buildbot
                 can run.  Use the following command-line
                     buildbot upgrade-master path/to/master
                 to upgrade the database, and try starting the buildmaster again.  You may want
                 to make a backup of your buildmaster before doing so.  If you are using MySQL,
                 you must specify the connector string on the upgrade-master command line:
-                    buildbot upgrade-master --db=<db-connector-string> path/to/master
+                    buildbot upgrade-master --db=<db-url> path/to/master
                 """)
+        d.addCallback(check_current)
 
-        self.db = connector.DBConnector(db_spec)
-        if self.changeCacheSize:
-            self.db.setChangeCacheSize(self.changeCacheSize)
-        self.db.start()
+        # set up the stuff that depends on the db
+        def set_up_db_dependents(r):
+            # TODO: this needs to go
+            self.botmaster.db = self.db
+            self.status.setDB(self.db)
+            self._change_subscriptions.subscribe(self.status.changeAdded)
 
-        self.botmaster.db = self.db
-        self.status.setDB(self.db)
+            self.db.subscribe_to("add-buildrequest",
+                                 self.botmaster.trigger_add_buildrequest)
 
-        self.db.subscribe_to("add-buildrequest",
-                             self.botmaster.trigger_add_buildrequest)
+            sm = SchedulerManager(self, self.db, self.change_svc)
+            self.db.subscribe_to("add-change", sm.trigger_add_change)
+            self.db.subscribe_to("modify-buildset", sm.trigger_modify_buildset)
 
-        sm = SchedulerManager(self, self.db, self.change_svc)
-        self.db.subscribe_to("add-change", sm.trigger_add_change)
-        self.db.subscribe_to("modify-buildset", sm.trigger_modify_buildset)
-
-        self.scheduler_manager = sm
-        sm.setServiceParent(self)
+            self.scheduler_manager = sm
+            sm.setServiceParent(self)
 
-        # Set db_poll_interval (perhaps to 30 seconds) if you are using
-        # multiple buildmasters that share a common database, such that the
-        # masters need to discover what each other is doing by polling the
-        # database. TODO: this will be replaced by the DBNotificationServer.
-        if db_poll_interval:
-            # it'd be nice if TimerService let us set now=False
-            t1 = TimerService(db_poll_interval, sm.trigger)
-            t1.setServiceParent(self)
-            t2 = TimerService(db_poll_interval, self.botmaster.loop.trigger)
-            t2.setServiceParent(self)
-        # adding schedulers (like when loadConfig happens) will trigger the
-        # scheduler loop at least once, which we need to jump-start things
-        # like Periodic.
+            # Set db_poll_interval (perhaps to 30 seconds) if you are using
+            # multiple buildmasters that share a common database, such that the
+            # masters need to discover what each other is doing by polling the
+            # database. TODO: this will be replaced by the DBNotificationServer.
+            if db_poll_interval:
+                # it'd be nice if TimerService let us set now=False
+                t1 = TimerService(db_poll_interval, sm.trigger)
+                t1.setServiceParent(self)
+                t2 = TimerService(db_poll_interval, self.botmaster.loop.trigger)
+                t2.setServiceParent(self)
+            # adding schedulers (like when loadConfig happens) will trigger the
+            # scheduler loop at least once, which we need to jump-start things
+            # like Periodic.
+        d.addCallback(set_up_db_dependents)
+        return d
 
     def loadConfig_Database(self, db_url, db_poll_interval):
         self.db_url = db_url
         self.db_poll_interval = db_poll_interval
-        db_spec = DBSpec.from_url(db_url, self.basedir)
-        self.loadDatabase(db_spec, db_poll_interval)
+        return self.loadDatabase(db_url, db_poll_interval)
 
     def loadConfig_Slaves(self, new_slaves):
         return self.botmaster.loadConfig_Slaves(new_slaves)
 
     def loadConfig_Sources(self, sources):
         if not sources:
             log.msg("warning: no ChangeSources specified in c['change_source']")
         # shut down any that were removed, start any that were added
@@ -1236,20 +1239,26 @@ class BuildMaster(service.MultiService):
                 if not s in self.statusTargets:
                     log.msg("adding IStatusReceiver", s)
                     s.setServiceParent(self)
                     self.statusTargets.append(s)
         d = defer.DeferredList(dl, fireOnOneErrback=1)
         d.addCallback(addNewOnes)
         return d
 
-
-    def addChange(self, change):
-        self.db.addChangeToDatabase(change)